From b96238f046f4dd3020ae5fc98835b69b6eabc1d4 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 23 Apr 2021 22:34:07 +0200 Subject: [PATCH 001/990] Doc: Link to example/README.md from /README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 57b3d717e..3880fcc4f 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,10 @@ them with operations related to the Aleph network. See [vm_connector/README.md](./vm_connector/README.md). +## Creating and running an Aleph Program + +See [examples/README.md](./examples/README.md). + --- ![aleph.im logo](https://aleph.im/assets/img/logo-wide.1832dbae.svg) From 200e38e879f64304b475f3e0d89935186f30fd71 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 28 Apr 2021 11:32:18 +0200 Subject: [PATCH 002/990] Clean: Split MicroVM stop() and teardown() --- firecracker/microvm.py | 11 ++++++++++- vm_supervisor/supervisor.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 0e3fa65a5..5eca05901 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -327,12 +327,21 @@ async def stop(self): if self.proc: self.proc.terminate() self.proc.kill() + await self.get_session().close() self.get_session.cache_clear() + + async def teardown(self): + """Stop the VM, cleanup network interface and remove data directory.""" + await self.stop() + name = f"tap{self.vm_id}" system(f"ip tuntap del {name} mode tap") + system(f"rm -fr {self.jailer_path}") + + def __del__(self): loop = asyncio.get_running_loop() - loop.create_task(self.stop()) + loop.create_task(self.teardown()) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index fffb8ed62..06207d585 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -84,7 +84,7 @@ async def run_code(request: web.Request): encoding=msg.content.code.encoding, scope=scope, ) - await vm.stop() + await vm.teardown() system(f"rm -fr {vm.jailer_path}") # TODO: Handle other content-types return web.Response(body=result, content_type="application/json") From 67981935e0bbcdc5d80428004ae5a3c060de7888 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 28 Apr 2021 15:39:15 +0200 Subject: [PATCH 003/990] Feature: Add setting FAKE_DATA for development --- examples/message_from_aleph.json | 64 ++++++++++++++++++++++++++++++++ vm_supervisor/conf.py | 2 + vm_supervisor/storage.py | 28 ++++++++++---- vm_supervisor/supervisor.py | 10 ++--- 4 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 examples/message_from_aleph.json diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json new file mode 100644 index 000000000..73fb8e484 --- /dev/null +++ b/examples/message_from_aleph.json @@ -0,0 +1,64 @@ +{ + "_id": { + "$oid": "6080402d7f44efefd611dc1e" + }, + "chain": "ETH", + "item_hash": "91c83eff3ba23d6b501a2aa3c4364ec235eb8283b6fa8ac20d235642a48791b8", + "sender": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", + "type": "POST", + "channel": "VM", + "confirmed": true, + "content": { + "type": "vm-function", + "address": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", + "content": { + "code": { + "encoding": "zip", + "entrypoint": "example_fastapi_2:app", + "ref": "7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003", + "latest_amend": true + }, + "on": { + "http": true + }, + "environment": { + "reproducible": true, + "internet": false, + "aleph_api": false + }, + "resources": { + "vcpus": 1, + "memory": 128, + "seconds": 1 + }, + "runtime": { + "ref": "5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51", + "latest_amend": true, + "comment": "Aleph Alpine Linux with Python 3.8" + }, + "data": { + "encoding": "tar.gzip", + "mount": "/mnt", + "ref": "7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003", + "latest_amend": true + }, + "export": { + "encoding": "tar.gzip", + "mount": "/mnt" + } + }, + "time": 1619017773.8950517 + }, + "item_content": "{\"type\":\"vm-function\",\"address\":\"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\",\"content\":{\"code\":{\"encoding\":\"zip\",\"entrypoint\":\"example_fastapi_2:app\",\"ref\":\"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\",\"latest_amend\":true},\"on\":{\"http\":true},\"environment\":{\"reproducible\":true,\"internet\":false,\"aleph_api\":false},\"resources\":{\"vcpus\":1,\"memory\":128,\"seconds\":1},\"runtime\":{\"ref\":\"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\",\"latest_amend\":true,\"comment\":\"Aleph Alpine Linux with Python 3.8\"},\"data\":{\"encoding\":\"tar.gzip\",\"mount\":\"/mnt\",\"ref\":\"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\",\"latest_amend\":true},\"export\":{\"encoding\":\"tar.gzip\",\"mount\":\"/mnt\"}},\"time\":1619017773.8950517}", + "item_type": "inline", + "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b", + "size": 749, + "time": 1619017773.8950577, + "confirmations": [ + { + "chain": "ETH", + "height": 12284734, + "hash": "0x67f2f3cde5e94e70615c92629c70d22dc959a118f46e9411b29659c2fce87cdc" + } + ] +} diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index ab64be101..c424aaf2f 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -29,6 +29,8 @@ class Settings: RUNTIME_CACHE: FilePath = getenv("ALEPH_RUNTIME_CACHE", join(CACHE_ROOT, "runtime")) DATA_CACHE: FilePath = getenv("ALEPH_DATA_CACHE", join(CACHE_ROOT, "data")) + FAKE_DATA: bool = getenv("ALEPH_FAKE_DATA", "false") == "true" + def update(self, **kwargs): for key, value in kwargs.items(): if hasattr(self, key): diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index 5776eb5b8..fc82e3fcb 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -40,10 +40,12 @@ async def download_file(url: str, local_path: FilePath) -> None: async def get_message(ref) -> FunctionMessage: - cache_path = FilePath(join(settings.MESSAGE_CACHE, ref) + ".json") - url = f"{settings.CONNECTOR_URL}/download/message/{ref}" - - await download_file(url, cache_path) + if settings.FAKE_DATA: + cache_path = os.path.abspath(join(__file__, '../examples/message_from_aleph.json')) + else: + cache_path = FilePath(join(settings.MESSAGE_CACHE, ref) + ".json") + url = f"{settings.CONNECTOR_URL}/download/message/{ref}" + await download_file(url, cache_path) with open(cache_path, "r") as cache_file: msg = json.load(cache_file) @@ -52,21 +54,33 @@ async def get_message(ref) -> FunctionMessage: return FunctionMessage(**msg_content) -async def get_code(ref) -> FilePath: +async def get_code_path(ref) -> FilePath: + if settings.FAKE_DATA: + return FilePath(os.path.abspath(join(__file__, + '../examples/example_fastapi_2.zip'))) + cache_path = FilePath(join(settings.CODE_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/code/{ref}" await download_file(url, cache_path) return cache_path -async def get_data(ref) -> FilePath: +async def get_data_path(ref) -> FilePath: + if settings.FAKE_DATA: + return FilePath(os.path.abspath(join(__file__, + '../examples/example_fastapi_2.zip'))) + cache_path = FilePath(join(settings.DATA_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/data/{ref}" await download_file(url, cache_path) return cache_path -async def get_runtime(ref) -> FilePath: +async def get_runtime_path(ref) -> FilePath: + if settings.FAKE_DATA + return FilePath(os.path.abspath(join(__file__, + '../runtimes/aleph-alpine-3.13-python/rootfs.ext4'))) + cache_path = FilePath(join(settings.RUNTIME_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" await download_file(url, cache_path) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 06207d585..e3436b035 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -16,7 +16,7 @@ from .conf import settings from .models import FilePath from .pool import VmPool -from .storage import get_code, get_runtime, get_message +from .storage import get_code_path, get_runtime_path, get_message, get_data_path logger = logging.getLogger(__name__) pool = VmPool() @@ -45,12 +45,12 @@ async def run_code(request: web.Request): code_ref: str = msg.content.code.ref runtime_ref: str = msg.content.runtime.ref - # data_ref: str = msg.content['data']['ref'] + data_ref: str = msg.content['data']['ref'] try: - code_path: FilePath = await get_code(code_ref) - rootfs_path: FilePath = await get_runtime(runtime_ref) - # data_path: FilePath = await get_data(data_ref) + code_path: FilePath = await get_code_path(code_ref) + rootfs_path: FilePath = await get_runtime_path(runtime_ref) + data_path: FilePath = await get_data_path(data_ref) except ClientResponseError as error: if error.status == 404: raise HTTPBadRequest(reason="Code or runtime not found") From 2fffc6a4f727440b1c21f25f5739e0f4d58e130f Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 28 Apr 2021 16:31:08 +0200 Subject: [PATCH 004/990] Fix: Error in --print-settings specification --- vm_supervisor/__main__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 4890d352e..fe830f2a9 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -54,8 +54,9 @@ def parse_args(args): ) parser.add_argument( "-p", - "--print-config", - dest="print_config", + "--print-settings", + dest="print_settings", + action="store_true", default=False, ) parser.add_argument( From 0e9c03e19ca9d9bb506931e559908d122ef35487 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 28 Apr 2021 16:31:54 +0200 Subject: [PATCH 005/990] Clean: Nicer display of settings --- vm_supervisor/conf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index c424aaf2f..898453193 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -53,10 +53,10 @@ def setup(self): os.makedirs(self.DATA_CACHE, exist_ok=True) def display(self) -> str: - result = "" - for annotation, value in self.__annotations__.items(): - result += f"{annotation} ({value.__name__}) = {getattr(self, annotation)}" - return result + return "\n".join( + f"{annotation:<17} = {getattr(self, annotation)}" + for annotation, value in self.__annotations__.items() + ) # Settings singleton From d205c138cc0a0fa5236b7dd76fe7e66c4e3f320d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 28 Apr 2021 16:57:20 +0200 Subject: [PATCH 006/990] Clean: Better type hints --- firecracker/microvm.py | 9 ++++++--- vm_connector/conf.py | 4 ++-- vm_supervisor/conf.py | 12 ++++++------ vm_supervisor/storage.py | 2 +- vm_supervisor/supervisor.py | 6 ++++-- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 5eca05901..6cccbac6a 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -8,6 +8,7 @@ from os import getuid from pathlib import Path from pwd import getpwnam +from typing import Optional import aiohttp from aiohttp import ClientResponse @@ -61,8 +62,8 @@ class MicroVM: vm_id: int use_jailer: bool firecracker_bin_path: str - jailer_bin_path: str - proc: asyncio.subprocess.Process = None + jailer_bin_path: Optional[str] + proc: Optional[asyncio.subprocess.Process] = None @property def jailer_path(self): @@ -88,7 +89,7 @@ def __init__( vm_id: int, firecracker_bin_path: str, use_jailer: bool = True, - jailer_bin_path: str = None, + jailer_bin_path: Optional[str] = None, ): self.vm_id = vm_id self.use_jailer = use_jailer @@ -143,6 +144,8 @@ async def start_firecracker(self) -> asyncio.subprocess.Process: return self.proc async def start_jailed_firecracker(self) -> asyncio.subprocess.Process: + if not self.jailer_bin_path: + raise ValueError("Jailer binary path is missing") uid = str(getpwnam("jailman").pw_uid) gid = str(getpwnam("jailman").pw_gid) logger.debug( diff --git a/vm_connector/conf.py b/vm_connector/conf.py index 1ebf4cfea..bb12dde20 100644 --- a/vm_connector/conf.py +++ b/vm_connector/conf.py @@ -5,8 +5,8 @@ class Settings: - ALEPH_SERVER: Url = getenv("ALEPH_API_SERVER", "https://api2.aleph.im") - IPFS_SERVER: Url = getenv("ALEPH_IPFS_SERVER", "https://ipfs.aleph.im/ipfs") + ALEPH_SERVER: Url = Url(getenv("ALEPH_API_SERVER", "https://api2.aleph.im")) + IPFS_SERVER: Url = Url(getenv("ALEPH_IPFS_SERVER", "https://ipfs.aleph.im/ipfs")) OFFLINE_TEST_MODE: bool = getenv("ALEPH_OFFLINE_TEST_MODE", "false") == "true" def update(self, **kwargs): diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 898453193..9ae9cfc33 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -21,13 +21,13 @@ class Settings: JAILER_PATH: str = getenv("ALEPH_JAILER_PATH", "/opt/firecracker/jailer") LINUX_PATH: str = getenv("ALEPH_LINUX_PATH", os.path.abspath("./kernels/vmlinux.bin")) - CONNECTOR_URL: Url = getenv("ALEPH_CONNECTOR_URL", "http://localhost:8000") + CONNECTOR_URL: Url = Url(getenv("ALEPH_CONNECTOR_URL", "http://localhost:8000")) - CACHE_ROOT: FilePath = getenv("ALEPH_CACHE_ROOT", "/tmp/aleph/vm_supervisor") - MESSAGE_CACHE: FilePath = getenv("ALEPH_MESSAGE_CACHE", join(CACHE_ROOT, "message")) - CODE_CACHE: FilePath = getenv("ALEPH_CODE_CACHE", join(CACHE_ROOT, "code")) - RUNTIME_CACHE: FilePath = getenv("ALEPH_RUNTIME_CACHE", join(CACHE_ROOT, "runtime")) - DATA_CACHE: FilePath = getenv("ALEPH_DATA_CACHE", join(CACHE_ROOT, "data")) + CACHE_ROOT: FilePath = FilePath(getenv("ALEPH_CACHE_ROOT", "/tmp/aleph/vm_supervisor")) + MESSAGE_CACHE: FilePath = FilePath(getenv("ALEPH_MESSAGE_CACHE", join(CACHE_ROOT, "message"))) + CODE_CACHE: FilePath = FilePath(getenv("ALEPH_CODE_CACHE", join(CACHE_ROOT, "code"))) + RUNTIME_CACHE: FilePath = FilePath(getenv("ALEPH_RUNTIME_CACHE", join(CACHE_ROOT, "runtime"))) + DATA_CACHE: FilePath = FilePath(getenv("ALEPH_DATA_CACHE", join(CACHE_ROOT, "data"))) FAKE_DATA: bool = getenv("ALEPH_FAKE_DATA", "false") == "true" diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index fc82e3fcb..6875c7ba3 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -77,7 +77,7 @@ async def get_data_path(ref) -> FilePath: async def get_runtime_path(ref) -> FilePath: - if settings.FAKE_DATA + if settings.FAKE_DATA: return FilePath(os.path.abspath(join(__file__, '../runtimes/aleph-alpine-3.13-python/rootfs.ext4'))) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index e3436b035..0daaf72c7 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -9,6 +9,7 @@ import logging import os.path from os import system +from typing import Optional from aiohttp import web, ClientResponseError, ClientConnectorError from aiohttp.web_exceptions import HTTPNotFound, HTTPBadRequest, HTTPServiceUnavailable @@ -23,7 +24,7 @@ async def index(request: web.Request): - assert request + assert request.method == "GET" return web.Response(text="Server: Aleph VM Supervisor") @@ -62,7 +63,8 @@ async def run_code(request: web.Request): kernel_image_path = settings.LINUX_PATH vm = await pool.get_a_vm( - kernel_image_path=kernel_image_path, rootfs_path=rootfs_path + kernel_image_path=kernel_image_path, + rootfs_path=rootfs_path, ) path = request.match_info["suffix"] From fd3486ca6e8dbba741dbaaefcedfad830bf301dc Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 28 Apr 2021 16:58:06 +0200 Subject: [PATCH 007/990] Feature: Add support for input data --- examples/message_from_aleph.json | 2 +- firecracker/microvm.py | 13 ++++++++---- runtimes/aleph-alpine-3.13-python/init1.py | 24 +++++++++++++++++----- vm_supervisor/models.py | 11 +++++++++- vm_supervisor/supervisor.py | 15 +++++++++++--- 5 files changed, 51 insertions(+), 14 deletions(-) diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index 73fb8e484..1ed13fba0 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -37,7 +37,7 @@ "comment": "Aleph Alpine Linux with Python 3.8" }, "data": { - "encoding": "tar.gzip", + "encoding": "zip", "mount": "/mnt", "ref": "7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003", "latest_amend": true diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 6cccbac6a..edd9c3342 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -297,20 +297,25 @@ async def unix_client_connected(*_): logger.debug("...signal from init received") async def run_code( - self, code: bytes, entrypoint: str, encoding: str = "plain", scope: dict = None + self, code: bytes, entrypoint: str, input_data: bytes = b"", + encoding: str = "plain", scope: dict = None ): scope = scope or {} reader, writer = await asyncio.open_unix_connection(path=self.vsock_path) + code_for_json: str if encoding == Encoding.zip: - code = base64.b64encode(code).decode() + code_for_json = base64.b64encode(code).decode() elif encoding == Encoding.plain: - code = code.decode() + code_for_json = code.decode() else: raise ValueError(f"Unknown encoding '{encoding}'") + input_data_b64: str = base64.b64encode(input_data).decode() + msg = { - "code": code, + "code": code_for_json, + "input_data": input_data_b64, "entrypoint": entrypoint, "encoding": encoding, "scope": scope, diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 038318484..5e7d74cbe 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -10,6 +10,7 @@ from os import system from io import StringIO from contextlib import redirect_stdout +from typing import Optional, Dict, Any s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) s.bind((socket.VMADDR_CID_ANY, 52)) @@ -28,11 +29,13 @@ class Encoding: zip = "zip" -async def run_python_code_http(code: str, entrypoint: str, encoding: str, scope: dict): +async def run_python_code_http(code: str, input_data: Optional[str], + entrypoint: str, encoding: str, scope: dict): if encoding == Encoding.zip: # Unzip in /opt and import the entrypoint from there decoded: bytes = b64decode(code) open("/opt/archive.zip", "wb").write(decoded) + del decoded os.system("unzip /opt/archive.zip -d /opt") sys.path.append("/opt") module_name, app_name = entrypoint.split(":", 1) @@ -40,18 +43,26 @@ async def run_python_code_http(code: str, entrypoint: str, encoding: str, scope: app = getattr(module, app_name) elif encoding == Encoding.plain: # Execute the code and extract the entrypoint - locals = {} + locals: Dict[str, Any] = {} exec(code, globals(), locals) app = locals[entrypoint] else: raise ValueError(f"Unknown encoding '{encoding}'") + if input_data: + # Unzip in /data + decoded_data: bytes = b64decode(code) + open("/opt/input.zip", "wb").write(decoded_data) + del decoded_data + os.makedirs("/input", exist_ok=True) + os.system("unzip /opt/input.zip -d /input") + with StringIO() as buf, redirect_stdout(buf): # Execute in the same process, saves ~20ms than a subprocess async def receive(): pass - send_queue = asyncio.Queue() + send_queue: asyncio.Queue = asyncio.Queue() async def send(dico): await send_queue.put(dico) @@ -66,9 +77,10 @@ async def send(dico): while True: client, addr = s.accept() data = client.recv(1000_1000) # Max 1 Mo - print("CID: {} port:{} data: {}".format(addr[0], addr[1], data)) + print("CID: {} port:{} data: {}".format(addr[0], addr[1], data.decode())) msg = data.decode().strip() + del data print("msg", [msg]) if msg == "halt": @@ -87,13 +99,15 @@ async def send(dico): # Python msg_ = json.loads(msg) code = msg_["code"] + input_data = msg_.get("input_data") entrypoint = msg_["entrypoint"] scope = msg_["scope"] encoding = msg_["encoding"] try: headers, body, output = asyncio.get_event_loop().run_until_complete( run_python_code_http( - code, entrypoint=entrypoint, encoding=encoding, scope=scope + code, input_data=input_data, + entrypoint=entrypoint, encoding=encoding, scope=scope ) ) client.send(body["body"]) diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index 13972854e..b9840d80d 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import NewType +from typing import NewType, Optional from pydantic import BaseModel @@ -9,6 +9,7 @@ class Encoding(str, Enum): plain = "plain" zip = "zip" + targz = "tar.gzip" class CodeContent(BaseModel): @@ -18,6 +19,13 @@ class CodeContent(BaseModel): latest_amend: bool = True +class DataContent(BaseModel): + encoding: Encoding + mount: str + ref: str + latest_amend: bool = True + + class FunctionTriggers(BaseModel): http: bool @@ -42,6 +50,7 @@ class FunctionRuntime(BaseModel): class FunctionContent(BaseModel): code: CodeContent + data: Optional[DataContent] on: FunctionTriggers environment: FunctionEnvironment resources: FunctionResources diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 0daaf72c7..6d6ca5199 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -46,12 +46,12 @@ async def run_code(request: web.Request): code_ref: str = msg.content.code.ref runtime_ref: str = msg.content.runtime.ref - data_ref: str = msg.content['data']['ref'] + data_ref: Optional[str] = msg.content.data.ref if msg.content.data else None try: code_path: FilePath = await get_code_path(code_ref) rootfs_path: FilePath = await get_runtime_path(runtime_ref) - data_path: FilePath = await get_data_path(data_ref) + data_path: Optional[FilePath] = await get_data_path(data_ref) if data_ref else None except ClientResponseError as error: if error.status == 404: raise HTTPBadRequest(reason="Code or runtime not found") @@ -80,9 +80,18 @@ async def run_code(request: web.Request): "headers": request.raw_headers, } with open(code_path, "rb") as code_file: + + input_data: bytes + if data_path: + with open(data_path, "rb") as data_file: + input_data = data_file.read() + else: + input_data = b'' + result = await vm.run_code( - code_file.read(), + code=code_file.read(), entrypoint=msg.content.code.entrypoint, + input_data=input_data, encoding=msg.content.code.encoding, scope=scope, ) From aa78721cf12ffef7ae7128e9873525756d51f81e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 28 Apr 2021 16:58:30 +0200 Subject: [PATCH 008/990] Fix: Wrong paths used for FAKE_DATA --- vm_supervisor/storage.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index 6875c7ba3..4277b3ebb 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -18,6 +18,7 @@ async def download_file(url: str, local_path: FilePath) -> None: + # TODO: Limit max size of download to the message specification if isfile(local_path): logger.debug(f"File already exists: {local_path}") else: @@ -41,7 +42,8 @@ async def download_file(url: str, local_path: FilePath) -> None: async def get_message(ref) -> FunctionMessage: if settings.FAKE_DATA: - cache_path = os.path.abspath(join(__file__, '../examples/message_from_aleph.json')) + cache_path = os.path.abspath(join(__file__, + '../../examples/message_from_aleph.json')) else: cache_path = FilePath(join(settings.MESSAGE_CACHE, ref) + ".json") url = f"{settings.CONNECTOR_URL}/download/message/{ref}" @@ -57,7 +59,7 @@ async def get_message(ref) -> FunctionMessage: async def get_code_path(ref) -> FilePath: if settings.FAKE_DATA: return FilePath(os.path.abspath(join(__file__, - '../examples/example_fastapi_2.zip'))) + '../../examples/example_fastapi_2.zip'))) cache_path = FilePath(join(settings.CODE_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/code/{ref}" @@ -68,7 +70,7 @@ async def get_code_path(ref) -> FilePath: async def get_data_path(ref) -> FilePath: if settings.FAKE_DATA: return FilePath(os.path.abspath(join(__file__, - '../examples/example_fastapi_2.zip'))) + '../../examples/example_fastapi_2.zip'))) cache_path = FilePath(join(settings.DATA_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/data/{ref}" @@ -79,7 +81,7 @@ async def get_data_path(ref) -> FilePath: async def get_runtime_path(ref) -> FilePath: if settings.FAKE_DATA: return FilePath(os.path.abspath(join(__file__, - '../runtimes/aleph-alpine-3.13-python/rootfs.ext4'))) + '../../runtimes/aleph-alpine-3.13-python/rootfs.ext4'))) cache_path = FilePath(join(settings.RUNTIME_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" From 39ed40eeeebe826a41c2512f78d970006b472235 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 10 May 2021 18:49:18 +0200 Subject: [PATCH 009/990] Add a host-guest API for VMs without internet requirement (#10) The guest API forwards API calls to an Aleph node, without the VM needing a network connection. More functionalities are expected to be added to the API in the future. Code within a VM can use `aleph-client` directly. Building the runtime requires a version of `aleph-client` without the package `secp256k1`. A branch is currently used for that purpose and NULS1 is temporarily not supported. The API between the host and VMs now uses `msgpack` for better binary format handling. Socat is used to expose Vsock on Unix socket. --- examples/README.md | 2 +- examples/example_fastapi_2/__init__.py | 24 ++- examples/example_fastapi_2/__main__.py | 20 -- firecracker/microvm.py | 52 ++--- guest_api/__init__.py | 0 guest_api/__main__.py | 30 +++ .../create_disk_image.sh | 13 +- runtimes/aleph-alpine-3.13-python/init0.sh | 28 +-- runtimes/aleph-alpine-3.13-python/init1.py | 150 ++++++++----- vm_connector/conf.py | 1 + vm_supervisor/conf.py | 24 ++- vm_supervisor/pool.py | 64 ++++-- vm_supervisor/storage.py | 37 ++-- vm_supervisor/supervisor.py | 142 ++++++------ vm_supervisor/vm/__init__.py | 4 +- vm_supervisor/vm/firecracker_microvm.py | 203 +++++++++++++++--- 16 files changed, 528 insertions(+), 266 deletions(-) delete mode 100644 examples/example_fastapi_2/__main__.py create mode 100644 guest_api/__init__.py create mode 100644 guest_api/__main__.py diff --git a/examples/README.md b/examples/README.md index 269961af2..dbdda3217 100644 --- a/examples/README.md +++ b/examples/README.md @@ -23,7 +23,7 @@ app = FastAPI() @app.get("/") -def read_root(): +def index(): return {"Hello": "World"} diff --git a/examples/example_fastapi_2/__init__.py b/examples/example_fastapi_2/__init__.py index 36d47d93d..52daaa634 100644 --- a/examples/example_fastapi_2/__init__.py +++ b/examples/example_fastapi_2/__init__.py @@ -1,13 +1,33 @@ +import logging from typing import Optional +logging.basicConfig(level=logging.DEBUG) +from aleph_client.asynchronous import get_messages + from fastapi import FastAPI app = FastAPI() +async def get_data_http(): + return "Have a look at /messages" + + @app.get("/") -def read_root(): - return {"Hello": "World"} +async def index(): + data = await get_data_http() + return { + "Example": "example_fastapi_2", + "endpoints": ["/messages", "/run/{item_id}"], + } + + +@app.get("/messages") +async def read_aleph_messages(): + data = await get_messages( + hashes=["f246f873c3e0f637a15c566e7a465d2ecbb83eaa024d54ccb8fb566b549a929e"] + ) + return {"Messages": data} @app.get("/run/{item_id}") diff --git a/examples/example_fastapi_2/__main__.py b/examples/example_fastapi_2/__main__.py deleted file mode 100644 index 36d47d93d..000000000 --- a/examples/example_fastapi_2/__main__.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import Optional - -from fastapi import FastAPI - -app = FastAPI() - - -@app.get("/") -def read_root(): - return {"Hello": "World"} - - -@app.get("/run/{item_id}") -def read_item(item_id: str, q: Optional[str] = None): - return {"pyz item_id": item_id, "q": q} - - -@app.post("/run/{item_id}") -def read_item_post(item_id: str, q: Optional[str] = None): - return {"pyz item_id_post": item_id, "q": q} diff --git a/firecracker/microvm.py b/firecracker/microvm.py index edd9c3342..a5143c1ea 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -1,5 +1,4 @@ import asyncio -import base64 import json import logging import os.path @@ -37,7 +36,7 @@ def default(self, obj): def system(command): - logger.debug(command) + logger.debug(f"shell {command}") return os.system(command) @@ -101,7 +100,7 @@ def get_session(self) -> aiohttp.ClientSession: conn = aiohttp.UnixConnector(path=self.socket_path) return aiohttp.ClientSession(connector=conn) - def cleanup_jailer(self): + def prepare_jailer(self): system(f"rm -fr {self.jailer_path}") # system(f"rm -fr {self.jailer_path}/run/") @@ -268,8 +267,6 @@ async def start_instance(self): session = self.get_session() response = await session.put("http://localhost/actions", json=data) response.raise_for_status() - logger.debug(response) - logger.debug(await response.text()) async def print_logs(self): while not self.proc: @@ -281,6 +278,16 @@ async def print_logs(self): else: await asyncio.sleep(0.001) + async def print_logs_stderr(self): + while not self.proc: + await asyncio.sleep(0.01) # Todo: Use signal here + while True: + stdout = await self.proc.stderr.readline() + if stdout: + print(stdout.decode().strip()) + else: + await asyncio.sleep(0.001) + async def wait_for_init(self): """Wait for a connection from the init in the VM""" logger.debug("Waiting for init...") @@ -296,41 +303,6 @@ async def unix_client_connected(*_): await queue.get() logger.debug("...signal from init received") - async def run_code( - self, code: bytes, entrypoint: str, input_data: bytes = b"", - encoding: str = "plain", scope: dict = None - ): - scope = scope or {} - reader, writer = await asyncio.open_unix_connection(path=self.vsock_path) - - code_for_json: str - if encoding == Encoding.zip: - code_for_json = base64.b64encode(code).decode() - elif encoding == Encoding.plain: - code_for_json = code.decode() - else: - raise ValueError(f"Unknown encoding '{encoding}'") - - input_data_b64: str = base64.b64encode(input_data).decode() - - msg = { - "code": code_for_json, - "input_data": input_data_b64, - "entrypoint": entrypoint, - "encoding": encoding, - "scope": scope, - } - writer.write(("CONNECT 52\n" + JSONBytesEncoder().encode(msg) + "\n").encode()) - await writer.drain() - - ack = await reader.readline() - logger.debug(f"ack={ack.decode()}") - response = await reader.read() - logger.debug(f"response= <<<\n{response.decode()}>>>") - writer.close() - await writer.wait_closed() - return response - async def stop(self): if self.proc: self.proc.terminate() diff --git a/guest_api/__init__.py b/guest_api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/guest_api/__main__.py b/guest_api/__main__.py new file mode 100644 index 000000000..2107ff328 --- /dev/null +++ b/guest_api/__main__.py @@ -0,0 +1,30 @@ +import logging + +import aiohttp +from aiohttp import web + +logging.basicConfig(level=logging.DEBUG) + +ALEPH_API_SERVER = "https://api2.aleph.im/" + + +async def proxy(request): + path = request.match_info.get('tail') + url = ALEPH_API_SERVER + path + + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + data = await response.read() + return web.Response(body=data, + status=response.status, + content_type=response.content_type) + + +def run_guest_api(unix_socket_path): + app = web.Application() + app.router.add_route(method='GET', path='/{tail:.*}', handler=proxy) + web.run_app(app=app, path=unix_socket_path) + + +if __name__ == '__main__': + run_guest_api("/tmp/guest-api") diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index 671565cae..7c602b679 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -1,10 +1,12 @@ #!/bin/sh +umount /mnt/rootfs + set -euf curl -fsSL -o ./alpine-miniroot.tgz https://dl-cdn.alpinelinux.org/alpine/v3.13/releases/x86_64/alpine-minirootfs-3.13.3-x86_64.tar.gz -dd if=/dev/zero of=./rootfs.ext4 bs=1M count=100 +dd if=/dev/zero of=./rootfs.ext4 bs=1M count=500 mkfs.ext4 ./rootfs.ext4 mkdir -p /mnt/rootfs mount ./rootfs.ext4 /mnt/rootfs @@ -17,11 +19,16 @@ apk update apk add util-linux apk add python3 apk add openssh-server -apk add py3-pip +apk add socat +apk add py3-pip +apk add py3-aiohttp py3-msgpack pip install fastapi -#echo -e "toor\ntoor" | passwd root +apk add git pkgconf gcc py3-wheel python3-dev musl-dev py3-cffi libffi-dev autoconf automake libtool make +pip install -e git+https://github.com/aleph-im/aleph-client@hoh-remove-secp256k1#egg=aleph-client coincurve==15.0.0 + +echo -e "toor\ntoor" | passwd root mkdir -p /overlay diff --git a/runtimes/aleph-alpine-3.13-python/init0.sh b/runtimes/aleph-alpine-3.13-python/init0.sh index db66b5146..a665aeb83 100644 --- a/runtimes/aleph-alpine-3.13-python/init0.sh +++ b/runtimes/aleph-alpine-3.13-python/init0.sh @@ -2,7 +2,7 @@ set -euf -echo "=== My Bash RC ===" +echo "init0.sh is launching" mount -t proc proc /proc -o nosuid,noexec,nodev @@ -17,10 +17,9 @@ pivot_root /mnt /mnt/rom mount --move /rom/proc /proc mount --move /rom/dev /dev -echo "Mounts" - -ls / -ls /dev +#echo "Mounts" +#ls / +#ls /dev mkdir -p /dev/pts mkdir -p /dev/shm @@ -32,17 +31,22 @@ mount -t devpts devpts /dev/pts -o mode=0620,gid=5,nosuid,noexec mount -t tmpfs shm /dev/shm -omode=1777,nosuid,nodev # TODO: Move in init1 -ip addr add 172.16.0.2/24 dev eth0 -ip link set eth0 up -ip route add default via 172.16.0.1 dev eth0 -ip addr +#ip addr add 172.0.5.2/24 dev eth0 +#ip link set eth0 up +#ip route add default via 172.0.5.1 dev eth0 +#ip addr echo "Net up" +#cat /proc/sys/kernel/random/entropy_avail + # TODO: Move in init1 -#/usr/sbin/sshd -E /var/log/sshd & -# -#echo "SSH UP" +/usr/sbin/sshd -E /var/log/sshd & +echo "SSH UP" + +echo "Setup socat" +socat UNIX-LISTEN:/tmp/socat-socket,fork,reuseaddr VSOCK-CONNECT:2:53 & +echo "Socat ready" # Replace this script with the manager exec /root/init1.py diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 5e7d74cbe..1573f1b92 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -1,41 +1,57 @@ #!/usr/bin/python3 + import asyncio -import json import os import socket import subprocess import sys import traceback -from base64 import b64decode -from os import system -from io import StringIO from contextlib import redirect_stdout -from typing import Optional, Dict, Any +from dataclasses import dataclass +from io import StringIO +from os import system +from shutil import make_archive +from typing import Optional, Dict, Any, Tuple, Iterator + +import msgpack + + +class Encoding: + plain = "plain" + zip = "zip" + + +@dataclass +class RunCodePayload: + code: bytes + input_data: Optional[bytes] + entrypoint: str + encoding: str + scope: Dict + +# Open a socket to receive instructions from the host s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) s.bind((socket.VMADDR_CID_ANY, 52)) s.listen() -# Send we are ready +# Send the host that we are ready s0 = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) s0.connect((2, 52)) s0.close() -print("INIT1 READY") +# Configure aleph-client to use the guest API +os.environ["ALEPH_API_UNIX_SOCKET"] = "/tmp/socat-socket" - -class Encoding: - plain = "plain" - zip = "zip" +print("init1.py is launching") -async def run_python_code_http(code: str, input_data: Optional[str], - entrypoint: str, encoding: str, scope: dict): +async def run_python_code_http(code: bytes, input_data: Optional[bytes], + entrypoint: str, encoding: str, scope: dict + ) -> Tuple[Dict, Dict, str, Optional[bytes]]: if encoding == Encoding.zip: # Unzip in /opt and import the entrypoint from there - decoded: bytes = b64decode(code) - open("/opt/archive.zip", "wb").write(decoded) - del decoded + open("/opt/archive.zip", "wb").write(code) os.system("unzip /opt/archive.zip -d /opt") sys.path.append("/opt") module_name, app_name = entrypoint.split(":", 1) @@ -51,11 +67,9 @@ async def run_python_code_http(code: str, input_data: Optional[str], if input_data: # Unzip in /data - decoded_data: bytes = b64decode(code) - open("/opt/input.zip", "wb").write(decoded_data) - del decoded_data - os.makedirs("/input", exist_ok=True) - os.system("unzip /opt/input.zip -d /input") + open("/opt/input.zip", "wb").write(input_data) + os.makedirs("/data", exist_ok=True) + os.system("unzip /opt/input.zip -d /data") with StringIO() as buf, redirect_stdout(buf): # Execute in the same process, saves ~20ms than a subprocess @@ -67,52 +81,84 @@ async def receive(): async def send(dico): await send_queue.put(dico) + # TODO: Better error handling await app(scope, receive, send) - headers = await send_queue.get() - body = await send_queue.get() + headers: Dict = await send_queue.get() + body: Dict = await send_queue.get() output = buf.getvalue() - return headers, body, output + os.makedirs("/data", exist_ok=True) + open('/data/hello.txt', 'w').write("Hello !") -while True: - client, addr = s.accept() - data = client.recv(1000_1000) # Max 1 Mo - print("CID: {} port:{} data: {}".format(addr[0], addr[1], data.decode())) + output_data: bytes + if os.listdir('/data'): + make_archive("/opt/output", 'zip', "/data") + with open("/opt/output.zip", "rb") as output_zipfile: + output_data = output_zipfile.read() + else: + output_data = b'' + + return headers, body, output, output_data - msg = data.decode().strip() - del data - print("msg", [msg]) - if msg == "halt": +def process_instruction(instruction: bytes) -> Iterator[bytes]: + if instruction == b"halt": system("sync") - client.send(b"STOP\n") + yield b"STOP\n" sys.exit() - elif msg.startswith("!"): - # Shell - msg = msg[1:] + elif instruction.startswith(b"!"): + # Execute shell commands in the form `!ls /` + msg = instruction[1:].decode() try: - output = subprocess.check_output(msg, stderr=subprocess.STDOUT, shell=True) - client.send(output) + process_output = subprocess.check_output(msg, stderr=subprocess.STDOUT, shell=True) + yield process_output except subprocess.CalledProcessError as error: - client.send(str(error).encode() + b"\n" + error.output) + yield str(error).encode() + b"\n" + error.output else: # Python - msg_ = json.loads(msg) - code = msg_["code"] - input_data = msg_.get("input_data") - entrypoint = msg_["entrypoint"] - scope = msg_["scope"] - encoding = msg_["encoding"] + msg_ = msgpack.loads(instruction, raw=False) + payload = RunCodePayload(**msg_) + try: - headers, body, output = asyncio.get_event_loop().run_until_complete( + headers: Dict + body: Dict + output: str + output_data: Optional[bytes] + + headers, body, output, output_data = asyncio.get_event_loop().run_until_complete( run_python_code_http( - code, input_data=input_data, - entrypoint=entrypoint, encoding=encoding, scope=scope + payload.code, input_data=payload.input_data, + entrypoint=payload.entrypoint, encoding=payload.encoding, scope=payload.scope ) ) - client.send(body["body"]) + result = { + "headers": headers, + "body": body, + "output": output, + "output_data": output_data, + } + yield msgpack.dumps(result, use_bin_type=True) except Exception as error: - client.send(str(error).encode() + str(traceback.format_exc()).encode()) + yield msgpack.dumps({ + "error": str(error), + "traceback": str(traceback.format_exc()), + "output": output + }) + + +def main(): + while True: + client, addr = s.accept() + data = client.recv(1000_1000) # Max 1 Mo + print("CID: {} port:{} data: {}".format(addr[0], addr[1], len(data))) + + print("Init received msg <<<\n\n", data, "\n\n>>>") + for result in process_instruction(instruction=data): + client.send(result) + + print("...DONE") + client.close() + - print("...DONE") - client.close() +if __name__ == '__main__': + main() diff --git a/vm_connector/conf.py b/vm_connector/conf.py index bb12dde20..cc3c6ffba 100644 --- a/vm_connector/conf.py +++ b/vm_connector/conf.py @@ -22,5 +22,6 @@ def display(self) -> str: result += f"{annotation} ({value.__name__}) = {getattr(self, annotation)}" return result + # Settings singleton settings = Settings() diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 9ae9cfc33..510d77043 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -19,15 +19,27 @@ class Settings: "ALEPH_FIRECRACKER_PATH", "/opt/firecracker/firecracker" ) JAILER_PATH: str = getenv("ALEPH_JAILER_PATH", "/opt/firecracker/jailer") - LINUX_PATH: str = getenv("ALEPH_LINUX_PATH", os.path.abspath("./kernels/vmlinux.bin")) + LINUX_PATH: str = getenv( + "ALEPH_LINUX_PATH", os.path.abspath("./kernels/vmlinux.bin") + ) CONNECTOR_URL: Url = Url(getenv("ALEPH_CONNECTOR_URL", "http://localhost:8000")) - CACHE_ROOT: FilePath = FilePath(getenv("ALEPH_CACHE_ROOT", "/tmp/aleph/vm_supervisor")) - MESSAGE_CACHE: FilePath = FilePath(getenv("ALEPH_MESSAGE_CACHE", join(CACHE_ROOT, "message"))) - CODE_CACHE: FilePath = FilePath(getenv("ALEPH_CODE_CACHE", join(CACHE_ROOT, "code"))) - RUNTIME_CACHE: FilePath = FilePath(getenv("ALEPH_RUNTIME_CACHE", join(CACHE_ROOT, "runtime"))) - DATA_CACHE: FilePath = FilePath(getenv("ALEPH_DATA_CACHE", join(CACHE_ROOT, "data"))) + CACHE_ROOT: FilePath = FilePath( + getenv("ALEPH_CACHE_ROOT", "/tmp/aleph/vm_supervisor") + ) + MESSAGE_CACHE: FilePath = FilePath( + getenv("ALEPH_MESSAGE_CACHE", join(CACHE_ROOT, "message")) + ) + CODE_CACHE: FilePath = FilePath( + getenv("ALEPH_CODE_CACHE", join(CACHE_ROOT, "code")) + ) + RUNTIME_CACHE: FilePath = FilePath( + getenv("ALEPH_RUNTIME_CACHE", join(CACHE_ROOT, "runtime")) + ) + DATA_CACHE: FilePath = FilePath( + getenv("ALEPH_DATA_CACHE", join(CACHE_ROOT, "data")) + ) FAKE_DATA: bool = getenv("ALEPH_FAKE_DATA", "false") == "true" diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index 6642fc938..beaf2d0d1 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -1,35 +1,55 @@ import asyncio +from typing import Dict, List from firecracker.microvm import MicroVM from vm_supervisor.conf import settings -from vm_supervisor.vm.firecracker_microvm import start_new_vm +from vm_supervisor.models import FunctionMessage +from vm_supervisor.vm.firecracker_microvm import ( + AlephFirecrackerVM, + AlephFirecrackerResources, +) -class VmPool: - """Pool of VMs pre-allocated in order to decrease response time. - The counter is used by the VMs to set their tap interface name and the corresponding - IPv4 subnet. - """ +# class VmPool: +# """Pool of VMs pre-allocated in order to decrease response time. +# The counter is used by the VMs to set their tap interface name and the corresponding +# IPv4 subnet. +# """ +# +# queue: asyncio.Queue +# counter: int # Used for network interfaces +# +# def __init__(self): +# self.queue = asyncio.Queue() +# self.counter = settings.VM_ID_START_INDEX +# +# async def provision(self, kernel_image_path, rootfs_path): +# self.counter += 1 +# vm = await start_new_vm( +# vm_id=self.counter, +# kernel_image_path=kernel_image_path, +# rootfs_path=rootfs_path, +# ) +# await self.queue.put(vm) +# return vm +# +# async def get_a_vm(self, kernel_image_path, rootfs_path) -> MicroVM: +# loop = asyncio.get_event_loop() +# loop.create_task(self.provision(kernel_image_path, rootfs_path)) +# # Return the first VM from the pool +# return await self.queue.get() + - queue: asyncio.Queue +class VmPool: counter: int # Used for network interfaces def __init__(self): - self.queue = asyncio.Queue() self.counter = settings.VM_ID_START_INDEX - async def provision(self, kernel_image_path, rootfs_path): - self.counter += 1 - vm = await start_new_vm( - vm_id=self.counter, - kernel_image_path=kernel_image_path, - rootfs_path=rootfs_path, - ) - await self.queue.put(vm) + async def get_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: + vm_resources = AlephFirecrackerResources(message) + await vm_resources.download_all() + vm = AlephFirecrackerVM(vm_id=self.counter, resources=vm_resources) + await vm.setup() + await vm.start() return vm - - async def get_a_vm(self, kernel_image_path, rootfs_path) -> MicroVM: - loop = asyncio.get_event_loop() - loop.create_task(self.provision(kernel_image_path, rootfs_path)) - # Return the first VM from the pool - return await self.queue.get() diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index 4277b3ebb..7ce5b67ab 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -7,7 +7,8 @@ import json import logging import os -from os.path import isfile, join +from os.path import isfile, join, abspath +from shutil import make_archive import aiohttp @@ -40,10 +41,11 @@ async def download_file(url: str, local_path: FilePath) -> None: raise -async def get_message(ref) -> FunctionMessage: +async def get_message(ref: str) -> FunctionMessage: if settings.FAKE_DATA: - cache_path = os.path.abspath(join(__file__, - '../../examples/message_from_aleph.json')) + cache_path = os.path.abspath( + join(__file__, "../../examples/message_from_aleph.json") + ) else: cache_path = FilePath(join(settings.MESSAGE_CACHE, ref) + ".json") url = f"{settings.CONNECTOR_URL}/download/message/{ref}" @@ -56,10 +58,15 @@ async def get_message(ref) -> FunctionMessage: return FunctionMessage(**msg_content) -async def get_code_path(ref) -> FilePath: +async def get_code_path(ref: str) -> FilePath: if settings.FAKE_DATA: - return FilePath(os.path.abspath(join(__file__, - '../../examples/example_fastapi_2.zip'))) + root_dir = abspath(join(__file__, "../../examples/")) + archive_path = join(root_dir, "example_fastapi_2") + # app_dir = abspath(join(__file__, "../../examples/visit_counter")) + make_archive( + archive_path, "zip", root_dir=root_dir, base_dir="example_fastapi_2" + ) + return FilePath(f"{archive_path}.zip") cache_path = FilePath(join(settings.CODE_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/code/{ref}" @@ -67,10 +74,11 @@ async def get_code_path(ref) -> FilePath: return cache_path -async def get_data_path(ref) -> FilePath: +async def get_data_path(ref: str) -> FilePath: if settings.FAKE_DATA: - return FilePath(os.path.abspath(join(__file__, - '../../examples/example_fastapi_2.zip'))) + data_dir = abspath(join(__file__, "../../examples/data")) + make_archive(data_dir, "zip", data_dir) + return FilePath(f"{data_dir}.zip") cache_path = FilePath(join(settings.DATA_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/data/{ref}" @@ -78,10 +86,13 @@ async def get_data_path(ref) -> FilePath: return cache_path -async def get_runtime_path(ref) -> FilePath: +async def get_runtime_path(ref: str) -> FilePath: if settings.FAKE_DATA: - return FilePath(os.path.abspath(join(__file__, - '../../runtimes/aleph-alpine-3.13-python/rootfs.ext4'))) + return FilePath( + os.path.abspath( + join(__file__, "../../runtimes/aleph-alpine-3.13-python/rootfs.ext4") + ) + ) cache_path = FilePath(join(settings.RUNTIME_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 6d6ca5199..7ec1b1011 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -5,37 +5,34 @@ At it's core, it is currently an asynchronous HTTP server using aiohttp, but this may evolve in the future. """ - import logging -import os.path -from os import system -from typing import Optional +from multiprocessing import set_start_method +import msgpack from aiohttp import web, ClientResponseError, ClientConnectorError -from aiohttp.web_exceptions import HTTPNotFound, HTTPBadRequest, HTTPServiceUnavailable +from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable +from msgpack import UnpackValueError from .conf import settings -from .models import FilePath +from .models import FilePath, FunctionMessage from .pool import VmPool -from .storage import get_code_path, get_runtime_path, get_message, get_data_path +from .storage import get_message logger = logging.getLogger(__name__) pool = VmPool() +set_start_method("spawn") + async def index(request: web.Request): assert request.method == "GET" return web.Response(text="Server: Aleph VM Supervisor") -async def run_code(request: web.Request): - """ - Execute the code corresponding to the 'code id' in the path. - """ - msg_ref: str = request.match_info["ref"] - +async def try_get_message(ref: str) -> FunctionMessage: + # Get the message or raise an aiohttp HTTP error try: - msg = await get_message(msg_ref) + return await get_message(ref) except ClientConnectorError: raise HTTPServiceUnavailable(reason="Aleph Connector unavailable") except ClientResponseError as error: @@ -44,61 +41,90 @@ async def run_code(request: web.Request): else: raise - code_ref: str = msg.content.code.ref - runtime_ref: str = msg.content.runtime.ref - data_ref: Optional[str] = msg.content.data.ref if msg.content.data else None - - try: - code_path: FilePath = await get_code_path(code_ref) - rootfs_path: FilePath = await get_runtime_path(runtime_ref) - data_path: Optional[FilePath] = await get_data_path(data_ref) if data_ref else None - except ClientResponseError as error: - if error.status == 404: - raise HTTPBadRequest(reason="Code or runtime not found") - else: - raise - - logger.debug("Got files") - - kernel_image_path = settings.LINUX_PATH - - vm = await pool.get_a_vm( - kernel_image_path=kernel_image_path, - rootfs_path=rootfs_path, - ) +def build_asgi_scope(request: web.Request): path = request.match_info["suffix"] if not path.startswith("/"): path = "/" + path - logger.debug(f"Using vm={vm.vm_id}") - scope = { + return { "type": "http", "path": path, "method": request.method, "query_string": request.query_string, "headers": request.raw_headers, } - with open(code_path, "rb") as code_file: - input_data: bytes - if data_path: - with open(data_path, "rb") as data_file: - input_data = data_file.read() - else: - input_data = b'' - result = await vm.run_code( - code=code_file.read(), - entrypoint=msg.content.code.entrypoint, +def load_file_content(path: FilePath) -> bytes: + if path: + with open(path, "rb") as fd: + return fd.read() + else: + return b"" + + +async def run_code(request: web.Request): + """ + Execute the code corresponding to the 'code id' in the path. + """ + message_ref: str = request.match_info["ref"] + message = await try_get_message(message_ref) + + # vm_resources = AlephFirecrackerResources(message) + # + # try: + # await vm_resources.download_all() + # except ClientResponseError as error: + # if error.status == 404: + # raise HTTPBadRequest(reason="Code, runtime or data not found") + # else: + # raise + + vm = await pool.get_a_vm(message) + await vm.start_guest_api() + logger.debug(f"Using vm={vm.vm_id}") + + scope = build_asgi_scope(request) + + code: bytes = load_file_content(vm.resources.code_path) + input_data: bytes = load_file_content(vm.resources.data_path) + + try: + result_raw: bytes = await vm.run_code( + code=code, + entrypoint=message.content.code.entrypoint, input_data=input_data, - encoding=msg.content.code.encoding, + encoding=message.content.code.encoding, scope=scope, ) - await vm.teardown() - system(f"rm -fr {vm.jailer_path}") - # TODO: Handle other content-types - return web.Response(body=result, content_type="application/json") + except UnpackValueError as error: + logger.exception(error) + return web.Response(status=502, reason="Invalid response from VM") + + try: + result = msgpack.loads(result_raw, raw=False) + # TODO: Handle other content-types + + logger.debug(f"Result from VM: <<<\n\n{str(result)}\n\n>>>") + + if "traceback" in result: + logger.warning(result["traceback"]) + return web.Response( + status=500, + reason="Error in VM execution", + body=result["traceback"], + content_type="text/plain", + ) + + return web.Response( + body=result["body"]["body"], content_type="application/json" + ) + except UnpackValueError as error: + logger.exception(error) + return web.Response(status=502, reason="Invalid response from VM") + finally: + await vm.teardown() app = web.Application() @@ -109,13 +135,5 @@ async def run_code(request: web.Request): def run(): """Run the VM Supervisor.""" - - # runtime = 'aleph-alpine-3.13-python' - kernel_image_path = os.path.abspath("./kernels/vmlinux.bin") - # rootfs_path = os.path.abspath(f"./runtimes/{runtime}/rootfs.ext4") - - for path in (settings.FIRECRACKER_PATH, settings.JAILER_PATH, kernel_image_path): - if not os.path.isfile(path): - raise FileNotFoundError(path) - + settings.check() web.run_app(app) diff --git a/vm_supervisor/vm/__init__.py b/vm_supervisor/vm/__init__.py index 7b4b9ce0b..ae7b3635f 100644 --- a/vm_supervisor/vm/__init__.py +++ b/vm_supervisor/vm/__init__.py @@ -1 +1,3 @@ -from .firecracker_microvm import start_new_vm as start_new_firecracker_vm +from .firecracker_microvm import AlephFirecrackerVM + +assert AlephFirecrackerVM diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 2c7450eac..dd3adaa53 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -1,41 +1,180 @@ import asyncio +import dataclasses import logging +from dataclasses import dataclass +from multiprocessing import Process +from os import system from os.path import isfile +from typing import Optional, Dict + +import msgpack -from vm_supervisor.conf import settings from firecracker.microvm import MicroVM, setfacl +from guest_api.__main__ import run_guest_api +from ..conf import settings +from ..models import FunctionMessage, FilePath +from ..storage import get_code_path, get_runtime_path, get_data_path logger = logging.getLogger(__name__) -async def start_new_vm(vm_id: int, kernel_image_path: str, rootfs_path: str) -> MicroVM: - logger.info("Created VM= %s", vm_id) - - assert isfile(kernel_image_path) - assert isfile(rootfs_path) - - await setfacl() - vm = MicroVM( - vm_id, - firecracker_bin_path=settings.FIRECRACKER_PATH, - use_jailer=settings.USE_JAILER, - jailer_bin_path=settings.JAILER_PATH, - ) - vm.cleanup_jailer() - await vm.start() - await vm.socket_is_ready() - await vm.set_boot_source( - kernel_image_path, enable_console=settings.PRINT_SYSTEM_LOGS - ) - await vm.set_rootfs(rootfs_path) - await vm.set_vsock() - await vm.set_network() - - if settings.PRINT_SYSTEM_LOGS: - asyncio.get_running_loop().create_task(vm.print_logs()) - - await asyncio.gather( - vm.start_instance(), - vm.wait_for_init(), - ) - return vm +@dataclass +class RunCodePayload: + code: bytes + input_data: bytes + entrypoint: str + encoding: str + scope: Dict + + def as_msgpack(self) -> bytes: + return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True) + + +class AlephFirecrackerResources: + + message: FunctionMessage + + kernel_image_path: FilePath + code_path: FilePath + rootfs_path: FilePath + data_path: Optional[FilePath] + + def __init__(self, message: FunctionMessage): + self.message = message + + async def download_kernel(self): + # Assumes kernel is already present on the host + self.kernel_image_path = settings.LINUX_PATH + assert isfile(self.kernel_image_path) + + async def download_code(self): + code_ref: str = self.message.content.code.ref + self.code_path = await get_code_path(code_ref) + assert isfile(self.code_path) + + async def download_runtime(self): + runtime_ref: str = self.message.content.runtime.ref + self.rootfs_path = await get_runtime_path(runtime_ref) + assert isfile(self.rootfs_path) + + async def download_data(self): + if self.message.content.data: + data_ref: str = self.message.content.data.ref + self.data_path = await get_data_path(data_ref) + assert isfile(self.data_path) + else: + self.data_path = None + + async def download_all(self): + await asyncio.gather( + self.download_kernel(), + self.download_code(), + self.download_runtime(), + self.download_data(), + ) + + +class AlephFirecrackerVM: + vm_id: int + resources: AlephFirecrackerResources + enable_console: bool + fvm: MicroVM + guest_api_process: Process + + def __init__( + self, + vm_id: int, + resources: AlephFirecrackerResources, + enable_console: Optional[bool] = None, + ): + self.vm_id = vm_id + self.resources = resources + if enable_console is None: + enable_console = settings.PRINT_SYSTEM_LOGS + self.enable_console = enable_console + + async def setup(self): + logger.debug("setup started") + await setfacl() + fvm = MicroVM( + vm_id=self.vm_id, + firecracker_bin_path=settings.FIRECRACKER_PATH, + use_jailer=settings.USE_JAILER, + jailer_bin_path=settings.JAILER_PATH, + ) + fvm.prepare_jailer() + await fvm.start() + await fvm.socket_is_ready() + await fvm.set_boot_source( + self.resources.kernel_image_path, + enable_console=self.enable_console, + ) + await fvm.set_rootfs(self.resources.rootfs_path) + await fvm.set_vsock() + await fvm.set_network() + logger.debug("setup done") + self.fvm = fvm + + async def start(self): + logger.debug(f"starting vm {self.vm_id}") + if not self.fvm: + raise ValueError("No VM found. Call setup() before start()") + + fvm = self.fvm + if self.enable_console: + asyncio.get_running_loop().create_task(fvm.print_logs()) + asyncio.get_running_loop().create_task(fvm.print_logs_stderr()) + + await asyncio.gather( + fvm.start_instance(), + fvm.wait_for_init(), + ) + logger.debug(f"started fvm {self.vm_id}") + + async def start_guest_api(self): + logger.debug(f"starting guest API for {self.vm_id}") + vsock_path = f"{self.fvm.vsock_path}_53" + self.guest_api_process = Process(target=run_guest_api, args=(vsock_path,)) + self.guest_api_process.start() + # FIXME: Wait for the API to open the socket + await asyncio.sleep(1) + system(f"chown jailman:jailman {vsock_path}") + logger.debug(f"started guest API for {self.vm_id}") + + async def stop_guest_api(self): + self.guest_api_process.terminate() + + async def teardown(self): + await self.fvm.teardown() + await self.stop_guest_api() + + async def run_code( + self, + code: bytes, + entrypoint: str, + input_data: bytes = b"", + encoding: str = "plain", + scope: dict = None, + ): + scope = scope or {} + reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path) + + payload = RunCodePayload( + code=code, + input_data=input_data, + entrypoint=entrypoint, + encoding=encoding, + scope=scope, + ) + + writer.write(b"CONNECT 52\n" + payload.as_msgpack()) + await writer.drain() + + ack: bytes = await reader.readline() + logger.debug(f"ack={ack.decode()}") + + response: bytes = await reader.read() + + writer.close() + await writer.wait_closed() + return response From f66c53ae7336121390d132b40caa49af9e1fbfbd Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 11 May 2021 11:46:15 +0200 Subject: [PATCH 010/990] Feature: Make VM network optional (#11) Helps testing in containers. --- firecracker/microvm.py | 6 ++++-- runtimes/aleph-alpine-3.13-python/init0.sh | 10 ++++++---- vm_supervisor/__main__.py | 7 +++++++ vm_supervisor/conf.py | 2 ++ vm_supervisor/pool.py | 5 ++++- vm_supervisor/vm/firecracker_microvm.py | 6 +++++- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/firecracker/microvm.py b/firecracker/microvm.py index a5143c1ea..226d2cfd1 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -63,6 +63,7 @@ class MicroVM: firecracker_bin_path: str jailer_bin_path: Optional[str] proc: Optional[asyncio.subprocess.Process] = None + network_tap: Optional[str] = None @property def jailer_path(self): @@ -233,6 +234,7 @@ async def set_vsock(self): async def set_network(self): """Configure the host network with a tap interface to the VM.""" name = f"tap{self.vm_id}" + self.network_tap = name system(f"ip tuntap add {name} mode tap") system( @@ -316,8 +318,8 @@ async def teardown(self): """Stop the VM, cleanup network interface and remove data directory.""" await self.stop() - name = f"tap{self.vm_id}" - system(f"ip tuntap del {name} mode tap") + if self.network_tap: + system(f"ip tuntap del {self.network_tap} mode tap") system(f"rm -fr {self.jailer_path}") diff --git a/runtimes/aleph-alpine-3.13-python/init0.sh b/runtimes/aleph-alpine-3.13-python/init0.sh index a665aeb83..309aedf0f 100644 --- a/runtimes/aleph-alpine-3.13-python/init0.sh +++ b/runtimes/aleph-alpine-3.13-python/init0.sh @@ -31,10 +31,12 @@ mount -t devpts devpts /dev/pts -o mode=0620,gid=5,nosuid,noexec mount -t tmpfs shm /dev/shm -omode=1777,nosuid,nodev # TODO: Move in init1 -#ip addr add 172.0.5.2/24 dev eth0 -#ip link set eth0 up -#ip route add default via 172.0.5.1 dev eth0 -#ip addr +if [[ -d /sys/class/net/eth0 ]]; then + ip addr add 172.0.5.2/24 dev eth0 + ip link set eth0 up + ip route add default via 172.0.5.1 dev eth0 + ip addr +fi echo "Net up" diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index fe830f2a9..779140eca 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -18,6 +18,12 @@ def parse_args(args): dest="system_logs", default=settings.PRINT_SYSTEM_LOGS, ) + parser.add_argument( + "--no-network", + action="store_false", + dest="allow_vm_networking", + default=settings.ALLOW_VM_NETWORKING, + ) parser.add_argument( "--no-jailer", action="store_false", @@ -75,6 +81,7 @@ def main(): USE_JAILER=args.use_jailer, PRINT_SYSTEM_LOGS=args.system_logs, PREALLOC_VM_COUNT=args.prealloc_vm_count, + ALLOW_VM_NETWORKING=args.allow_vm_networking, ) if args.print_settings: print(settings.display()) diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 510d77043..7564cd8a1 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -15,6 +15,8 @@ class Settings: USE_JAILER: bool = getenv("ALEPH_USER_JAILER", "true") == "true" # System logs make boot ~2x slower PRINT_SYSTEM_LOGS: bool = getenv("ALEPH_PRINT_SYSTEM_LOGS", "false") == "true" + # Networking does not work inside Docker/Podman + ALLOW_VM_NETWORKING: bool = getenv("ALEPH_PRINT_SYSTEM_LOGS", "true") == "true" FIRECRACKER_PATH: str = getenv( "ALEPH_FIRECRACKER_PATH", "/opt/firecracker/firecracker" ) diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index beaf2d0d1..ff409c14b 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -49,7 +49,10 @@ def __init__(self): async def get_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: vm_resources = AlephFirecrackerResources(message) await vm_resources.download_all() - vm = AlephFirecrackerVM(vm_id=self.counter, resources=vm_resources) + vm = AlephFirecrackerVM( + vm_id=self.counter, + resources=vm_resources, + enable_networking=message.content.environment.internet) await vm.setup() await vm.start() return vm diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index dd3adaa53..cddae7a03 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -78,6 +78,7 @@ class AlephFirecrackerVM: vm_id: int resources: AlephFirecrackerResources enable_console: bool + enable_networking: bool fvm: MicroVM guest_api_process: Process @@ -85,10 +86,12 @@ def __init__( self, vm_id: int, resources: AlephFirecrackerResources, + enable_networking: bool = False, enable_console: Optional[bool] = None, ): self.vm_id = vm_id self.resources = resources + self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING if enable_console is None: enable_console = settings.PRINT_SYSTEM_LOGS self.enable_console = enable_console @@ -111,7 +114,8 @@ async def setup(self): ) await fvm.set_rootfs(self.resources.rootfs_path) await fvm.set_vsock() - await fvm.set_network() + if self.enable_networking: + await fvm.set_network() logger.debug("setup done") self.fvm = fvm From 15afc5b373c18dab9cd8dcc16a0fb96c60e27a8c Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 11 May 2021 11:53:37 +0200 Subject: [PATCH 011/990] Feature: Add profiling logs and --benchmark CLI (#12) This makes it easier to profile the performance of an Aleph VM. --- docker/run_benchmark_in_docker.sh | 0 examples/example_fastapi_2/__init__.py | 8 +++- firecracker/microvm.py | 2 + guest_api/__main__.py | 3 -- runtimes/aleph-alpine-3.13-python/init0.sh | 15 ++++--- runtimes/aleph-alpine-3.13-python/init1.py | 31 +++++++++++--- vm_supervisor/README.md | 2 +- vm_supervisor/__main__.py | 50 +++++++++++++++++++++- vm_supervisor/conf.py | 5 +++ vm_supervisor/storage.py | 1 - vm_supervisor/supervisor.py | 5 +-- vm_supervisor/vm/firecracker_microvm.py | 3 +- 12 files changed, 99 insertions(+), 26 deletions(-) create mode 100644 docker/run_benchmark_in_docker.sh diff --git a/docker/run_benchmark_in_docker.sh b/docker/run_benchmark_in_docker.sh new file mode 100644 index 000000000..e69de29bb diff --git a/examples/example_fastapi_2/__init__.py b/examples/example_fastapi_2/__init__.py index 52daaa634..f30c949c5 100644 --- a/examples/example_fastapi_2/__init__.py +++ b/examples/example_fastapi_2/__init__.py @@ -1,10 +1,14 @@ import logging from typing import Optional +logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.DEBUG) -from aleph_client.asynchronous import get_messages +logger.debug("") +logger.debug("import aleph_client") +from aleph_client.asynchronous import get_messages +logger.debug("import fastapi") from fastapi import FastAPI +logger.debug("imports done") app = FastAPI() diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 226d2cfd1..63e088362 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -233,6 +233,8 @@ async def set_vsock(self): async def set_network(self): """Configure the host network with a tap interface to the VM.""" + logger.debug("Network setup") + name = f"tap{self.vm_id}" self.network_tap = name diff --git a/guest_api/__main__.py b/guest_api/__main__.py index 2107ff328..f52a994a6 100644 --- a/guest_api/__main__.py +++ b/guest_api/__main__.py @@ -1,9 +1,6 @@ -import logging - import aiohttp from aiohttp import web -logging.basicConfig(level=logging.DEBUG) ALEPH_API_SERVER = "https://api2.aleph.im/" diff --git a/runtimes/aleph-alpine-3.13-python/init0.sh b/runtimes/aleph-alpine-3.13-python/init0.sh index 309aedf0f..e815ed4cd 100644 --- a/runtimes/aleph-alpine-3.13-python/init0.sh +++ b/runtimes/aleph-alpine-3.13-python/init0.sh @@ -2,10 +2,13 @@ set -euf -echo "init0.sh is launching" - mount -t proc proc /proc -o nosuid,noexec,nodev +function log() { + echo `cat /proc/uptime | awk '{printf $1}'` '|S' $@ +} +log "init0.sh is launching" + # Switch root from read-only ext4 to to read-write overlay mkdir -p /overlay /bin/mount -t tmpfs -o noatime,mode=0755 tmpfs /overlay @@ -38,17 +41,17 @@ if [[ -d /sys/class/net/eth0 ]]; then ip addr fi -echo "Net up" +log "Net up" #cat /proc/sys/kernel/random/entropy_avail # TODO: Move in init1 /usr/sbin/sshd -E /var/log/sshd & -echo "SSH UP" +log "SSH UP" -echo "Setup socat" +log "Setup socat" socat UNIX-LISTEN:/tmp/socat-socket,fork,reuseaddr VSOCK-CONNECT:2:53 & -echo "Socat ready" +log "Socat ready" # Replace this script with the manager exec /root/init1.py diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 1573f1b92..c19a9b543 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -1,5 +1,14 @@ #!/usr/bin/python3 +import logging +logging.basicConfig( + level=logging.DEBUG, + format="%(relativeCreated)4f |V %(levelname)s | %(message)s", +) +logger = logging.getLogger(__name__) + +logger.debug("Imports starting") + import asyncio import os import socket @@ -15,6 +24,7 @@ import msgpack +logger.debug("Imports finished") class Encoding: plain = "plain" @@ -43,18 +53,21 @@ class RunCodePayload: # Configure aleph-client to use the guest API os.environ["ALEPH_API_UNIX_SOCKET"] = "/tmp/socat-socket" -print("init1.py is launching") +logger.debug("init1.py is launching") async def run_python_code_http(code: bytes, input_data: Optional[bytes], entrypoint: str, encoding: str, scope: dict ) -> Tuple[Dict, Dict, str, Optional[bytes]]: + logger.debug("Extracting code") if encoding == Encoding.zip: # Unzip in /opt and import the entrypoint from there open("/opt/archive.zip", "wb").write(code) + logger.debug("Run unzip") os.system("unzip /opt/archive.zip -d /opt") sys.path.append("/opt") module_name, app_name = entrypoint.split(":", 1) + logger.debug("import module") module = __import__(module_name) app = getattr(module, app_name) elif encoding == Encoding.plain: @@ -65,12 +78,14 @@ async def run_python_code_http(code: bytes, input_data: Optional[bytes], else: raise ValueError(f"Unknown encoding '{encoding}'") + logger.debug("Extracting data") if input_data: # Unzip in /data open("/opt/input.zip", "wb").write(input_data) os.makedirs("/data", exist_ok=True) os.system("unzip /opt/input.zip -d /data") + logger.debug("Running code") with StringIO() as buf, redirect_stdout(buf): # Execute in the same process, saves ~20ms than a subprocess async def receive(): @@ -87,9 +102,7 @@ async def send(dico): body: Dict = await send_queue.get() output = buf.getvalue() - os.makedirs("/data", exist_ok=True) - open('/data/hello.txt', 'w').write("Hello !") - + logger.debug("Getting output data") output_data: bytes if os.listdir('/data'): make_archive("/opt/output", 'zip', "/data") @@ -98,6 +111,7 @@ async def send(dico): else: output_data = b'' + logger.debug("Returning result") return headers, body, output, output_data @@ -116,7 +130,9 @@ def process_instruction(instruction: bytes) -> Iterator[bytes]: yield str(error).encode() + b"\n" + error.output else: # Python + logger.debug("msgpack.loads (") msg_ = msgpack.loads(instruction, raw=False) + logger.debug("msgpack.loads )") payload = RunCodePayload(**msg_) try: @@ -150,13 +166,14 @@ def main(): while True: client, addr = s.accept() data = client.recv(1000_1000) # Max 1 Mo - print("CID: {} port:{} data: {}".format(addr[0], addr[1], len(data))) + logger.debug("CID: {} port:{} data: {}".format(addr[0], addr[1], len(data))) - print("Init received msg <<<\n\n", data, "\n\n>>>") + logger.debug("Init received msg") + print(f"<<<\n\n{data}\n\n>>>") for result in process_instruction(instruction=data): client.send(result) - print("...DONE") + logger.debug("...DONE") client.close() diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index b481751d2..fc0dc4ae0 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -52,7 +52,7 @@ when running the VM Supervisor. ```shell apt update -apt install -y git python3 python3-aiohttp sudo acl curl systemd-container +apt install -y git python3 python3-aiohttp python3-msgpack sudo acl curl systemd-container useradd jailman ``` diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 779140eca..20328f0b5 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -1,6 +1,12 @@ import argparse +import asyncio import logging import sys +import time +from statistics import mean +from typing import List + +from aiohttp.web import Response from . import supervisor from .conf import settings @@ -71,12 +77,33 @@ def parse_args(args): dest="do_not_run", default=False, ) + parser.add_argument( + "--profile", + dest="profile", + action="store_true", + default=False, + help="Add extra info for profiling", + ) + parser.add_argument( + "--benchmark", + dest="benchmark", + type=int, + default=0, + help="Number of benchmarks to run", + ) return parser.parse_args(args) def main(): args = parse_args(sys.argv[1:]) - logging.basicConfig(level=args.loglevel) + + log_format = "%(relativeCreated)4f | %(levelname)s | %(message)s" if args.profile \ + else "%(asctime)s | %(levelname)s | %(message)s" + logging.basicConfig( + level=args.loglevel, + format=log_format, + ) + settings.update( USE_JAILER=args.use_jailer, PRINT_SYSTEM_LOGS=args.system_logs, @@ -88,6 +115,27 @@ def main(): settings.check() + if args.benchmark > 0: + class FakeRequest: pass + fake_request = FakeRequest() + fake_request.match_info = {"ref": "vmid", "suffix": "/path"} + fake_request.method = "GET" + fake_request.query_string = "" + fake_request.headers = [] + fake_request.raw_headers = [] + + logger.info("--- Start benchmark ---") + + bench: List[float] = [] + for run in range(args.benchmark): + t0 = time.time() + response: Response = asyncio.run(supervisor.run_code(request=fake_request)) + assert response.status == 200 + bench.append(time.time() - t0) + + logger.info(f"BENCHMARK: n={len(bench)} avg={mean(bench):03f} " + f"min={min(bench):03f} max={max(bench):03f}") + if args.do_not_run: logger.info("Option --do-not-run, exiting") else: diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 7564cd8a1..604780c1f 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -1,3 +1,4 @@ +import logging import os from os import getenv from os.path import isfile, join @@ -5,6 +6,8 @@ from .models import FilePath +logger = logging.getLogger(__name__) + Url = NewType("Url", str) @@ -47,6 +50,8 @@ class Settings: def update(self, **kwargs): for key, value in kwargs.items(): + if key != key.upper(): + logger.warning(f"Setting {key} is not uppercase") if hasattr(self, key): setattr(self, key, value) else: diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index 7ce5b67ab..3cd0dfdc4 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -62,7 +62,6 @@ async def get_code_path(ref: str) -> FilePath: if settings.FAKE_DATA: root_dir = abspath(join(__file__, "../../examples/")) archive_path = join(root_dir, "example_fastapi_2") - # app_dir = abspath(join(__file__, "../../examples/visit_counter")) make_archive( archive_path, "zip", root_dir=root_dir, base_dir="example_fastapi_2" ) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 7ec1b1011..932f55ff6 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -6,7 +6,6 @@ evolve in the future. """ import logging -from multiprocessing import set_start_method import msgpack from aiohttp import web, ClientResponseError, ClientConnectorError @@ -21,8 +20,6 @@ logger = logging.getLogger(__name__) pool = VmPool() -set_start_method("spawn") - async def index(request: web.Request): assert request.method == "GET" @@ -64,7 +61,7 @@ def load_file_content(path: FilePath) -> bytes: return b"" -async def run_code(request: web.Request): +async def run_code(request: web.Request) -> web.Response: """ Execute the code corresponding to the 'code id' in the path. """ diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index cddae7a03..2da5969e1 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -2,7 +2,7 @@ import dataclasses import logging from dataclasses import dataclass -from multiprocessing import Process +from multiprocessing import Process, set_start_method from os import system from os.path import isfile from typing import Optional, Dict @@ -16,6 +16,7 @@ from ..storage import get_code_path, get_runtime_path, get_data_path logger = logging.getLogger(__name__) +set_start_method("spawn") @dataclass From fcd2a180576da42a1d14058cb5b62924e2595b5d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 11 May 2021 12:28:47 +0200 Subject: [PATCH 012/990] Fix: Watch for the socket instead of sleeping for 1 second (#13) --- vm_supervisor/vm/firecracker_microvm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 2da5969e1..5370f8ebb 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from multiprocessing import Process, set_start_method from os import system -from os.path import isfile +from os.path import isfile, exists from typing import Optional, Dict import msgpack @@ -141,8 +141,8 @@ async def start_guest_api(self): vsock_path = f"{self.fvm.vsock_path}_53" self.guest_api_process = Process(target=run_guest_api, args=(vsock_path,)) self.guest_api_process.start() - # FIXME: Wait for the API to open the socket - await asyncio.sleep(1) + while not exists(vsock_path): + await asyncio.sleep(0.01) system(f"chown jailman:jailman {vsock_path}") logger.debug(f"started guest API for {self.vm_id}") From 417e9a333eac506ee466fd96f2515f1821647a76 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 11 May 2021 14:59:53 +0200 Subject: [PATCH 013/990] Feature: Add Dockerfile and script to run supervisor in Docker for development --- docker/run_vm_supervisor.sh | 8 ++++++++ docker/vm_supervisor.dockerfile | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100755 docker/run_vm_supervisor.sh create mode 100644 docker/vm_supervisor.dockerfile diff --git a/docker/run_vm_supervisor.sh b/docker/run_vm_supervisor.sh new file mode 100755 index 000000000..015d5f476 --- /dev/null +++ b/docker/run_vm_supervisor.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +docker build -ti -t aleph-vm-supervisor -f docker/vm_supervisor.dockerfile . +docker run -ti --rm \ + -v $(pwd):/root/aleph-vm \ + --device /dev/kvm \ + aleph-vm-supervisor \ + python3 -m vm_supervisor -p -vv --system-logs --benchmark 1 --profile diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile new file mode 100644 index 000000000..64a26719a --- /dev/null +++ b/docker/vm_supervisor.dockerfile @@ -0,0 +1,36 @@ +# This is mainly a copy of the installation instructions from [vm_supervisor/README.md] + +FROM debian:buster + +RUN apt-get update && apt-get -y upgrade && apt-get install -y \ + sudo acl curl systemd-container \ + python3 python3-aiohttp python3-msgpack python3-pip \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd jailman + +RUN mkdir /opt/firecracker +RUN chown $(whoami) /opt/firecracker +RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v0.24.2/firecracker-v0.24.2-x86_64.tgz | tar -xz --directory /opt/firecracker + +# Link binaries on version-agnostic paths: +RUN ln /opt/firecracker/firecracker-v* /opt/firecracker/firecracker +RUN ln /opt/firecracker/jailer-v* /opt/firecracker/jailer + +RUN pip3 install typing-extensions + +RUN mkdir /srv/jailer + +ENV PYTHONPATH /mnt + +# Networking does not work in Docker containers +ENV ALLOW_VM_NETWORKING false +# Jailer does not work in Docker containers +ENV ALEPH_USER_JAILER false +# Use fake test data +ENV ALEPH_FAKE_DATA true + +# Make it easy to enter this command from a shell script +RUN echo "python3 -m vm_supervisor -p -vv --system-logs --benchmark 1 --profile" >> /root/.bash_history + +WORKDIR /root/aleph-vm From 01461e4ad25b6ced03262119d9805f6a4bbbd93d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 11 May 2021 18:21:05 +0200 Subject: [PATCH 014/990] Optimization: Compile Python bytecode during runtime build --- runtimes/aleph-alpine-3.13-python/create_disk_image.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index 7c602b679..367d3dc66 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -26,7 +26,10 @@ apk add py3-aiohttp py3-msgpack pip install fastapi apk add git pkgconf gcc py3-wheel python3-dev musl-dev py3-cffi libffi-dev autoconf automake libtool make -pip install -e git+https://github.com/aleph-im/aleph-client@hoh-remove-secp256k1#egg=aleph-client coincurve==15.0.0 +pip install git+https://github.com/aleph-im/aleph-client@hoh-remove-secp256k1#egg=aleph-client coincurve==15.0.0 + +# Compile all Python bytecode +python3 -m compileall /usr/lib/python3.8/site-packages echo -e "toor\ntoor" | passwd root From 8d0f125e9a173db0daf339cdc586105f8a8fb67f Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 12 May 2021 13:16:04 +0200 Subject: [PATCH 015/990] Clean: Switch to main branch of aleph-client --- runtimes/aleph-alpine-3.13-python/create_disk_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index 367d3dc66..e2fd9aff3 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -26,7 +26,7 @@ apk add py3-aiohttp py3-msgpack pip install fastapi apk add git pkgconf gcc py3-wheel python3-dev musl-dev py3-cffi libffi-dev autoconf automake libtool make -pip install git+https://github.com/aleph-im/aleph-client@hoh-remove-secp256k1#egg=aleph-client coincurve==15.0.0 +pip install git+https://github.com/aleph-im/aleph-client coincurve==15.0.0 # Compile all Python bytecode python3 -m compileall /usr/lib/python3.8/site-packages From a54fe648f1b346168ad233d9ff66a0c52ee16e28 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 12 May 2021 16:31:57 +0200 Subject: [PATCH 016/990] Enh: Force pre-compile all Python code with optimizations --- runtimes/aleph-alpine-3.13-python/create_disk_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index e2fd9aff3..076e02c64 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -29,7 +29,7 @@ apk add git pkgconf gcc py3-wheel python3-dev musl-dev py3-cffi libffi-dev autoc pip install git+https://github.com/aleph-im/aleph-client coincurve==15.0.0 # Compile all Python bytecode -python3 -m compileall /usr/lib/python3.8/site-packages +python3 -m compileall -o 1 -o 2 -f /usr/lib/python3.8/site-packages echo -e "toor\ntoor" | passwd root From 88f057d954247f3abb4e98503ada32662383e4b8 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 12 May 2021 18:10:53 +0200 Subject: [PATCH 017/990] Fix: Don't run supervisor in benchmark mode --- vm_supervisor/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 20328f0b5..172f08f85 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -136,8 +136,8 @@ class FakeRequest: pass logger.info(f"BENCHMARK: n={len(bench)} avg={mean(bench):03f} " f"min={min(bench):03f} max={max(bench):03f}") - if args.do_not_run: - logger.info("Option --do-not-run, exiting") + if args.do_not_run or args.benchmark: + logger.info("Option --do-not-run or --benchmark, exiting") else: settings.setup() supervisor.run() From aaa15bc28333941f3b01e31c9f3e17b7f16c30a0 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 12 May 2021 18:12:53 +0200 Subject: [PATCH 018/990] Refactor: Better management of VM stdout task --- firecracker/microvm.py | 16 +++++++++++++++- vm_supervisor/vm/firecracker_microvm.py | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 63e088362..f9d74b4c7 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -2,12 +2,13 @@ import json import logging import os.path +from asyncio import Task from enum import Enum from functools import lru_cache from os import getuid from pathlib import Path from pwd import getpwnam -from typing import Optional +from typing import Optional, Tuple import aiohttp from aiohttp import ClientResponse @@ -64,6 +65,8 @@ class MicroVM: jailer_bin_path: Optional[str] proc: Optional[asyncio.subprocess.Process] = None network_tap: Optional[str] = None + stdout_task: Optional[Task] = None + stderr_task: Optional[Task] = None @property def jailer_path(self): @@ -292,6 +295,12 @@ async def print_logs_stderr(self): else: await asyncio.sleep(0.001) + def start_printing_logs(self) -> Tuple[Task, Task]: + loop = asyncio.get_running_loop() + self.stdout_task = loop.create_task(self.print_logs()) + self.stderr_task = loop.create_task(self.print_logs_stderr()) + return self.stdout_task, self.stderr_task + async def wait_for_init(self): """Wait for a connection from the init in the VM""" logger.debug("Waiting for init...") @@ -320,6 +329,11 @@ async def teardown(self): """Stop the VM, cleanup network interface and remove data directory.""" await self.stop() + if self.stdout_task: + self.stdout_task.cancel() + if self.stderr_task: + self.stderr_task.cancel() + if self.network_tap: system(f"ip tuntap del {self.network_tap} mode tap") diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 5370f8ebb..03579ddff 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -126,9 +126,9 @@ async def start(self): raise ValueError("No VM found. Call setup() before start()") fvm = self.fvm + if self.enable_console: - asyncio.get_running_loop().create_task(fvm.print_logs()) - asyncio.get_running_loop().create_task(fvm.print_logs_stderr()) + fvm.start_printing_logs() await asyncio.gather( fvm.start_instance(), From 1685ccd73e811e08252c1ab87429630625edc2f2 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 12 May 2021 18:13:16 +0200 Subject: [PATCH 019/990] Fix: Variable was not always defined --- runtimes/aleph-alpine-3.13-python/init1.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index c19a9b543..51bdcd39e 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python3 -OO import logging logging.basicConfig( @@ -135,10 +135,10 @@ def process_instruction(instruction: bytes) -> Iterator[bytes]: logger.debug("msgpack.loads )") payload = RunCodePayload(**msg_) + output: Optional[str] = None try: headers: Dict body: Dict - output: str output_data: Optional[bytes] headers, body, output, output_data = asyncio.get_event_loop().run_until_complete( From 8e9d573f5f24cf35f28e16b0d777496c47ea7af1 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 12 May 2021 18:14:21 +0200 Subject: [PATCH 020/990] Enhancement: Start guest API from separate task --- vm_supervisor/supervisor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 932f55ff6..deb52df7f 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -5,6 +5,7 @@ At it's core, it is currently an asynchronous HTTP server using aiohttp, but this may evolve in the future. """ +import asyncio import logging import msgpack @@ -79,7 +80,8 @@ async def run_code(request: web.Request) -> web.Response: # raise vm = await pool.get_a_vm(message) - await vm.start_guest_api() + loop = asyncio.get_event_loop() + guest_api = loop.create_task(vm.start_guest_api()) logger.debug(f"Using vm={vm.vm_id}") scope = build_asgi_scope(request) @@ -122,6 +124,7 @@ async def run_code(request: web.Request) -> web.Response: return web.Response(status=502, reason="Invalid response from VM") finally: await vm.teardown() + guest_api.cancel() app = web.Application() From 9b782818ac5154950a201e9335c1dcb611bf3fe8 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 12 May 2021 18:15:13 +0200 Subject: [PATCH 021/990] Enhancement: Add more loging for profiling --- vm_supervisor/vm/firecracker_microvm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 03579ddff..68e08d05a 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -161,6 +161,7 @@ async def run_code( encoding: str = "plain", scope: dict = None, ): + logger.debug("running code") scope = scope or {} reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path) @@ -178,8 +179,10 @@ async def run_code( ack: bytes = await reader.readline() logger.debug(f"ack={ack.decode()}") + logger.debug("waiting for VM response") response: bytes = await reader.read() + logger.debug("cleaning VM resources") writer.close() await writer.wait_closed() return response From 0bfda19f9040b2e09f9bd8901fca2fc02140ce9a Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 12 May 2021 18:15:37 +0200 Subject: [PATCH 022/990] Fix: VM result could be very long -> crop it. --- vm_supervisor/supervisor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index deb52df7f..f9468e41d 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -105,7 +105,7 @@ async def run_code(request: web.Request) -> web.Response: result = msgpack.loads(result_raw, raw=False) # TODO: Handle other content-types - logger.debug(f"Result from VM: <<<\n\n{str(result)}\n\n>>>") + logger.debug(f"Result from VM: <<<\n\n{str(result)[:1000]}\n\n>>>") if "traceback" in result: logger.warning(result["traceback"]) From f5b9d20c9e3715602d6a94e310d8606e874d2888 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 17 May 2021 16:29:19 +0200 Subject: [PATCH 023/990] Feature: Keep a VM ready to accept another call After every execution, a VM is kept running for in case the same function gets called again. The VM is stopped if no call is made during a configurable duration. --- runtimes/aleph-alpine-3.13-python/init1.py | 14 ++-- vm_supervisor/__main__.py | 55 ++++++++----- vm_supervisor/conf.py | 2 + vm_supervisor/models.py | 21 +++-- vm_supervisor/pool.py | 94 ++++++++++++++-------- vm_supervisor/supervisor.py | 10 +-- 6 files changed, 122 insertions(+), 74 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 51bdcd39e..c2358cd53 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -62,9 +62,10 @@ async def run_python_code_http(code: bytes, input_data: Optional[bytes], logger.debug("Extracting code") if encoding == Encoding.zip: # Unzip in /opt and import the entrypoint from there - open("/opt/archive.zip", "wb").write(code) - logger.debug("Run unzip") - os.system("unzip /opt/archive.zip -d /opt") + if not os.path.exists("/opt/archive.zip"): + open("/opt/archive.zip", "wb").write(code) + logger.debug("Run unzipp") + os.system("unzip /opt/archive.zip -d /opt") sys.path.append("/opt") module_name, app_name = entrypoint.split(":", 1) logger.debug("import module") @@ -81,9 +82,10 @@ async def run_python_code_http(code: bytes, input_data: Optional[bytes], logger.debug("Extracting data") if input_data: # Unzip in /data - open("/opt/input.zip", "wb").write(input_data) - os.makedirs("/data", exist_ok=True) - os.system("unzip /opt/input.zip -d /data") + if not os.path.exists("/opt/input.zip"): + open("/opt/input.zip", "wb").write(input_data) + os.makedirs("/data", exist_ok=True) + os.system("unzip /opt/input.zip -d /data") logger.debug("Running code") with StringIO() as buf, redirect_stdout(buf): diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 172f08f85..7b7678df4 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -94,6 +94,34 @@ def parse_args(args): return parser.parse_args(args) +async def benchmark(runs: int): + """Measure performance by immediately running the supervisor + with fake requests. + """ + class FakeRequest: pass + + fake_request = FakeRequest() + fake_request.match_info = {"ref": "vmid", "suffix": "/path"} + fake_request.method = "GET" + fake_request.query_string = "" + fake_request.headers = [] + fake_request.raw_headers = [] + + logger.info("--- Start benchmark ---") + + bench: List[float] = [] + + for run in range(runs): + t0 = time.time() + response: Response = await supervisor.run_code(request=fake_request) + assert response.status == 200 + bench.append(time.time() - t0) + + logger.info(f"BENCHMARK: n={len(bench)} avg={mean(bench):03f} " + f"min={min(bench):03f} max={max(bench):03f}") + logger.info(bench) + + def main(): args = parse_args(sys.argv[1:]) @@ -116,28 +144,11 @@ def main(): settings.check() if args.benchmark > 0: - class FakeRequest: pass - fake_request = FakeRequest() - fake_request.match_info = {"ref": "vmid", "suffix": "/path"} - fake_request.method = "GET" - fake_request.query_string = "" - fake_request.headers = [] - fake_request.raw_headers = [] - - logger.info("--- Start benchmark ---") - - bench: List[float] = [] - for run in range(args.benchmark): - t0 = time.time() - response: Response = asyncio.run(supervisor.run_code(request=fake_request)) - assert response.status == 200 - bench.append(time.time() - t0) - - logger.info(f"BENCHMARK: n={len(bench)} avg={mean(bench):03f} " - f"min={min(bench):03f} max={max(bench):03f}") - - if args.do_not_run or args.benchmark: - logger.info("Option --do-not-run or --benchmark, exiting") + loop = asyncio.get_event_loop() + loop.run_until_complete(benchmark(runs=args.benchmark)) + print("Finished") + elif args.do_not_run: + logger.info("Option --do-not-run, exiting") else: settings.setup() supervisor.run() diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 604780c1f..f142cd36f 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -14,6 +14,8 @@ class Settings: VM_ID_START_INDEX: int = int(getenv("ALEPH_VM_START_ID_INDEX", 4)) PREALLOC_VM_COUNT: int = int(getenv("ALEPH_PREALLOC_VM_COUNT", 0)) + REUSE_TIMEOUT: float = float(getenv("ALEPH_REUSE_TIMEOUT", 120.)) + API_SERVER: str = getenv("ALEPH_API_SERVER", "https://api2.aleph.im") USE_JAILER: bool = getenv("ALEPH_USER_JAILER", "true") == "true" # System logs make boot ~2x slower diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index b9840d80d..d1e7795bd 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -6,49 +6,54 @@ FilePath = NewType("FilePath", str) +class HashableModel(BaseModel): + def __hash__(self): + return hash(self.__class__) + hash(tuple(self.__dict__.values())) + + class Encoding(str, Enum): plain = "plain" zip = "zip" targz = "tar.gzip" -class CodeContent(BaseModel): +class CodeContent(HashableModel): encoding: Encoding entrypoint: str ref: str latest_amend: bool = True -class DataContent(BaseModel): +class DataContent(HashableModel): encoding: Encoding mount: str ref: str latest_amend: bool = True -class FunctionTriggers(BaseModel): +class FunctionTriggers(HashableModel): http: bool -class FunctionEnvironment(BaseModel): +class FunctionEnvironment(HashableModel): reproducible: bool = False internet: bool = False aleph_api: bool = False -class FunctionResources(BaseModel): +class FunctionResources(HashableModel): vcpus: int = 1 memory: int = 128 seconds: int = 1 -class FunctionRuntime(BaseModel): +class FunctionRuntime(HashableModel): ref: str latest_amend: bool = True comment: str -class FunctionContent(BaseModel): +class FunctionContent(HashableModel): code: CodeContent data: Optional[DataContent] on: FunctionTriggers @@ -57,7 +62,7 @@ class FunctionContent(BaseModel): runtime: FunctionRuntime -class FunctionMessage(BaseModel): +class FunctionMessage(HashableModel): type: str address: str content: FunctionContent diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index ff409c14b..bbf5143de 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -1,7 +1,7 @@ import asyncio -from typing import Dict, List +import logging +from typing import Dict, Optional -from firecracker.microvm import MicroVM from vm_supervisor.conf import settings from vm_supervisor.models import FunctionMessage from vm_supervisor.vm.firecracker_microvm import ( @@ -9,50 +9,78 @@ AlephFirecrackerResources, ) +logger = logging.getLogger(__name__) -# class VmPool: -# """Pool of VMs pre-allocated in order to decrease response time. -# The counter is used by the VMs to set their tap interface name and the corresponding -# IPv4 subnet. -# """ -# -# queue: asyncio.Queue -# counter: int # Used for network interfaces -# -# def __init__(self): -# self.queue = asyncio.Queue() -# self.counter = settings.VM_ID_START_INDEX -# -# async def provision(self, kernel_image_path, rootfs_path): -# self.counter += 1 -# vm = await start_new_vm( -# vm_id=self.counter, -# kernel_image_path=kernel_image_path, -# rootfs_path=rootfs_path, -# ) -# await self.queue.put(vm) -# return vm -# -# async def get_a_vm(self, kernel_image_path, rootfs_path) -> MicroVM: -# loop = asyncio.get_event_loop() -# loop.create_task(self.provision(kernel_image_path, rootfs_path)) -# # Return the first VM from the pool -# return await self.queue.get() + +class StartedVM: + vm: AlephFirecrackerVM + timeout_task: Optional[asyncio.Task] + + def __init__(self, vm: AlephFirecrackerVM): + self.vm = vm + self.timeout_task = None class VmPool: - counter: int # Used for network interfaces + """Pool of VMs already started and used to decrease response time. + After running, a VM is saved for future reuse from the same function during a + configurable duration. + + The counter is used by the VMs to set their tap interface name and the corresponding + IPv4 subnet. + """ + + counter: int # Used to provide distinct ids to network interfaces + started_vms_cache: Dict[FunctionMessage, StartedVM] def __init__(self): self.counter = settings.VM_ID_START_INDEX + self.started_vms_cache = {} - async def get_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: + async def create_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: + """Create a new Aleph Firecracker VM from an Aleph function message.""" vm_resources = AlephFirecrackerResources(message) await vm_resources.download_all() + self.counter += 1 vm = AlephFirecrackerVM( vm_id=self.counter, resources=vm_resources, - enable_networking=message.content.environment.internet) + enable_networking=message.content.environment.internet, + ) await vm.setup() await vm.start() + await vm.start_guest_api() return vm + + async def get_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: + """Provision a VM in the pool, then return the first VM from the pool.""" + try: + started_vm = self.started_vms_cache.pop(message) + started_vm.timeout_task.cancel() + return started_vm.vm + except KeyError: + return await self.create_a_vm(message) + + def keep_in_cache( + self, vm: AlephFirecrackerVM, message: FunctionMessage, timeout: float = 1.0 + ) -> None: + """Keep a VM running for `timeout` seconds in case another query comes by.""" + + if message in self.started_vms_cache: + logger.warning("VM already in keep_in_cache, not caching") + return + + started_vm = StartedVM(vm=vm) + self.started_vms_cache[message] = started_vm + + loop = asyncio.get_event_loop() + started_vm.timeout_task = loop.create_task(self.expire(vm, message, timeout)) + + async def expire( + self, vm: AlephFirecrackerVM, message: FunctionMessage, timeout: float + ): + """Coroutine that will stop the VM after 'timeout' seconds.""" + await asyncio.sleep(timeout) + assert self.started_vms_cache[message].vm is vm + del self.started_vms_cache[message] + await vm.teardown() diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index f9468e41d..43e519891 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -5,7 +5,6 @@ At it's core, it is currently an asynchronous HTTP server using aiohttp, but this may evolve in the future. """ -import asyncio import logging import msgpack @@ -80,8 +79,6 @@ async def run_code(request: web.Request) -> web.Response: # raise vm = await pool.get_a_vm(message) - loop = asyncio.get_event_loop() - guest_api = loop.create_task(vm.start_guest_api()) logger.debug(f"Using vm={vm.vm_id}") scope = build_asgi_scope(request) @@ -97,6 +94,7 @@ async def run_code(request: web.Request) -> web.Response: encoding=message.content.code.encoding, scope=scope, ) + except UnpackValueError as error: logger.exception(error) return web.Response(status=502, reason="Invalid response from VM") @@ -123,8 +121,10 @@ async def run_code(request: web.Request) -> web.Response: logger.exception(error) return web.Response(status=502, reason="Invalid response from VM") finally: - await vm.teardown() - guest_api.cancel() + if settings.REUSE_TIMEOUT > 0: + pool.keep_in_cache(vm, message, timeout=settings.REUSE_TIMEOUT) + else: + await vm.teardown() app = web.Application() From a732d15b2b5c8491c1beaab2e4eeffd9f9ab7cb7 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 17 May 2021 19:17:28 +0200 Subject: [PATCH 024/990] Refactor: Use Pydantic BaseSettings for configuration --- docker/vm_supervisor.dockerfile | 6 ++-- vm_supervisor/conf.py | 62 ++++++++++++++------------------- vm_supervisor/pool.py | 2 +- 3 files changed, 30 insertions(+), 40 deletions(-) diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile index 64a26719a..c45cf5fd2 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor.dockerfile @@ -24,11 +24,11 @@ RUN mkdir /srv/jailer ENV PYTHONPATH /mnt # Networking does not work in Docker containers -ENV ALLOW_VM_NETWORKING false +ENV ALLOW_VM_NETWORKING False # Jailer does not work in Docker containers -ENV ALEPH_USER_JAILER false +ENV ALEPH_VM_USE_JAILER False # Use fake test data -ENV ALEPH_FAKE_DATA true +ENV ALEPH_VM_FAKE_DATA True # Make it easy to enter this command from a shell script RUN echo "python3 -m vm_supervisor -p -vv --system-logs --benchmark 1 --profile" >> /root/.bash_history diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index f142cd36f..b3bfcbe60 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -1,9 +1,9 @@ import logging import os -from os import getenv from os.path import isfile, join from typing import NewType +from pydantic import BaseSettings from .models import FilePath logger = logging.getLogger(__name__) @@ -11,44 +11,30 @@ Url = NewType("Url", str) -class Settings: - VM_ID_START_INDEX: int = int(getenv("ALEPH_VM_START_ID_INDEX", 4)) - PREALLOC_VM_COUNT: int = int(getenv("ALEPH_PREALLOC_VM_COUNT", 0)) - REUSE_TIMEOUT: float = float(getenv("ALEPH_REUSE_TIMEOUT", 120.)) +class Settings(BaseSettings): + START_ID_INDEX: int = 4 + PREALLOC_VM_COUNT: int = 0 + REUSE_TIMEOUT: float = 60 * 60.0 - API_SERVER: str = getenv("ALEPH_API_SERVER", "https://api2.aleph.im") - USE_JAILER: bool = getenv("ALEPH_USER_JAILER", "true") == "true" + API_SERVER: str = "https://api2.aleph.im" + USE_JAILER: bool = True # System logs make boot ~2x slower - PRINT_SYSTEM_LOGS: bool = getenv("ALEPH_PRINT_SYSTEM_LOGS", "false") == "true" + PRINT_SYSTEM_LOGS: bool = False # Networking does not work inside Docker/Podman - ALLOW_VM_NETWORKING: bool = getenv("ALEPH_PRINT_SYSTEM_LOGS", "true") == "true" - FIRECRACKER_PATH: str = getenv( - "ALEPH_FIRECRACKER_PATH", "/opt/firecracker/firecracker" - ) - JAILER_PATH: str = getenv("ALEPH_JAILER_PATH", "/opt/firecracker/jailer") - LINUX_PATH: str = getenv( - "ALEPH_LINUX_PATH", os.path.abspath("./kernels/vmlinux.bin") - ) - - CONNECTOR_URL: Url = Url(getenv("ALEPH_CONNECTOR_URL", "http://localhost:8000")) - - CACHE_ROOT: FilePath = FilePath( - getenv("ALEPH_CACHE_ROOT", "/tmp/aleph/vm_supervisor") - ) - MESSAGE_CACHE: FilePath = FilePath( - getenv("ALEPH_MESSAGE_CACHE", join(CACHE_ROOT, "message")) - ) - CODE_CACHE: FilePath = FilePath( - getenv("ALEPH_CODE_CACHE", join(CACHE_ROOT, "code")) - ) - RUNTIME_CACHE: FilePath = FilePath( - getenv("ALEPH_RUNTIME_CACHE", join(CACHE_ROOT, "runtime")) - ) - DATA_CACHE: FilePath = FilePath( - getenv("ALEPH_DATA_CACHE", join(CACHE_ROOT, "data")) - ) - - FAKE_DATA: bool = getenv("ALEPH_FAKE_DATA", "false") == "true" + ALLOW_VM_NETWORKING: bool = True + FIRECRACKER_PATH: str = "/opt/firecracker/firecracker" + JAILER_PATH: str = "/opt/firecracker/jailer" + LINUX_PATH: str = os.path.abspath("./kernels/vmlinux.bin") + + CONNECTOR_URL: Url = Url("http://localhost:8000") + + CACHE_ROOT: FilePath = FilePath("/tmp/aleph/vm_supervisor") + MESSAGE_CACHE: FilePath = FilePath(join(CACHE_ROOT, "message")) + CODE_CACHE: FilePath = FilePath(join(CACHE_ROOT, "code")) + RUNTIME_CACHE: FilePath = FilePath(join(CACHE_ROOT, "runtime")) + DATA_CACHE: FilePath = FilePath(join(CACHE_ROOT, "data")) + + FAKE_DATA: bool = False def update(self, **kwargs): for key, value in kwargs.items(): @@ -79,6 +65,10 @@ def display(self) -> str: for annotation, value in self.__annotations__.items() ) + class Config: + env_prefix = "ALEPH_VM_" + case_sensitive = False + # Settings singleton settings = Settings() diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index bbf5143de..c2e83e10d 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -34,7 +34,7 @@ class VmPool: started_vms_cache: Dict[FunctionMessage, StartedVM] def __init__(self): - self.counter = settings.VM_ID_START_INDEX + self.counter = settings.START_ID_INDEX self.started_vms_cache = {} async def create_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: From c5726a45adc2d6f665d12b9f27937b48772a32b1 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 17 May 2021 19:23:02 +0200 Subject: [PATCH 025/990] Fix: Guest API did not forward query string --- guest_api/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/guest_api/__main__.py b/guest_api/__main__.py index f52a994a6..eb3abaab9 100644 --- a/guest_api/__main__.py +++ b/guest_api/__main__.py @@ -7,7 +7,8 @@ async def proxy(request): path = request.match_info.get('tail') - url = ALEPH_API_SERVER + path + query_string = request.rel_url.query_string + url = f"{ALEPH_API_SERVER}{path}?{query_string}" async with aiohttp.ClientSession() as session: async with session.get(url) as response: From 7ebc3d0938a34df5cee9366b69a016c40df8056d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 18 May 2021 11:36:31 +0200 Subject: [PATCH 026/990] Fix: Networking was hardcoded for 1st VM It would not work in the next virtual machines launched. --- examples/message_from_aleph.json | 2 +- firecracker/microvm.py | 15 ++++++++++++-- runtimes/aleph-alpine-3.13-python/init0.sh | 10 --------- runtimes/aleph-alpine-3.13-python/init1.py | 24 ++++++++++++++++++++++ vm_supervisor/pool.py | 1 + vm_supervisor/vm/firecracker_microvm.py | 19 +++++++++++++++++ 6 files changed, 58 insertions(+), 13 deletions(-) diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index 1ed13fba0..097dc0bd2 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -23,7 +23,7 @@ }, "environment": { "reproducible": true, - "internet": false, + "internet": true, "aleph_api": false }, "resources": { diff --git a/firecracker/microvm.py b/firecracker/microvm.py index f9d74b4c7..d048e0670 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -87,6 +87,14 @@ def vsock_path(self): else: return f"{VSOCK_PATH}" + @property + def guest_ip(self): + return f"172.{self.vm_id // 256}.{self.vm_id % 256}.2" + + @property + def host_ip(self): + return f"172.{self.vm_id // 256}.{self.vm_id % 256}.1" + def __init__( self, vm_id: int, @@ -238,12 +246,12 @@ async def set_network(self): """Configure the host network with a tap interface to the VM.""" logger.debug("Network setup") - name = f"tap{self.vm_id}" + name = f"vmtap{self.vm_id}" self.network_tap = name system(f"ip tuntap add {name} mode tap") system( - f"ip addr add 172.{self.vm_id // 256}.{self.vm_id % 256}.1/24 dev {name}" + f"ip addr add {self.host_ip}/24 dev {name}" ) system(f"ip link set {name} up") system('sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"') @@ -320,6 +328,7 @@ async def stop(self): if self.proc: self.proc.terminate() self.proc.kill() + self.proc = None await self.get_session().close() self.get_session.cache_clear() @@ -335,6 +344,8 @@ async def teardown(self): self.stderr_task.cancel() if self.network_tap: + await asyncio.sleep(0.01) # Used to prevent `ioctl(TUNSETIFF): Device or resource busy` + logger.debug(f"Removing interface {self.network_tap}") system(f"ip tuntap del {self.network_tap} mode tap") system(f"rm -fr {self.jailer_path}") diff --git a/runtimes/aleph-alpine-3.13-python/init0.sh b/runtimes/aleph-alpine-3.13-python/init0.sh index e815ed4cd..028eed32c 100644 --- a/runtimes/aleph-alpine-3.13-python/init0.sh +++ b/runtimes/aleph-alpine-3.13-python/init0.sh @@ -33,16 +33,6 @@ mount -t tmpfs run /run -o mode=0755,nosuid,nodev mount -t devpts devpts /dev/pts -o mode=0620,gid=5,nosuid,noexec mount -t tmpfs shm /dev/shm -omode=1777,nosuid,nodev -# TODO: Move in init1 -if [[ -d /sys/class/net/eth0 ]]; then - ip addr add 172.0.5.2/24 dev eth0 - ip link set eth0 up - ip route add default via 172.0.5.1 dev eth0 - ip addr -fi - -log "Net up" - #cat /proc/sys/kernel/random/entropy_avail # TODO: Move in init1 diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index c2358cd53..e76aca453 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -31,6 +31,12 @@ class Encoding: zip = "zip" +@dataclass +class ConfigurationPayload: + ip: str + route: str + + @dataclass class RunCodePayload: code: bytes @@ -56,6 +62,17 @@ class RunCodePayload: logger.debug("init1.py is launching") +def setup_network(ip: str, route: str): + """Setup the system with info from the host.""" + if os.path.exists("/sys/class/net/eth0"): + logger.debug("Setting up networking") + system(f"ip addr add {ip}/24 dev eth0") + system("ip link set eth0 up") + system(f"ip route add default via {route} dev eth0") + else: + logger.info("No network interface eth0") + + async def run_python_code_http(code: bytes, input_data: Optional[bytes], entrypoint: str, encoding: str, scope: dict ) -> Tuple[Dict, Dict, str, Optional[bytes]]: @@ -165,6 +182,13 @@ def process_instruction(instruction: bytes) -> Iterator[bytes]: def main(): + client, addr = s.accept() + data = client.recv(1000_1000) + msg_ = msgpack.loads(data, raw=False) + + payload = ConfigurationPayload(**msg_) + setup_network(payload.ip, payload.route) + while True: client, addr = s.accept() data = client.recv(1000_1000) # Max 1 Mo diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index c2e83e10d..c73332215 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -49,6 +49,7 @@ async def create_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: ) await vm.setup() await vm.start() + await vm.configure() await vm.start_guest_api() return vm diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 68e08d05a..d2cc23e6e 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -19,6 +19,15 @@ set_start_method("spawn") +@dataclass +class ConfigurationPayload: + ip: Optional[str] + route: Optional[str] + + def as_msgpack(self) -> bytes: + return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True) + + @dataclass class RunCodePayload: code: bytes @@ -136,6 +145,16 @@ async def start(self): ) logger.debug(f"started fvm {self.vm_id}") + async def configure(self): + """Configure the VM by sending configuration info to it's init""" + reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path) + payload = ConfigurationPayload( + ip=self.fvm.guest_ip if self.enable_networking else None, + route=self.fvm.host_ip if self.enable_console else None, + ) + writer.write(b"CONNECT 52\n" + payload.as_msgpack()) + await writer.drain() + async def start_guest_api(self): logger.debug(f"starting guest API for {self.vm_id}") vsock_path = f"{self.fvm.vsock_path}_53" From 59485d3547c0cd8daaf3277e7b36791ef4a1c41c Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 18 May 2021 11:52:06 +0200 Subject: [PATCH 027/990] Cleanup: Shell syntax was not completely standard --- runtimes/aleph-alpine-3.13-python/init0.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init0.sh b/runtimes/aleph-alpine-3.13-python/init0.sh index 028eed32c..7860b9399 100644 --- a/runtimes/aleph-alpine-3.13-python/init0.sh +++ b/runtimes/aleph-alpine-3.13-python/init0.sh @@ -4,8 +4,8 @@ set -euf mount -t proc proc /proc -o nosuid,noexec,nodev -function log() { - echo `cat /proc/uptime | awk '{printf $1}'` '|S' $@ +log() { + echo "$(cat /proc/uptime | awk '{printf $1}')" '|S' "$@" } log "init0.sh is launching" From 13a517331e29d6e38b2bfdf1ef49ecf222c9b0e7 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 18 May 2021 13:39:43 +0200 Subject: [PATCH 028/990] Fix: Handle empty IP config in init --- runtimes/aleph-alpine-3.13-python/init1.py | 26 +++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index e76aca453..d638a37a7 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -33,8 +33,8 @@ class Encoding: @dataclass class ConfigurationPayload: - ip: str - route: str + ip: Optional[str] + route: Optional[str] @dataclass @@ -62,15 +62,25 @@ class RunCodePayload: logger.debug("init1.py is launching") -def setup_network(ip: str, route: str): +def setup_network(ip: Optional[str], route: Optional[str]): """Setup the system with info from the host.""" - if os.path.exists("/sys/class/net/eth0"): - logger.debug("Setting up networking") - system(f"ip addr add {ip}/24 dev eth0") - system("ip link set eth0 up") + if not os.path.exists("/sys/class/net/eth0"): + logger.info("No network interface eth0") + return + + if not ip: + logger.info("No network IP") + return + + logger.debug("Setting up networking") + system(f"ip addr add {ip}/24 dev eth0") + system("ip link set eth0 up") + + if route: system(f"ip route add default via {route} dev eth0") + logger.debug("IP and route set") else: - logger.info("No network interface eth0") + logger.warning("IP set with no network route") async def run_python_code_http(code: bytes, input_data: Optional[bytes], From 3d8218c7787f168ed0ebe0aa85072deb9cfd2fa1 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 18 May 2021 15:59:26 +0200 Subject: [PATCH 029/990] Feature: Network interface was not configurable --- firecracker/microvm.py | 18 +++++++++++++++--- vm_supervisor/conf.py | 4 +++- vm_supervisor/vm/firecracker_microvm.py | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/firecracker/microvm.py b/firecracker/microvm.py index d048e0670..825700dc9 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -65,6 +65,7 @@ class MicroVM: jailer_bin_path: Optional[str] proc: Optional[asyncio.subprocess.Process] = None network_tap: Optional[str] = None + network_interface: Optional[str] = None stdout_task: Optional[Task] = None stderr_task: Optional[Task] = None @@ -242,10 +243,12 @@ async def set_vsock(self): response = await session.put("http://localhost/vsock", json=data) response.raise_for_status() - async def set_network(self): + async def set_network(self, interface: str = "eth0"): """Configure the host network with a tap interface to the VM.""" logger.debug("Network setup") + self.network_interface = interface + name = f"vmtap{self.vm_id}" self.network_tap = name @@ -256,11 +259,11 @@ async def set_network(self): system(f"ip link set {name} up") system('sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"') # TODO: Don't fill iptables with duplicate rules; purge rules on delete - system("iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE") + system(f"iptables -t nat -A POSTROUTING -o {interface} -j MASQUERADE") system( "iptables -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT" ) - system(f"iptables -A FORWARD -i {name} -o eth0 -j ACCEPT") + system(f"iptables -A FORWARD -i {name} -o {interface} -j ACCEPT") data = { "iface_id": "eth0", @@ -347,7 +350,16 @@ async def teardown(self): await asyncio.sleep(0.01) # Used to prevent `ioctl(TUNSETIFF): Device or resource busy` logger.debug(f"Removing interface {self.network_tap}") system(f"ip tuntap del {self.network_tap} mode tap") + logger.debug("Removing iptables rules") + system(f"iptables -t nat -D POSTROUTING -o {self.network_interface} -j MASQUERADE") + system( + "iptables -D FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT" + ) + system( + f"iptables -D FORWARD -i {self.network_tap} -o {self.network_interface} -j ACCEPT" + ) + logger.debug("Removing files") system(f"rm -fr {self.jailer_path}") diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index b3bfcbe60..b8f8baf17 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -1,6 +1,6 @@ import logging import os -from os.path import isfile, join +from os.path import isfile, join, exists from typing import NewType from pydantic import BaseSettings @@ -15,6 +15,7 @@ class Settings(BaseSettings): START_ID_INDEX: int = 4 PREALLOC_VM_COUNT: int = 0 REUSE_TIMEOUT: float = 60 * 60.0 + NETWORK_INTERFACE: str = "eth0" API_SERVER: str = "https://api2.aleph.im" USE_JAILER: bool = True @@ -52,6 +53,7 @@ def check(self): assert self.CONNECTOR_URL.startswith( "http://" ) or self.CONNECTOR_URL.startswith("https://") + assert exists(f"/sys/class/net/{self.NETWORK_INTERFACE}") def setup(self): os.makedirs(self.MESSAGE_CACHE, exist_ok=True) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index d2cc23e6e..c2beb9440 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -125,7 +125,7 @@ async def setup(self): await fvm.set_rootfs(self.resources.rootfs_path) await fvm.set_vsock() if self.enable_networking: - await fvm.set_network() + await fvm.set_network(interface=settings.NETWORK_INTERFACE) logger.debug("setup done") self.fvm = fvm From 2bc74c6e9e21d8438408a3793a1db514c1986a59 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 19 May 2021 11:39:58 +0200 Subject: [PATCH 030/990] Feature: Add support of HEAD and OPTIONS in guest API --- guest_api/__main__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/guest_api/__main__.py b/guest_api/__main__.py index eb3abaab9..c05ac8743 100644 --- a/guest_api/__main__.py +++ b/guest_api/__main__.py @@ -5,13 +5,14 @@ ALEPH_API_SERVER = "https://api2.aleph.im/" -async def proxy(request): +async def proxy(request: web.Request): + path = request.match_info.get('tail') query_string = request.rel_url.query_string url = f"{ALEPH_API_SERVER}{path}?{query_string}" async with aiohttp.ClientSession() as session: - async with session.get(url) as response: + async with session.request(method=request.method, url=url) as response: data = await response.read() return web.Response(body=data, status=response.status, @@ -21,6 +22,8 @@ async def proxy(request): def run_guest_api(unix_socket_path): app = web.Application() app.router.add_route(method='GET', path='/{tail:.*}', handler=proxy) + app.router.add_route(method='HEAD', path='/{tail:.*}', handler=proxy) + app.router.add_route(method='OPTIONS', path='/{tail:.*}', handler=proxy) web.run_app(app=app, path=unix_socket_path) From d1704e691752d95252fa37df98d5690eaf6a6483 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 19 May 2021 11:40:32 +0200 Subject: [PATCH 031/990] Fix: Network interface check failed when networking was disabled --- vm_supervisor/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index b8f8baf17..fad5474a3 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -53,7 +53,8 @@ def check(self): assert self.CONNECTOR_URL.startswith( "http://" ) or self.CONNECTOR_URL.startswith("https://") - assert exists(f"/sys/class/net/{self.NETWORK_INTERFACE}") + if self.ALLOW_VM_NETWORKING: + assert exists(f"/sys/class/net/{self.NETWORK_INTERFACE}") def setup(self): os.makedirs(self.MESSAGE_CACHE, exist_ok=True) From 48ecc9ea3d717e934e15f4befb8967bb8ccbb246 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 19 May 2021 11:41:00 +0200 Subject: [PATCH 032/990] Enhancement: Better logging in init1.py --- runtimes/aleph-alpine-3.13-python/init1.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index d638a37a7..f3a5fbef4 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -205,7 +205,10 @@ def main(): logger.debug("CID: {} port:{} data: {}".format(addr[0], addr[1], len(data))) logger.debug("Init received msg") - print(f"<<<\n\n{data}\n\n>>>") + if logger.level <= logging.DEBUG: + data_to_print = f"{data[:500]}..." if len(data) > 500 else data + logger.debug(f"<<<\n\n{data_to_print}\n\n>>>") + for result in process_instruction(instruction=data): client.send(result) @@ -214,4 +217,5 @@ def main(): if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) main() From 1a42c2af81b6fd3f9207433f835aa4840d1f5aff Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 19 May 2021 11:38:04 +0200 Subject: [PATCH 033/990] Feature: Support reverse-proxy with hash in hostname --- vm_supervisor/README.md | 102 +++++++++++++++++++++++++++++++++--- vm_supervisor/__main__.py | 6 ++- vm_supervisor/supervisor.py | 25 +++++++-- 3 files changed, 119 insertions(+), 14 deletions(-) diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index fc0dc4ae0..db1a9854a 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -36,13 +36,17 @@ of Ubuntu as well (20.04+). Bare metal servers from most hosting providers should be compatible with the VM Supervisor. A few hosting providers offer compatible virtual machines. -- Compatible ✓ : DigitalOcean Droplet. -- Incompatible ✖ : AWS EC2. +- Compatible ✓ : DigitalOcean Droplet. AWS ECS Bare Metal. +- Incompatible ✖ : AWS EC2 other than Bare Metal. + +Probably [Google Cloud instances with Nested Virtualization](https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances). ### Note on containers While not supported at the moment, it is possible to run the VM Supervisor inside a Docker -container. This will be less secure since the `Jailer` tool used to secure Firecracker MicroVMs +container. + +This will be less secure since the `Jailer` tool used to secure Firecracker MicroVMs will not run inside containers. Pass the command-lien argument `--no-jailer` to disable the Jailer when running the VM Supervisor. @@ -92,7 +96,7 @@ apt install -y --no-install-recommends --no-install-suggests python3-pip pip3 install pydantic ``` -### 2.f. Setup the jailer working directory: +### 2.f. Create the jailer working directory: ```shell mkdir /srv/jailer @@ -100,6 +104,11 @@ mkdir /srv/jailer ### 2.g. Download a Linux kernel +This downloads the example kernel built by the Firecracker team. + +A more optimized kernel will be made available in the future. +See section _Compile your kernel_ below to build your own. + ```shell curl -fsSL -o ./kernels/vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin ``` @@ -130,14 +139,91 @@ and using environment variables, which can be found using: python3 -m vm_supervisor --print-config --do-not-run ``` -## 5. Production configuration +## 5. Reverse-proxy + +A reverse-proxy is required for production use. It allows: + + - A different domain name for each VM function + - Secure connections using HTTPS + - Load balancing between multiple servers + +Using a different domain name for each VM function is important when running web applications, +both for security and usability purposes. + +The VM Supervisor supports using domains in the form `https://identifer.vm.yourdomain.org`, where +_identifier_ is the identifier/hash of the message describing the VM function and `yourdomain.org` +represents your domain name. + +### 5.a. Wildcard certificates + +A wildcard certificate is recommended to allow any subdomain of your domain to work. + +You can create one using [Let's Encrypt](https://letsencrypt.org/) and +[Certbot](https://certbot.eff.org/) with the following instructions. + +```shell +sudo apt install -y certbot + +certbot certonly --manual --email email@yourdomain.org --preferred-challenges dns \ + --server https://acme-v02.api.letsencrypt.org/directory --agree-tos \ + -d 'vm.yourdomain.org,*.vm.youdomain.org' +``` + +### 5.b. Reverse Proxy + +In this documentation, we will install the modern [Caddy](https://caddyserver.com/) reverse-proxy. + +To install on Debian/Ubuntu, according to the +[official instructions](https://caddyserver.com/docs/install#debian-ubuntu-raspbian): +```shell +sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https +curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo apt-key add - +curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list +sudo apt update +sudo apt install caddy +``` + +Then give Caddy access to the certificates generated by Certbot: +```shell +chmod 750 /etc/letsencrypt/live/ +chmod 750 /etc/letsencrypt/archive/ +chmod 640 /etc/letsencrypt/archive/vm.yourdomain.org/privkey1.pem +chgrp -R caddy /etc/letsencrypt/archive/ +chgrp -R caddy /etc/letsencrypt/live/ +``` + +Configure Caddy: +```shell +cat >/etc/caddy/Caddyfile < bytes: return b"" -async def run_code(request: web.Request) -> web.Response: +async def run_code(message_ref: str, request: web.Request) -> web.Response: """ Execute the code corresponding to the 'code id' in the path. """ - message_ref: str = request.match_info["ref"] + message = await try_get_message(message_ref) # vm_resources = AlephFirecrackerResources(message) @@ -127,10 +128,26 @@ async def run_code(request: web.Request) -> web.Response: await vm.teardown() +def run_code_from_path(request: web.Request) -> Awaitable[web.Response]: + message_ref: str = request.match_info["ref"] + return run_code(message_ref, request) + + +async def run_code_from_hostname(request: web.Request) -> web.Response: + hostname = request.host + split = hostname.split(".") + + if len(split) < 3 or split[1] != "vm": + return web.Response(status=404, reason="Domain does not contain a message ref") + + message_ref = hostname.split(".")[0] + return await run_code(message_ref, request) + + app = web.Application() -app.add_routes([web.get("/", index)]) -app.add_routes([web.route("*", "/vm/function/{ref}{suffix:.*}", run_code)]) +app.add_routes([web.route("*", "/vm/function/{ref}{suffix:.*}", run_code_from_path)]) +app.add_routes([web.route("*", "/{suffix:.*}", run_code_from_hostname)]) def run(): From 7461b00ced736ed621040c4bd92fd61b3b1be09c Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 21 May 2021 12:38:32 +0200 Subject: [PATCH 034/990] Fix: Allow wildcard hostnames without 'vm' --- vm_supervisor/supervisor.py | 56 +++++++++++++------------ vm_supervisor/vm/firecracker_microvm.py | 28 +++++++++++-- 2 files changed, 54 insertions(+), 30 deletions(-) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 28a25b782..c459a8764 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -6,17 +6,18 @@ evolve in the future. """ import logging -from typing import Awaitable +from typing import Awaitable, Dict, Any import msgpack from aiohttp import web, ClientResponseError, ClientConnectorError -from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable +from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable, HTTPBadRequest from msgpack import UnpackValueError from .conf import settings from .models import FilePath, FunctionMessage from .pool import VmPool from .storage import get_message +from .vm.firecracker_microvm import ResourceDownloadError logger = logging.getLogger(__name__) pool = VmPool() @@ -40,11 +41,7 @@ async def try_get_message(ref: str) -> FunctionMessage: raise -def build_asgi_scope(request: web.Request): - path = request.match_info["suffix"] - if not path.startswith("/"): - path = "/" + path - +def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: return { "type": "http", "path": path, @@ -62,27 +59,22 @@ def load_file_content(path: FilePath) -> bytes: return b"" -async def run_code(message_ref: str, request: web.Request) -> web.Response: +async def run_code(message_ref: str, path: str, request: web.Request) -> web.Response: """ Execute the code corresponding to the 'code id' in the path. """ message = await try_get_message(message_ref) - # vm_resources = AlephFirecrackerResources(message) - # - # try: - # await vm_resources.download_all() - # except ClientResponseError as error: - # if error.status == 404: - # raise HTTPBadRequest(reason="Code, runtime or data not found") - # else: - # raise - - vm = await pool.get_a_vm(message) + try: + vm = await pool.get_a_vm(message) + except ResourceDownloadError as error: + logger.exception(error) + raise HTTPBadRequest(reason="Code, runtime or data not available") + logger.debug(f"Using vm={vm.vm_id}") - scope = build_asgi_scope(request) + scope: Dict = build_asgi_scope(path, request) code: bytes = load_file_content(vm.resources.code_path) input_data: bytes = load_file_content(vm.resources.data_path) @@ -129,19 +121,29 @@ async def run_code(message_ref: str, request: web.Request) -> web.Response: def run_code_from_path(request: web.Request) -> Awaitable[web.Response]: + """Allow running an Aleph VM function from a URL path + + The path is expected to follow the scheme defined in `app.add_routes` below, + where the identifier of the message is named `ref`. + """ + path = request.match_info["suffix"] + path = path if path.startswith("/") else f"/{path}" + message_ref: str = request.match_info["ref"] - return run_code(message_ref, request) + return run_code(message_ref, path, request) async def run_code_from_hostname(request: web.Request) -> web.Response: - hostname = request.host - split = hostname.split(".") + """Allow running an Aleph VM function from a hostname - if len(split) < 3 or split[1] != "vm": - return web.Response(status=404, reason="Domain does not contain a message ref") + The first component of the hostname is used as identifier of the message defining the + Aleph VM function. + """ + path = request.match_info["suffix"] + path = path if path.startswith("/") else f"/{path}" - message_ref = hostname.split(".")[0] - return await run_code(message_ref, request) + message_ref = request.host.split(".")[0] + return await run_code(message_ref, path, request) app = web.Application() diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index c2beb9440..7432df957 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -8,6 +8,7 @@ from typing import Optional, Dict import msgpack +from aiohttp import ClientResponseError from firecracker.microvm import MicroVM, setfacl from guest_api.__main__ import run_guest_api @@ -19,6 +20,18 @@ set_start_method("spawn") +class ResourceDownloadError(ClientResponseError): + """An error occurred while downloading a VM resource file""" + + def __init__(self, error: ClientResponseError): + super().__init__( + request_info=error.request_info, + history=error.history, + status=error.status, + message=error.message, + headers=error.headers, + ) + @dataclass class ConfigurationPayload: ip: Optional[str] @@ -59,18 +72,27 @@ async def download_kernel(self): async def download_code(self): code_ref: str = self.message.content.code.ref - self.code_path = await get_code_path(code_ref) + try: + self.code_path = await get_code_path(code_ref) + except ClientResponseError as error: + raise ResourceDownloadError(error) assert isfile(self.code_path) async def download_runtime(self): runtime_ref: str = self.message.content.runtime.ref - self.rootfs_path = await get_runtime_path(runtime_ref) + try: + self.rootfs_path = await get_runtime_path(runtime_ref) + except ClientResponseError as error: + raise ResourceDownloadError(error) assert isfile(self.rootfs_path) async def download_data(self): if self.message.content.data: data_ref: str = self.message.content.data.ref - self.data_path = await get_data_path(data_ref) + try: + self.data_path = await get_data_path(data_ref) + except ClientResponseError as error: + raise ResourceDownloadError(error) assert isfile(self.data_path) else: self.data_path = None From 0d35efff4efed345730b6a8916ce3a8ad13b40ae Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 21 May 2021 12:40:13 +0200 Subject: [PATCH 035/990] Fix: Issue in Caddy configuration --- vm_supervisor/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index db1a9854a..c29fc4990 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -198,7 +198,10 @@ cat >/etc/caddy/Caddyfile < Date: Fri, 21 May 2021 14:26:13 +0200 Subject: [PATCH 036/990] Fix: Unix sessions only closed at VM teardown They could accumulate on heavy workload, leading to performance issues. --- firecracker/microvm.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 825700dc9..6170998cb 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -108,7 +108,6 @@ def __init__( self.firecracker_bin_path = firecracker_bin_path self.jailer_bin_path = jailer_bin_path - @lru_cache() def get_session(self) -> aiohttp.ClientSession: conn = aiohttp.UnixConnector(path=self.socket_path) return aiohttp.ClientSession(connector=conn) @@ -210,10 +209,10 @@ async def set_boot_source( # Add console=ttyS0 for debugging, but it makes the boot twice slower "boot_args": f"{console} reboot=k panic=1 pci=off ro noapic nomodules random.trust_cpu=on", } - session = self.get_session() - response: ClientResponse = await session.put( - "http://localhost/boot-source", json=data - ) + async with self.get_session() as session: + response: ClientResponse = await session.put( + "http://localhost/boot-source", json=data + ) response.raise_for_status() async def set_rootfs(self, path_on_host: str): @@ -229,8 +228,8 @@ async def set_rootfs(self, path_on_host: str): "is_root_device": True, "is_read_only": True, } - session = self.get_session() - response = await session.put("http://localhost/drives/rootfs", json=data) + async with self.get_session() as session: + response = await session.put("http://localhost/drives/rootfs", json=data) response.raise_for_status() async def set_vsock(self): @@ -239,8 +238,8 @@ async def set_vsock(self): "guest_cid": 3, "uds_path": VSOCK_PATH, } - session = self.get_session() - response = await session.put("http://localhost/vsock", json=data) + async with self.get_session() as session: + response = await session.put("http://localhost/vsock", json=data) response.raise_for_status() async def set_network(self, interface: str = "eth0"): @@ -270,10 +269,10 @@ async def set_network(self, interface: str = "eth0"): "guest_mac": f"AA:FC:00:00:00:01", "host_dev_name": name, } - session = self.get_session() - response = await session.put( - "http://localhost/network-interfaces/eth0", json=data - ) + async with self.get_session() as session: + response = await session.put( + "http://localhost/network-interfaces/eth0", json=data + ) logger.debug(response) logger.debug(await response.text()) response.raise_for_status() @@ -282,8 +281,8 @@ async def start_instance(self): data = { "action_type": "InstanceStart", } - session = self.get_session() - response = await session.put("http://localhost/actions", json=data) + async with self.get_session() as session: + response = await session.put("http://localhost/actions", json=data) response.raise_for_status() async def print_logs(self): @@ -333,10 +332,6 @@ async def stop(self): self.proc.kill() self.proc = None - await self.get_session().close() - self.get_session.cache_clear() - - async def teardown(self): """Stop the VM, cleanup network interface and remove data directory.""" await self.stop() From 89407b1eb07b2a42e8bc1a78ef3790277b3d3646 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 21 May 2021 14:54:38 +0200 Subject: [PATCH 037/990] Fix: Status,headers from VM were not forwarded --- vm_supervisor/supervisor.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index c459a8764..5477bb3aa 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -107,8 +107,13 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res content_type="text/plain", ) + headers = {key.decode(): value.decode() + for key, value in result['headers']['headers']} + return web.Response( - body=result["body"]["body"], content_type="application/json" + status=result['headers']['status'], + body=result["body"]["body"], + headers=headers, ) except UnpackValueError as error: logger.exception(error) From bf35550eb08ae9b4b040a77994d76ebe5ca134ea Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 21 May 2021 15:05:35 +0200 Subject: [PATCH 038/990] Enhancement: Test internet connection in example_fastapi_2 --- examples/example_fastapi_2/__init__.py | 28 +++++++++++--------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/examples/example_fastapi_2/__init__.py b/examples/example_fastapi_2/__init__.py index f30c949c5..0efdc5de6 100644 --- a/examples/example_fastapi_2/__init__.py +++ b/examples/example_fastapi_2/__init__.py @@ -1,8 +1,9 @@ import logging -from typing import Optional logger = logging.getLogger(__name__) -logger.debug("") +logger.debug("import aiohttp") +import aiohttp + logger.debug("import aleph_client") from aleph_client.asynchronous import get_messages @@ -13,32 +14,27 @@ app = FastAPI() -async def get_data_http(): - return "Have a look at /messages" - - @app.get("/") async def index(): - data = await get_data_http() return { "Example": "example_fastapi_2", - "endpoints": ["/messages", "/run/{item_id}"], + "endpoints": ["/messages", "/internet"], } @app.get("/messages") async def read_aleph_messages(): + """Read data from Aleph using the Aleph Client library.""" data = await get_messages( hashes=["f246f873c3e0f637a15c566e7a465d2ecbb83eaa024d54ccb8fb566b549a929e"] ) return {"Messages": data} -@app.get("/run/{item_id}") -def read_item(item_id: str, q: Optional[str] = None): - return {"pyz item_id": item_id, "q": q} - - -@app.post("/run/{item_id}") -def read_item_post(item_id: str, q: Optional[str] = None): - return {"pyz item_id_post": item_id, "q": q} +@app.get("/internet") +async def read_internet(): + """Read data from the public Internet using aiohttp.""" + async with aiohttp.ClientSession(connector=aiohttp.TCPConnector()) as session: + async with session.get("https://aleph.im/") as resp: + resp.raise_for_status() + return {"result": resp.status, "headers": resp.headers} From e357b91ff26c43391910317e362fb2c56206de12 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 21 May 2021 16:24:17 +0200 Subject: [PATCH 039/990] Enhancement: Add support to extract DNS servers from host Both from resolv.conf (Debian) and from systemd-resolved (Ubuntu) --- runtimes/aleph-alpine-3.13-python/init1.py | 11 ++-- vm_supervisor/__main__.py | 2 +- vm_supervisor/conf.py | 62 +++++++++++++++++++++- vm_supervisor/vm/firecracker_microvm.py | 4 +- 4 files changed, 73 insertions(+), 6 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index f3a5fbef4..92d1c1d13 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -20,7 +20,7 @@ from io import StringIO from os import system from shutil import make_archive -from typing import Optional, Dict, Any, Tuple, Iterator +from typing import Optional, Dict, Any, Tuple, Iterator, List import msgpack @@ -35,6 +35,7 @@ class Encoding: class ConfigurationPayload: ip: Optional[str] route: Optional[str] + dns_servers: List[str] @dataclass @@ -62,7 +63,7 @@ class RunCodePayload: logger.debug("init1.py is launching") -def setup_network(ip: Optional[str], route: Optional[str]): +def setup_network(ip: Optional[str], route: Optional[str], dns_servers: List[str] = []): """Setup the system with info from the host.""" if not os.path.exists("/sys/class/net/eth0"): logger.info("No network interface eth0") @@ -82,6 +83,10 @@ def setup_network(ip: Optional[str], route: Optional[str]): else: logger.warning("IP set with no network route") + with open("/etc/resolv.conf", "wb") as resolvconf_fd: + for server in dns_servers: + resolvconf_fd.write(f"nameserver {server}\n".encode()) + async def run_python_code_http(code: bytes, input_data: Optional[bytes], entrypoint: str, encoding: str, scope: dict @@ -197,7 +202,7 @@ def main(): msg_ = msgpack.loads(data, raw=False) payload = ConfigurationPayload(**msg_) - setup_network(payload.ip, payload.route) + setup_network(payload.ip, payload.route, payload.dns_servers) while True: client, addr = s.accept() diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 1e933ed94..c2619ad4b 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -140,6 +140,7 @@ def main(): PREALLOC_VM_COUNT=args.prealloc_vm_count, ALLOW_VM_NETWORKING=args.allow_vm_networking, ) + settings.setup() if args.print_settings: print(settings.display()) @@ -152,7 +153,6 @@ def main(): elif args.do_not_run: logger.info("Option --do-not-run, exiting") else: - settings.setup() supervisor.run() diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index fad5474a3..1b6315962 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -1,7 +1,10 @@ import logging import os +import re +from enum import Enum from os.path import isfile, join, exists -from typing import NewType +from subprocess import check_output +from typing import NewType, Optional, List from pydantic import BaseSettings from .models import FilePath @@ -11,11 +14,56 @@ Url = NewType("Url", str) +class DnsResolver(str, Enum): + resolv_conf = "resolv.conf" # Simply copy from /etc/resolv.conf + resolvectl = "resolvectl" # Systemd-resolved, common on Ubuntu + + +def etc_resolv_conf_dns_servers(): + with open("/etc/resolv.conf", "r") as resolv_file: + for line in resolv_file.readlines(): + ip = re.findall(r"^nameserver\s+([\w.]+)$", line) + if ip: + yield ip[0] + + +def systemd_resolved_dns_servers(interface): + ## Example output format from systemd-resolve --status {interface}: + # Link 2 (enp7s0) + # Current Scopes: DNS + # DefaultRoute setting: yes + # LLMNR setting: yes + # MulticastDNS setting: no + # DNSOverTLS setting: no + # DNSSEC setting: no + # DNSSEC supported: no + # Current DNS Server: 213.133.100.100 + # DNS Servers: 213.133.100.100 + # 213.133.98.98 + # 213.133.99.99 + # 2a01:4f8:0:1::add:9898 + # 2a01:4f8:0:1::add:1010 + # 2a01:4f8:0:1::add:9999 + output = check_output(["/usr/bin/systemd-resolve", "--status", interface]) + nameserver_line = False + for line in output.split(b"\n"): + if b"DNS Servers" in line: + nameserver_line = True + _, ip = line.decode().split(":", 1) + yield ip.strip() + elif nameserver_line: + ip = line.decode().strip() + if ip: + yield ip + + class Settings(BaseSettings): START_ID_INDEX: int = 4 PREALLOC_VM_COUNT: int = 0 REUSE_TIMEOUT: float = 60 * 60.0 NETWORK_INTERFACE: str = "eth0" + DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf + DNS_NAMESERVERS: Optional[List[str]] = None API_SERVER: str = "https://api2.aleph.im" USE_JAILER: bool = True @@ -62,6 +110,18 @@ def setup(self): os.makedirs(self.RUNTIME_CACHE, exist_ok=True) os.makedirs(self.DATA_CACHE, exist_ok=True) + if self.DNS_NAMESERVERS is None and self.DNS_RESOLUTION: + if self.DNS_RESOLUTION == DnsResolver.resolv_conf: + self.DNS_NAMESERVERS = list(etc_resolv_conf_dns_servers()) + + + elif self.DNS_RESOLUTION == DnsResolver.resolvectl: + self.DNS_NAMESERVERS = list(systemd_resolved_dns_servers( + interface=self.NETWORK_INTERFACE)) + else: + assert "This should never happen" + + def display(self) -> str: return "\n".join( f"{annotation:<17} = {getattr(self, annotation)}" diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 7432df957..c322fb5df 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -5,7 +5,7 @@ from multiprocessing import Process, set_start_method from os import system from os.path import isfile, exists -from typing import Optional, Dict +from typing import Optional, Dict, List import msgpack from aiohttp import ClientResponseError @@ -36,6 +36,7 @@ def __init__(self, error: ClientResponseError): class ConfigurationPayload: ip: Optional[str] route: Optional[str] + dns_servers: List[str] def as_msgpack(self) -> bytes: return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True) @@ -173,6 +174,7 @@ async def configure(self): payload = ConfigurationPayload( ip=self.fvm.guest_ip if self.enable_networking else None, route=self.fvm.host_ip if self.enable_console else None, + dns_servers=settings.DNS_NAMESERVERS, ) writer.write(b"CONNECT 52\n" + payload.as_msgpack()) await writer.drain() From 48c06590558c74f74316e72e2b222b997f632d49 Mon Sep 17 00:00:00 2001 From: Jonathan Schemoul Date: Fri, 21 May 2021 18:03:05 +0200 Subject: [PATCH 040/990] add env file --- vm_supervisor/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 1b6315962..100fc9fa0 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -131,6 +131,7 @@ def display(self) -> str: class Config: env_prefix = "ALEPH_VM_" case_sensitive = False + env_file = '.env' # Settings singleton From fb232989628102ae05a7ff5be0c346f6b7b5b4bd Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 26 May 2021 14:26:31 +0200 Subject: [PATCH 041/990] Enhancement: Add support to configure hardware resources --- firecracker/microvm.py | 12 ++++++++++++ vm_supervisor/pool.py | 1 + vm_supervisor/vm/firecracker_microvm.py | 7 ++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 6170998cb..ea1170037 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -277,6 +277,18 @@ async def set_network(self, interface: str = "eth0"): logger.debug(await response.text()) response.raise_for_status() + async def set_resources(self, vcpus: int = 1, memory: int = 128, + ht_enabled: bool = False): + """Set machine resources (number of CPU cores, memory)""" + data = { + "vcpu_count": vcpus, + "mem_size_mib": memory, + "ht_enabled": ht_enabled, + } + async with self.get_session() as session: + response = await session.put("http://localhost/machine-config", json=data) + response.raise_for_status() + async def start_instance(self): data = { "action_type": "InstanceStart", diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index c73332215..d72729737 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -46,6 +46,7 @@ async def create_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: vm_id=self.counter, resources=vm_resources, enable_networking=message.content.environment.internet, + hardware_resources=message.content.resources, ) await vm.setup() await vm.start() diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index c322fb5df..45a07e9da 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -13,7 +13,7 @@ from firecracker.microvm import MicroVM, setfacl from guest_api.__main__ import run_guest_api from ..conf import settings -from ..models import FunctionMessage, FilePath +from ..models import FunctionMessage, FilePath, FunctionResources from ..storage import get_code_path, get_runtime_path, get_data_path logger = logging.getLogger(__name__) @@ -112,6 +112,7 @@ class AlephFirecrackerVM: resources: AlephFirecrackerResources enable_console: bool enable_networking: bool + hardware_resources: FunctionResources fvm: MicroVM guest_api_process: Process @@ -121,6 +122,7 @@ def __init__( resources: AlephFirecrackerResources, enable_networking: bool = False, enable_console: Optional[bool] = None, + hardware_resources: FunctionResources = FunctionResources() ): self.vm_id = vm_id self.resources = resources @@ -128,6 +130,7 @@ def __init__( if enable_console is None: enable_console = settings.PRINT_SYSTEM_LOGS self.enable_console = enable_console + self.hardware_resources = hardware_resources async def setup(self): logger.debug("setup started") @@ -147,6 +150,8 @@ async def setup(self): ) await fvm.set_rootfs(self.resources.rootfs_path) await fvm.set_vsock() + await fvm.set_resources(vcpus=self.hardware_resources.vcpus, + memory=self.hardware_resources.memory) if self.enable_networking: await fvm.set_network(interface=settings.NETWORK_INTERFACE) logger.debug("setup done") From 3ae7bede5c32e18b4d198fc8815cf50637582da0 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 26 May 2021 15:52:40 +0200 Subject: [PATCH 042/990] Refactor: Use aleph-message to validate PROGRAM messages The old model was obsoleted by the new specification. --- vm_supervisor/models.py | 68 +------------------------ vm_supervisor/pool.py | 18 +++---- vm_supervisor/storage.py | 9 ++-- vm_supervisor/supervisor.py | 16 +++--- vm_supervisor/vm/firecracker_microvm.py | 22 ++++---- 5 files changed, 35 insertions(+), 98 deletions(-) diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index d1e7795bd..b79743982 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -1,69 +1,3 @@ -from enum import Enum -from typing import NewType, Optional - -from pydantic import BaseModel +from typing import NewType FilePath = NewType("FilePath", str) - - -class HashableModel(BaseModel): - def __hash__(self): - return hash(self.__class__) + hash(tuple(self.__dict__.values())) - - -class Encoding(str, Enum): - plain = "plain" - zip = "zip" - targz = "tar.gzip" - - -class CodeContent(HashableModel): - encoding: Encoding - entrypoint: str - ref: str - latest_amend: bool = True - - -class DataContent(HashableModel): - encoding: Encoding - mount: str - ref: str - latest_amend: bool = True - - -class FunctionTriggers(HashableModel): - http: bool - - -class FunctionEnvironment(HashableModel): - reproducible: bool = False - internet: bool = False - aleph_api: bool = False - - -class FunctionResources(HashableModel): - vcpus: int = 1 - memory: int = 128 - seconds: int = 1 - - -class FunctionRuntime(HashableModel): - ref: str - latest_amend: bool = True - comment: str - - -class FunctionContent(HashableModel): - code: CodeContent - data: Optional[DataContent] - on: FunctionTriggers - environment: FunctionEnvironment - resources: FunctionResources - runtime: FunctionRuntime - - -class FunctionMessage(HashableModel): - type: str - address: str - content: FunctionContent - time: float diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index d72729737..8647d916d 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -2,8 +2,8 @@ import logging from typing import Dict, Optional +from aleph_message.models import ProgramContent from vm_supervisor.conf import settings -from vm_supervisor.models import FunctionMessage from vm_supervisor.vm.firecracker_microvm import ( AlephFirecrackerVM, AlephFirecrackerResources, @@ -31,22 +31,22 @@ class VmPool: """ counter: int # Used to provide distinct ids to network interfaces - started_vms_cache: Dict[FunctionMessage, StartedVM] + started_vms_cache: Dict[ProgramContent, StartedVM] def __init__(self): self.counter = settings.START_ID_INDEX self.started_vms_cache = {} - async def create_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: + async def create_a_vm(self, message_content: ProgramContent) -> AlephFirecrackerVM: """Create a new Aleph Firecracker VM from an Aleph function message.""" - vm_resources = AlephFirecrackerResources(message) + vm_resources = AlephFirecrackerResources(message_content) await vm_resources.download_all() self.counter += 1 vm = AlephFirecrackerVM( vm_id=self.counter, resources=vm_resources, - enable_networking=message.content.environment.internet, - hardware_resources=message.content.resources, + enable_networking=message_content.environment.internet, + hardware_resources=message_content.resources, ) await vm.setup() await vm.start() @@ -54,7 +54,7 @@ async def create_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: await vm.start_guest_api() return vm - async def get_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: + async def get_a_vm(self, message: ProgramContent) -> AlephFirecrackerVM: """Provision a VM in the pool, then return the first VM from the pool.""" try: started_vm = self.started_vms_cache.pop(message) @@ -64,7 +64,7 @@ async def get_a_vm(self, message: FunctionMessage) -> AlephFirecrackerVM: return await self.create_a_vm(message) def keep_in_cache( - self, vm: AlephFirecrackerVM, message: FunctionMessage, timeout: float = 1.0 + self, vm: AlephFirecrackerVM, message: ProgramContent, timeout: float = 1.0 ) -> None: """Keep a VM running for `timeout` seconds in case another query comes by.""" @@ -79,7 +79,7 @@ def keep_in_cache( started_vm.timeout_task = loop.create_task(self.expire(vm, message, timeout)) async def expire( - self, vm: AlephFirecrackerVM, message: FunctionMessage, timeout: float + self, vm: AlephFirecrackerVM, message: ProgramContent, timeout: float ): """Coroutine that will stop the VM after 'timeout' seconds.""" await asyncio.sleep(timeout) diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index 3cd0dfdc4..afee7be13 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -12,8 +12,9 @@ import aiohttp +from aleph_message.models import ProgramMessage from .conf import settings -from .models import FunctionMessage, FilePath +from .models import FilePath logger = logging.getLogger(__name__) @@ -41,7 +42,7 @@ async def download_file(url: str, local_path: FilePath) -> None: raise -async def get_message(ref: str) -> FunctionMessage: +async def get_message(ref: str) -> ProgramMessage: if settings.FAKE_DATA: cache_path = os.path.abspath( join(__file__, "../../examples/message_from_aleph.json") @@ -53,9 +54,7 @@ async def get_message(ref: str) -> FunctionMessage: with open(cache_path, "r") as cache_file: msg = json.load(cache_file) - # TODO: Define VM Function type instead of wrapping in 'content' key - msg_content = msg["content"] - return FunctionMessage(**msg_content) + return ProgramMessage(**msg) async def get_code_path(ref: str) -> FilePath: diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 5477bb3aa..c3226cd01 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -13,8 +13,9 @@ from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable, HTTPBadRequest from msgpack import UnpackValueError +from aleph_message.models import ProgramMessage, ProgramContent from .conf import settings -from .models import FilePath, FunctionMessage +from .models import FilePath from .pool import VmPool from .storage import get_message from .vm.firecracker_microvm import ResourceDownloadError @@ -28,7 +29,7 @@ async def index(request: web.Request): return web.Response(text="Server: Aleph VM Supervisor") -async def try_get_message(ref: str) -> FunctionMessage: +async def try_get_message(ref: str) -> ProgramMessage: # Get the message or raise an aiohttp HTTP error try: return await get_message(ref) @@ -64,10 +65,11 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res Execute the code corresponding to the 'code id' in the path. """ - message = await try_get_message(message_ref) + message: ProgramMessage = await try_get_message(message_ref) + message_content: ProgramContent = message.content try: - vm = await pool.get_a_vm(message) + vm = await pool.get_a_vm(message_content) except ResourceDownloadError as error: logger.exception(error) raise HTTPBadRequest(reason="Code, runtime or data not available") @@ -82,9 +84,9 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res try: result_raw: bytes = await vm.run_code( code=code, - entrypoint=message.content.code.entrypoint, + entrypoint=message_content.code.entrypoint, input_data=input_data, - encoding=message.content.code.encoding, + encoding=message_content.code.encoding, scope=scope, ) @@ -120,7 +122,7 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res return web.Response(status=502, reason="Invalid response from VM") finally: if settings.REUSE_TIMEOUT > 0: - pool.keep_in_cache(vm, message, timeout=settings.REUSE_TIMEOUT) + pool.keep_in_cache(vm, message_content, timeout=settings.REUSE_TIMEOUT) else: await vm.teardown() diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 45a07e9da..81d85a786 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -10,10 +10,12 @@ import msgpack from aiohttp import ClientResponseError +from aleph_message.models import ProgramContent +from aleph_message.models.program import MachineResources from firecracker.microvm import MicroVM, setfacl from guest_api.__main__ import run_guest_api from ..conf import settings -from ..models import FunctionMessage, FilePath, FunctionResources +from ..models import FilePath from ..storage import get_code_path, get_runtime_path, get_data_path logger = logging.getLogger(__name__) @@ -56,15 +58,15 @@ def as_msgpack(self) -> bytes: class AlephFirecrackerResources: - message: FunctionMessage + message_content: ProgramContent kernel_image_path: FilePath code_path: FilePath rootfs_path: FilePath data_path: Optional[FilePath] - def __init__(self, message: FunctionMessage): - self.message = message + def __init__(self, message_content: ProgramContent): + self.message_content = message_content async def download_kernel(self): # Assumes kernel is already present on the host @@ -72,7 +74,7 @@ async def download_kernel(self): assert isfile(self.kernel_image_path) async def download_code(self): - code_ref: str = self.message.content.code.ref + code_ref: str = self.message_content.code.ref try: self.code_path = await get_code_path(code_ref) except ClientResponseError as error: @@ -80,7 +82,7 @@ async def download_code(self): assert isfile(self.code_path) async def download_runtime(self): - runtime_ref: str = self.message.content.runtime.ref + runtime_ref: str = self.message_content.runtime.ref try: self.rootfs_path = await get_runtime_path(runtime_ref) except ClientResponseError as error: @@ -88,8 +90,8 @@ async def download_runtime(self): assert isfile(self.rootfs_path) async def download_data(self): - if self.message.content.data: - data_ref: str = self.message.content.data.ref + if self.message_content.data: + data_ref: str = self.message_content.data.ref try: self.data_path = await get_data_path(data_ref) except ClientResponseError as error: @@ -112,7 +114,7 @@ class AlephFirecrackerVM: resources: AlephFirecrackerResources enable_console: bool enable_networking: bool - hardware_resources: FunctionResources + hardware_resources: MachineResources fvm: MicroVM guest_api_process: Process @@ -122,7 +124,7 @@ def __init__( resources: AlephFirecrackerResources, enable_networking: bool = False, enable_console: Optional[bool] = None, - hardware_resources: FunctionResources = FunctionResources() + hardware_resources: MachineResources = MachineResources() ): self.vm_id = vm_id self.resources = resources From a078276a27d63932a7a81a02f60f03179bf6c242 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 26 May 2021 17:26:02 +0200 Subject: [PATCH 043/990] Fix: Code not in IPFS could not be downloaded --- vm_connector/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vm_connector/main.py b/vm_connector/main.py index 7743d817d..b4879ace9 100644 --- a/vm_connector/main.py +++ b/vm_connector/main.py @@ -98,7 +98,10 @@ async def download_code( return Response(status_code=404, content="Hash not found") data_hash = msg["content"]["item_hash"] - url = f"{settings.IPFS_SERVER}/{data_hash}" + if msg["content"]["item_type"] == "ipfs": + url = f"{settings.IPFS_SERVER}/{data_hash}" + else: + url = f"{settings.ALEPH_SERVER}/api/v0/storage/raw/{data_hash}" return StreamingResponse(stream_url_chunks(url), media_type="application/zip") From 7ddc101d077085f4b190aaffc0c835467ffadd61 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 26 May 2021 17:26:33 +0200 Subject: [PATCH 044/990] Fix: Compileall tool does not accept -o option anymore --- runtimes/aleph-alpine-3.13-python/create_disk_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index 076e02c64..691a053ab 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -29,7 +29,7 @@ apk add git pkgconf gcc py3-wheel python3-dev musl-dev py3-cffi libffi-dev autoc pip install git+https://github.com/aleph-im/aleph-client coincurve==15.0.0 # Compile all Python bytecode -python3 -m compileall -o 1 -o 2 -f /usr/lib/python3.8/site-packages +python3 -m compileall -f /usr/lib/python3.8/site-packages echo -e "toor\ntoor" | passwd root From 23cd5bcc42de8f8b3a94cc6959cbccb9daae5243 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 26 May 2021 17:27:21 +0200 Subject: [PATCH 045/990] Fix: Run benchmarks with a real message ref --- vm_supervisor/__main__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index c2619ad4b..2a1ff3229 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -98,12 +98,12 @@ async def benchmark(runs: int): """Measure performance by immediately running the supervisor with fake requests. """ - ref = "vmid" + ref = "7dbf00da1c2b88ab32c0ba98504b499d13dceb62abdb928022f8420fc5d692a4" class FakeRequest: pass fake_request = FakeRequest() - fake_request.match_info = {"ref": ref, "suffix": "/path"} + fake_request.match_info = {"ref": ref, "suffix": "/"} fake_request.method = "GET" fake_request.query_string = "" fake_request.headers = [] @@ -115,7 +115,9 @@ class FakeRequest: pass for run in range(runs): t0 = time.time() - response: Response = await supervisor.run_code(message_ref=ref, request=fake_request) + response: Response = await supervisor.run_code(message_ref=ref, + path="/", + request=fake_request) assert response.status == 200 bench.append(time.time() - t0) From 243e0fcce1e0172f6d95f80093e060c06b75afd9 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 26 May 2021 17:27:55 +0200 Subject: [PATCH 046/990] Fix: Init was crashing when no data configured --- runtimes/aleph-alpine-3.13-python/init1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 92d1c1d13..1a1c1d409 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -138,7 +138,7 @@ async def send(dico): logger.debug("Getting output data") output_data: bytes - if os.listdir('/data'): + if os.path.isdir('/data') and os.listdir('/data'): make_archive("/opt/output", 'zip', "/data") with open("/opt/output.zip", "rb") as output_zipfile: output_data = output_zipfile.read() From 93f2867371eec535a7c7252760bf04e595c0eff8 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 26 May 2021 17:33:37 +0200 Subject: [PATCH 047/990] Fix: Use a recent real message for fake data --- examples/message_from_aleph.json | 105 +++++++++++++------------------ 1 file changed, 45 insertions(+), 60 deletions(-) diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index 097dc0bd2..aef5e583a 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -1,64 +1,49 @@ { - "_id": { - "$oid": "6080402d7f44efefd611dc1e" + "_id": { + "$oid": "60ae674509e67ee2839b67cb" + }, + "chain": "ETH", + "item_hash": "2197b6d13fbeecee029807b2de5d3576e71364b94e9379f500ac26eb0d529ae9", + "sender": "0x0bE24CB9568dA8ec4d33c8E2aA25Fb841550e607", + "type": "PROGRAM", + "channel": "TEST", + "confirmed": false, + "content": { + "address": "0x0bE24CB9568dA8ec4d33c8E2aA25Fb841550e607", + "time": 1622042437.3603787, + "type": "vm-function", + "allow_amend": false, + "code": { + "encoding": "zip", + "entrypoint": "example_fastapi_2:app", + "ref": "78eda627a1a51cb783197cda49c24b66864c4f18843adfec6d9675a5337eb48a", + "use_latest": false }, - "chain": "ETH", - "item_hash": "91c83eff3ba23d6b501a2aa3c4364ec235eb8283b6fa8ac20d235642a48791b8", - "sender": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", - "type": "POST", - "channel": "VM", - "confirmed": true, - "content": { - "type": "vm-function", - "address": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", - "content": { - "code": { - "encoding": "zip", - "entrypoint": "example_fastapi_2:app", - "ref": "7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003", - "latest_amend": true - }, - "on": { - "http": true - }, - "environment": { - "reproducible": true, - "internet": true, - "aleph_api": false - }, - "resources": { - "vcpus": 1, - "memory": 128, - "seconds": 1 - }, - "runtime": { - "ref": "5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51", - "latest_amend": true, - "comment": "Aleph Alpine Linux with Python 3.8" - }, - "data": { - "encoding": "zip", - "mount": "/mnt", - "ref": "7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003", - "latest_amend": true - }, - "export": { - "encoding": "tar.gzip", - "mount": "/mnt" - } - }, - "time": 1619017773.8950517 + "data": null, + "export": null, + "on": { + "http": true }, - "item_content": "{\"type\":\"vm-function\",\"address\":\"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\",\"content\":{\"code\":{\"encoding\":\"zip\",\"entrypoint\":\"example_fastapi_2:app\",\"ref\":\"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\",\"latest_amend\":true},\"on\":{\"http\":true},\"environment\":{\"reproducible\":true,\"internet\":false,\"aleph_api\":false},\"resources\":{\"vcpus\":1,\"memory\":128,\"seconds\":1},\"runtime\":{\"ref\":\"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\",\"latest_amend\":true,\"comment\":\"Aleph Alpine Linux with Python 3.8\"},\"data\":{\"encoding\":\"tar.gzip\",\"mount\":\"/mnt\",\"ref\":\"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\",\"latest_amend\":true},\"export\":{\"encoding\":\"tar.gzip\",\"mount\":\"/mnt\"}},\"time\":1619017773.8950517}", - "item_type": "inline", - "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b", - "size": 749, - "time": 1619017773.8950577, - "confirmations": [ - { - "chain": "ETH", - "height": 12284734, - "hash": "0x67f2f3cde5e94e70615c92629c70d22dc959a118f46e9411b29659c2fce87cdc" - } - ] + "environment": { + "reproducible": false, + "internet": true, + "aleph_api": true + }, + "resources": { + "vcpus": 1, + "memory": 128, + "seconds": 30 + }, + "runtime": { + "ref": "c05879dd0229e686eeaaf3e1a55cf7d1503a3d9ae11854a2cf6e5a3913e1a080", + "use_latest": true, + "comment": "Aleph Alpine Linux with Python 3.8" + }, + "replaces": null + }, + "item_content": "{\"address\":\"0x0bE24CB9568dA8ec4d33c8E2aA25Fb841550e607\",\"time\":1622042437.3603787,\"type\":\"vm-function\",\"allow_amend\":false,\"code\":{\"encoding\":\"zip\",\"entrypoint\":\"example_fastapi_2:app\",\"ref\":\"78eda627a1a51cb783197cda49c24b66864c4f18843adfec6d9675a5337eb48a\",\"use_latest\":false},\"data\":null,\"export\":null,\"on\":{\"http\":true},\"environment\":{\"reproducible\":false,\"internet\":true,\"aleph_api\":true},\"resources\":{\"vcpus\":1,\"memory\":128,\"seconds\":30},\"runtime\":{\"ref\":\"c05879dd0229e686eeaaf3e1a55cf7d1503a3d9ae11854a2cf6e5a3913e1a080\",\"use_latest\":true,\"comment\":\"Aleph Alpine Linux with Python 3.8\"},\"replaces\":null}", + "item_type": "inline", + "signature": "0x6d26aed5d9968d32e5536e058f558d3c5ef9524686a17bfe0e944517276cd308578f746a403a911c7ef07bd60833bb5bfbc42e48c5f3a800a58971da81ab15d61b", + "size": 609, + "time": 1622042437.3610504 } From 3c79edae59c9443ef57a48e0534c249a2724cc33 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 26 May 2021 17:33:55 +0200 Subject: [PATCH 048/990] Cleanup: Remove unused import --- firecracker/microvm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/firecracker/microvm.py b/firecracker/microvm.py index ea1170037..fe13e5cd7 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -4,7 +4,6 @@ import os.path from asyncio import Task from enum import Enum -from functools import lru_cache from os import getuid from pathlib import Path from pwd import getpwnam From 98ad5e3ee1659c028644e4764b3dacbf6cdf51fc Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 27 May 2021 10:28:01 +0200 Subject: [PATCH 049/990] Fix: Localhost was not working in VMs --- runtimes/aleph-alpine-3.13-python/init1.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 1a1c1d409..0c5e8ebe9 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -74,6 +74,9 @@ def setup_network(ip: Optional[str], route: Optional[str], dns_servers: List[str return logger.debug("Setting up networking") + system("ip addr add 127.0.0.1/8 dev lo brd + scope host") + system("ip addr add ::1/128 dev lo") + system("ip link set lo up") system(f"ip addr add {ip}/24 dev eth0") system("ip link set eth0 up") From 439f242fc907f7309d26d50dcdf4dba7f186a61d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 27 May 2021 10:58:25 +0200 Subject: [PATCH 050/990] Fix: Upgrade benchmark runtime hash --- vm_supervisor/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 2a1ff3229..a34da948a 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -98,7 +98,7 @@ async def benchmark(runs: int): """Measure performance by immediately running the supervisor with fake requests. """ - ref = "7dbf00da1c2b88ab32c0ba98504b499d13dceb62abdb928022f8420fc5d692a4" + ref = "fe488a08a7bed020515f069ce9a52847092af468beca79c66c8c0108bdab98a1" class FakeRequest: pass From 5f81d77b9eb1489f65206222515f19393a6002df Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 27 May 2021 11:08:58 +0200 Subject: [PATCH 051/990] Fix: Recent version of pydantic is required in doc --- vm_supervisor/README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index c29fc4990..d6c02c144 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -84,16 +84,10 @@ cd aleph-vm/ [PyDantic](https://pydantic-docs.helpmanual.io/) is used to parse and validate Aleph messages. -Via `apt` if available (on Debian 11+, Ubuntu 20.04+): - -```shell -apt install -y python3-pydantic -``` - -else (on Debian 10): ```shell apt install -y --no-install-recommends --no-install-suggests python3-pip -pip3 install pydantic +pip3 install pydantic[dotenv] +pip3 install aleph-message>=0.1.5 ``` ### 2.f. Create the jailer working directory: From 578903e97842977603b474b7c382fffc2baa75d3 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 27 May 2021 11:17:04 +0200 Subject: [PATCH 052/990] Fix: Use fixes from newer aleph-message --- vm_supervisor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index d6c02c144..a23b5177e 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -87,7 +87,7 @@ is used to parse and validate Aleph messages. ```shell apt install -y --no-install-recommends --no-install-suggests python3-pip pip3 install pydantic[dotenv] -pip3 install aleph-message>=0.1.5 +pip3 install aleph-message>=0.1.6 ``` ### 2.f. Create the jailer working directory: From a9334bee47b3902823bbb66a17114260dc2f577d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 27 May 2021 12:42:08 +0200 Subject: [PATCH 053/990] Fix: Apps in VMS were restarted for every call Performance did suffer from this. Solution: Keep the same app running between calls. Copy the code and data during VM setup instead of doing it at every call. --- runtimes/aleph-alpine-3.13-python/init1.py | 57 ++++++++++++---------- vm_supervisor/supervisor.py | 21 +------- vm_supervisor/vm/firecracker_microvm.py | 42 ++++++++++------ 3 files changed, 59 insertions(+), 61 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 0c5e8ebe9..aaa41b267 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -20,12 +20,15 @@ from io import StringIO from os import system from shutil import make_archive -from typing import Optional, Dict, Any, Tuple, Iterator, List +from typing import Optional, Dict, Any, Tuple, Iterator, List, NewType import msgpack logger.debug("Imports finished") +ASGIApplication = NewType('AsgiApplication', Any) + + class Encoding: plain = "plain" zip = "zip" @@ -36,14 +39,14 @@ class ConfigurationPayload: ip: Optional[str] route: Optional[str] dns_servers: List[str] + code: bytes + encoding: Encoding + entrypoint: str + input_data: bytes @dataclass class RunCodePayload: - code: bytes - input_data: Optional[bytes] - entrypoint: str - encoding: str scope: Dict @@ -91,36 +94,41 @@ def setup_network(ip: Optional[str], route: Optional[str], dns_servers: List[str resolvconf_fd.write(f"nameserver {server}\n".encode()) -async def run_python_code_http(code: bytes, input_data: Optional[bytes], - entrypoint: str, encoding: str, scope: dict - ) -> Tuple[Dict, Dict, str, Optional[bytes]]: +def setup_input_data(input_data: bytes): + logger.debug("Extracting data") + if input_data: + # Unzip in /data + if not os.path.exists("/opt/input.zip"): + open("/opt/input.zip", "wb").write(input_data) + os.makedirs("/data", exist_ok=True) + os.system("unzip /opt/input.zip -d /data") + + +def setup_code(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplication: logger.debug("Extracting code") if encoding == Encoding.zip: # Unzip in /opt and import the entrypoint from there if not os.path.exists("/opt/archive.zip"): open("/opt/archive.zip", "wb").write(code) - logger.debug("Run unzipp") + logger.debug("Run unzip") os.system("unzip /opt/archive.zip -d /opt") sys.path.append("/opt") module_name, app_name = entrypoint.split(":", 1) logger.debug("import module") module = __import__(module_name) - app = getattr(module, app_name) + app: ASGIApplication = getattr(module, app_name) elif encoding == Encoding.plain: # Execute the code and extract the entrypoint locals: Dict[str, Any] = {} exec(code, globals(), locals) - app = locals[entrypoint] + app: ASGIApplication = locals[entrypoint] else: raise ValueError(f"Unknown encoding '{encoding}'") + return app - logger.debug("Extracting data") - if input_data: - # Unzip in /data - if not os.path.exists("/opt/input.zip"): - open("/opt/input.zip", "wb").write(input_data) - os.makedirs("/data", exist_ok=True) - os.system("unzip /opt/input.zip -d /data") + +async def run_python_code_http(application: ASGIApplication, scope: dict + ) -> Tuple[Dict, Dict, str, Optional[bytes]]: logger.debug("Running code") with StringIO() as buf, redirect_stdout(buf): @@ -134,7 +142,7 @@ async def send(dico): await send_queue.put(dico) # TODO: Better error handling - await app(scope, receive, send) + await application(scope, receive, send) headers: Dict = await send_queue.get() body: Dict = await send_queue.get() output = buf.getvalue() @@ -152,7 +160,7 @@ async def send(dico): return headers, body, output, output_data -def process_instruction(instruction: bytes) -> Iterator[bytes]: +def process_instruction(instruction: bytes, application: ASGIApplication) -> Iterator[bytes]: if instruction == b"halt": system("sync") yield b"STOP\n" @@ -179,10 +187,7 @@ def process_instruction(instruction: bytes) -> Iterator[bytes]: output_data: Optional[bytes] headers, body, output, output_data = asyncio.get_event_loop().run_until_complete( - run_python_code_http( - payload.code, input_data=payload.input_data, - entrypoint=payload.entrypoint, encoding=payload.encoding, scope=payload.scope - ) + run_python_code_http(application=application, scope=payload.scope) ) result = { "headers": headers, @@ -206,6 +211,8 @@ def main(): payload = ConfigurationPayload(**msg_) setup_network(payload.ip, payload.route, payload.dns_servers) + setup_input_data(payload.input_data) + app: ASGIApplication = setup_code(payload.code, payload.encoding, payload.entrypoint) while True: client, addr = s.accept() @@ -217,7 +224,7 @@ def main(): data_to_print = f"{data[:500]}..." if len(data) > 500 else data logger.debug(f"<<<\n\n{data_to_print}\n\n>>>") - for result in process_instruction(instruction=data): + for result in process_instruction(instruction=data, application=app): client.send(result) logger.debug("...DONE") diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index c3226cd01..38a45ac1d 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -15,7 +15,6 @@ from aleph_message.models import ProgramMessage, ProgramContent from .conf import settings -from .models import FilePath from .pool import VmPool from .storage import get_message from .vm.firecracker_microvm import ResourceDownloadError @@ -52,14 +51,6 @@ def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: } -def load_file_content(path: FilePath) -> bytes: - if path: - with open(path, "rb") as fd: - return fd.read() - else: - return b"" - - async def run_code(message_ref: str, path: str, request: web.Request) -> web.Response: """ Execute the code corresponding to the 'code id' in the path. @@ -78,18 +69,8 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res scope: Dict = build_asgi_scope(path, request) - code: bytes = load_file_content(vm.resources.code_path) - input_data: bytes = load_file_content(vm.resources.data_path) - try: - result_raw: bytes = await vm.run_code( - code=code, - entrypoint=message_content.code.entrypoint, - input_data=input_data, - encoding=message_content.code.encoding, - scope=scope, - ) - + result_raw: bytes = await vm.run_code(scope=scope) except UnpackValueError as error: logger.exception(error) return web.Response(status=502, reason="Invalid response from VM") diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 81d85a786..bb266521d 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -12,7 +12,7 @@ from aleph_message.models import ProgramContent from aleph_message.models.program import MachineResources -from firecracker.microvm import MicroVM, setfacl +from firecracker.microvm import MicroVM, setfacl, Encoding from guest_api.__main__ import run_guest_api from ..conf import settings from ..models import FilePath @@ -22,6 +22,14 @@ set_start_method("spawn") +def load_file_content(path: FilePath) -> bytes: + if path: + with open(path, "rb") as fd: + return fd.read() + else: + return b"" + + class ResourceDownloadError(ClientResponseError): """An error occurred while downloading a VM resource file""" @@ -39,6 +47,10 @@ class ConfigurationPayload: ip: Optional[str] route: Optional[str] dns_servers: List[str] + code: bytes + encoding: str + entrypoint: str + input_data: bytes def as_msgpack(self) -> bytes: return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True) @@ -46,10 +58,6 @@ def as_msgpack(self) -> bytes: @dataclass class RunCodePayload: - code: bytes - input_data: bytes - entrypoint: str - encoding: str scope: Dict def as_msgpack(self) -> bytes: @@ -62,11 +70,15 @@ class AlephFirecrackerResources: kernel_image_path: FilePath code_path: FilePath + code_encoding: Encoding + code_entrypoint: str rootfs_path: FilePath data_path: Optional[FilePath] def __init__(self, message_content: ProgramContent): self.message_content = message_content + self.code_encoding = message_content.code.encoding + self.code_entrypoint = message_content.code.entrypoint async def download_kernel(self): # Assumes kernel is already present on the host @@ -177,11 +189,19 @@ async def start(self): async def configure(self): """Configure the VM by sending configuration info to it's init""" + + code: bytes = load_file_content(self.resources.code_path) + input_data: bytes = load_file_content(self.resources.data_path) + reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path) payload = ConfigurationPayload( ip=self.fvm.guest_ip if self.enable_networking else None, route=self.fvm.host_ip if self.enable_console else None, dns_servers=settings.DNS_NAMESERVERS, + code=code, + encoding=self.resources.code_encoding, + entrypoint=self.resources.code_entrypoint, + input_data=input_data, ) writer.write(b"CONNECT 52\n" + payload.as_msgpack()) await writer.drain() @@ -205,23 +225,13 @@ async def teardown(self): async def run_code( self, - code: bytes, - entrypoint: str, - input_data: bytes = b"", - encoding: str = "plain", scope: dict = None, ): logger.debug("running code") scope = scope or {} reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path) - payload = RunCodePayload( - code=code, - input_data=input_data, - entrypoint=entrypoint, - encoding=encoding, - scope=scope, - ) + payload = RunCodePayload(scope=scope) writer.write(b"CONNECT 52\n" + payload.as_msgpack()) await writer.drain() From ad301a4a2895fdbe6e0d9d59d8229f2fa5f53d35 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 27 May 2021 15:13:38 +0200 Subject: [PATCH 054/990] Change: Run VMs on /vm/{hash} instead of /vm/function/{hash} --- examples/README.md | 2 +- vm_supervisor/supervisor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/README.md b/examples/README.md index dbdda3217..b469d3d5c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -80,4 +80,4 @@ Update the `entrypoint` field according to your app if necessary. Open the HTTP interface of a node running the VM Supervisor: -http://ip-of-supervisor:8080/vm/function/{message_hash}/ +http://ip-of-supervisor:8080/vm/{message_hash}/ diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 38a45ac1d..74c8d80c1 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -136,7 +136,7 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: app = web.Application() -app.add_routes([web.route("*", "/vm/function/{ref}{suffix:.*}", run_code_from_path)]) +app.add_routes([web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path)]) app.add_routes([web.route("*", "/{suffix:.*}", run_code_from_hostname)]) From 4aa930ccdff8bd1f016ee6fd25fa54b7e3d8ad7c Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 27 May 2021 16:19:40 +0200 Subject: [PATCH 055/990] Fix: License was missing --- LICENSE.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 000000000..74446c70b --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Hugo Herter + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 2250ada2943efed417e840dc52cf5f0e0d7653f3 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 28 May 2021 10:32:05 +0200 Subject: [PATCH 056/990] Doc: Document configuration via .env file --- vm_supervisor/README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index a23b5177e..e5298604b 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -124,15 +124,26 @@ http://localhost:8080/ ## 4. Configuration -The VM Supervisor can be configured using command-line arguments: +The VM Supervisor can be configured using command-line arguments or using environment variables. + +List the available command-lien arguments using: ```shell python3 -m vm_supervisor --help ``` -and using environment variables, which can be found using: + +List available using environment variables using: ```shell python3 -m vm_supervisor --print-config --do-not-run ``` +Configuration environment variables can be stored in a file named `.env` in the local directory. + +Example content for `.env`: +```shell +ALEPH_VM_DNS_RESOLUTION=resolvectl +ALEPH_VM_NETWORK_INTERFACE=enp7s0 +``` + ## 5. Reverse-proxy A reverse-proxy is required for production use. It allows: From 55359f824064514084dd2589ff00580b5052ddc3 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 28 May 2021 12:00:10 +0200 Subject: [PATCH 057/990] Fix: Hostname labels must be < 64 chars Message ids are usually encoded in hexadecimal (base16) across Aleph.im, with a length of 64 characters. These cannot be used in hostnames since the each label from the hostname (between dots) must be from 1 to 63 characters long. Since hostnames are case insensitive, base64 cannot be used to encode the hashes. Solution: Accept base32 encoding of the message ids in the hostname. --- vm_supervisor/supervisor.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 74c8d80c1..9a7d06524 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -5,7 +5,9 @@ At it's core, it is currently an asynchronous HTTP server using aiohttp, but this may evolve in the future. """ +import binascii import logging +from base64 import b32decode, b16encode from typing import Awaitable, Dict, Any import msgpack @@ -121,16 +123,33 @@ def run_code_from_path(request: web.Request) -> Awaitable[web.Response]: return run_code(message_ref, path, request) +def b32_to_b16(hash: str) -> bytes: + """Convert base32 encoded bytes to base16 encoded bytes.""" + # Add padding + hash_b32: str = hash.upper() + "=" * (56 - len(hash)) + hash_bytes: bytes = b32decode(hash_b32.encode()) + return b16encode(hash_bytes).lower() + + async def run_code_from_hostname(request: web.Request) -> web.Response: """Allow running an Aleph VM function from a hostname The first component of the hostname is used as identifier of the message defining the Aleph VM function. + + Since hostname labels are limited to 63 characters and hex(sha256(...)) has a length of 64, + we expect the hash to be encoded in base32 instead of hexadecimal. Padding is added + automatically. """ path = request.match_info["suffix"] path = path if path.startswith("/") else f"/{path}" - message_ref = request.host.split(".")[0] + message_ref_base32 = request.host.split(".")[0] + try: + message_ref = b32_to_b16(message_ref_base32).decode() + except binascii.Error: + raise HTTPNotFound(reason="Invalid message reference") + return await run_code(message_ref, path, request) From 140e83f6d7f08d0c970fb44db4bb4139ec0ae523 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 28 May 2021 13:08:54 +0200 Subject: [PATCH 058/990] Fix: Handle exceptions during program initialisation --- runtimes/aleph-alpine-3.13-python/init1.py | 12 +++++++++++- vm_supervisor/supervisor.py | 8 ++++++-- vm_supervisor/vm/firecracker_microvm.py | 18 ++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index aaa41b267..f3b903e1d 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -212,7 +212,17 @@ def main(): payload = ConfigurationPayload(**msg_) setup_network(payload.ip, payload.route, payload.dns_servers) setup_input_data(payload.input_data) - app: ASGIApplication = setup_code(payload.code, payload.encoding, payload.entrypoint) + + try: + app: ASGIApplication = setup_code(payload.code, payload.encoding, payload.entrypoint) + client.send(msgpack.dumps({"success": True})) + except Exception as error: + logger.exception("Program could not be started") + client.send(msgpack.dumps({ + "success": False, + "error": str(error), + "traceback": str(traceback.format_exc()), + })) while True: client, addr = s.accept() diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 9a7d06524..efe852fba 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -12,14 +12,15 @@ import msgpack from aiohttp import web, ClientResponseError, ClientConnectorError -from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable, HTTPBadRequest +from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable, HTTPBadRequest, \ + HTTPInternalServerError from msgpack import UnpackValueError from aleph_message.models import ProgramMessage, ProgramContent from .conf import settings from .pool import VmPool from .storage import get_message -from .vm.firecracker_microvm import ResourceDownloadError +from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError logger = logging.getLogger(__name__) pool = VmPool() @@ -66,6 +67,9 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res except ResourceDownloadError as error: logger.exception(error) raise HTTPBadRequest(reason="Code, runtime or data not available") + except VmSetupError as error: + logger.exception(error) + raise HTTPInternalServerError(reason="Error during program initialisation") logger.debug(f"Using vm={vm.vm_id}") diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index bb266521d..c79b722f2 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -56,6 +56,13 @@ def as_msgpack(self) -> bytes: return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True) +@dataclass +class ConfigurationResponse: + success: bool + error: Optional[str] = None + traceback: Optional[str] = None + + @dataclass class RunCodePayload: scope: Dict @@ -121,6 +128,10 @@ async def download_all(self): ) +class VmSetupError(Exception): + pass + + class AlephFirecrackerVM: vm_id: int resources: AlephFirecrackerResources @@ -206,6 +217,13 @@ async def configure(self): writer.write(b"CONNECT 52\n" + payload.as_msgpack()) await writer.drain() + await reader.readline() # Ignore the acknowledgement from the socket + response_raw = await reader.read(1000_000) + response = ConfigurationResponse( + **msgpack.loads(response_raw, raw=False)) + if response.success is False: + raise VmSetupError(response.error) + async def start_guest_api(self): logger.debug(f"starting guest API for {self.vm_id}") vsock_path = f"{self.fvm.vsock_path}_53" From bc93d5632aa24eb44719332474941e630e1af864 Mon Sep 17 00:00:00 2001 From: Moshe Malawach Date: Fri, 28 May 2021 14:45:12 +0200 Subject: [PATCH 059/990] fix fake data to hardcode message_ref in domain --- vm_supervisor/supervisor.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index efe852fba..d536b6f52 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -149,10 +149,13 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: path = path if path.startswith("/") else f"/{path}" message_ref_base32 = request.host.split(".")[0] - try: - message_ref = b32_to_b16(message_ref_base32).decode() - except binascii.Error: - raise HTTPNotFound(reason="Invalid message reference") + if settings.FAKE_DATA: + message_ref = "test" + else: + try: + message_ref = b32_to_b16(message_ref_base32).decode() + except binascii.Error: + raise HTTPNotFound(reason="Invalid message reference") return await run_code(message_ref, path, request) From bb1461961e5d3164495f481b2b011555179e03da Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 28 May 2021 14:22:35 +0200 Subject: [PATCH 060/990] WIP: Add support to call executables in VM localhost Tested successfully on Python code Untested on binary --- runtimes/aleph-alpine-3.13-python/init1.py | 107 +++++++++++++++--- .../aleph-alpine-3.13-python/update_inits.sh | 16 +++ vm_supervisor/vm/firecracker_microvm.py | 12 ++ 3 files changed, 120 insertions(+), 15 deletions(-) create mode 100755 runtimes/aleph-alpine-3.13-python/update_inits.sh diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index f3b903e1d..1bd14db65 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -12,6 +12,7 @@ import asyncio import os import socket +from enum import Enum import subprocess import sys import traceback @@ -20,8 +21,9 @@ from io import StringIO from os import system from shutil import make_archive -from typing import Optional, Dict, Any, Tuple, Iterator, List, NewType +from typing import Optional, Dict, Any, Tuple, Iterator, List, NewType, Union +import aiohttp import msgpack logger.debug("Imports finished") @@ -29,11 +31,16 @@ ASGIApplication = NewType('AsgiApplication', Any) -class Encoding: +class Encoding(str, Enum): plain = "plain" zip = "zip" +class Interface(str, Enum): + asgi = "asgi" + executable = "executable" + + @dataclass class ConfigurationPayload: ip: Optional[str] @@ -43,6 +50,7 @@ class ConfigurationPayload: encoding: Encoding entrypoint: str input_data: bytes + interface: Interface @dataclass @@ -66,8 +74,10 @@ class RunCodePayload: logger.debug("init1.py is launching") -def setup_network(ip: Optional[str], route: Optional[str], dns_servers: List[str] = []): +def setup_network(ip: Optional[str], route: Optional[str], + dns_servers: Optional[List[str]] = None): """Setup the system with info from the host.""" + dns_servers = dns_servers or [] if not os.path.exists("/sys/class/net/eth0"): logger.info("No network interface eth0") return @@ -104,7 +114,7 @@ def setup_input_data(input_data: bytes): os.system("unzip /opt/input.zip -d /data") -def setup_code(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplication: +def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplication: logger.debug("Extracting code") if encoding == Encoding.zip: # Unzip in /opt and import the entrypoint from there @@ -127,8 +137,38 @@ def setup_code(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplicat return app +def setup_code_executable(code: bytes, encoding: Encoding, entrypoint: str) -> subprocess.Popen: + logger.debug("Extracting code") + if encoding == Encoding.zip: + open("/opt/archive.zip", "wb").write(code) + logger.debug("Run unzip") + os.system("unzip /opt/archive.zip -d /opt") + path = f"/opt/{entrypoint}" + os.system(f"chmod +x {path}") + elif encoding == Encoding.plain: + path = f"/opt/executable {entrypoint}" + open(path, "wb").write(code) + os.system(f"chmod +x {path}") + else: + raise ValueError(f"Unknown encoding '{encoding}'. This should never happen.") + + process = subprocess.Popen(path) + return process + + +def setup_code(code: bytes, encoding: Encoding, entrypoint: str, interface: Interface + ) -> Union[ASGIApplication, subprocess.Popen]: + + if interface == Interface.asgi: + return setup_code_asgi(code=code, encoding=encoding, entrypoint=entrypoint) + elif interface == Interface.executable: + return setup_code_executable(code=code, encoding=encoding, entrypoint=entrypoint) + else: + raise ValueError("Invalid interface. This should never happen.") + + async def run_python_code_http(application: ASGIApplication, scope: dict - ) -> Tuple[Dict, Dict, str, Optional[bytes]]: + ) -> Tuple[Dict, Dict, str, Optional[bytes]]: logger.debug("Running code") with StringIO() as buf, redirect_stdout(buf): @@ -160,7 +200,33 @@ async def send(dico): return headers, body, output, output_data -def process_instruction(instruction: bytes, application: ASGIApplication) -> Iterator[bytes]: +async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[bytes]]: + logger.debug("Calling localhost") + + async with aiohttp.ClientSession(conn_timeout=2) as session: + async with session.request( + scope["method"], + url="http://localhost:8080{}".format(scope["path"]), + params=scope["query_string"], + headers=[(a.decode('utf-8'), b.decode('utf-8')) + for a, b in scope['headers']] + ) as resp: + headers = { + 'headers': [(a.encode('utf-8'), b.encode('utf-8')) + for a, b in resp.headers.items()], + 'status': resp.status + } + body = { + 'body': await resp.content.read() + } + + output = "" + output_data = None + logger.debug("Returning result") + return headers, body, output, output_data + + +def process_instruction(instruction: bytes, interface: Interface, application) -> Iterator[bytes]: if instruction == b"halt": system("sync") yield b"STOP\n" @@ -186,9 +252,17 @@ def process_instruction(instruction: bytes, application: ASGIApplication) -> Ite body: Dict output_data: Optional[bytes] - headers, body, output, output_data = asyncio.get_event_loop().run_until_complete( - run_python_code_http(application=application, scope=payload.scope) - ) + if interface == Interface.asgi: + headers, body, output, output_data = asyncio.get_event_loop().run_until_complete( + run_python_code_http(application=application, scope=payload.scope) + ) + elif interface == Interface.executable: + headers, body, output, output_data = asyncio.get_event_loop().run_until_complete( + run_executable_http(scope=payload.scope) + ) + else: + raise ValueError("Unknown interface. This should never happen") + result = { "headers": headers, "body": body, @@ -209,20 +283,22 @@ def main(): data = client.recv(1000_1000) msg_ = msgpack.loads(data, raw=False) - payload = ConfigurationPayload(**msg_) - setup_network(payload.ip, payload.route, payload.dns_servers) - setup_input_data(payload.input_data) + config = ConfigurationPayload(**msg_) + setup_network(config.ip, config.route, config.dns_servers) + setup_input_data(config.input_data) try: - app: ASGIApplication = setup_code(payload.code, payload.encoding, payload.entrypoint) + app: Union[ASGIApplication, subprocess.Popen] = setup_code( + config.code, config.encoding, config.entrypoint, config.interface) client.send(msgpack.dumps({"success": True})) except Exception as error: - logger.exception("Program could not be started") client.send(msgpack.dumps({ "success": False, "error": str(error), "traceback": str(traceback.format_exc()), })) + logger.exception("Program could not be started") + raise while True: client, addr = s.accept() @@ -234,7 +310,8 @@ def main(): data_to_print = f"{data[:500]}..." if len(data) > 500 else data logger.debug(f"<<<\n\n{data_to_print}\n\n>>>") - for result in process_instruction(instruction=data, application=app): + for result in process_instruction(instruction=data, interface=config.interface, + application=app): client.send(result) logger.debug("...DONE") diff --git a/runtimes/aleph-alpine-3.13-python/update_inits.sh b/runtimes/aleph-alpine-3.13-python/update_inits.sh new file mode 100755 index 000000000..0daa9b16a --- /dev/null +++ b/runtimes/aleph-alpine-3.13-python/update_inits.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +umount /mnt/rootfs + +set -euf + +mount ./rootfs.ext4 /mnt/rootfs + +cp ./init0.sh /mnt/rootfs/sbin/init +cp ./init1.py /mnt/rootfs/root/init1.py +chmod +x /mnt/rootfs/sbin/init +chmod +x /mnt/rootfs/root/init1.py + +umount /mnt/rootfs + +echo "OK" diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index c79b722f2..6d8f711d3 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -2,6 +2,7 @@ import dataclasses import logging from dataclasses import dataclass +from enum import Enum from multiprocessing import Process, set_start_method from os import system from os.path import isfile, exists @@ -42,6 +43,12 @@ def __init__(self, error: ClientResponseError): headers=error.headers, ) + +class Interface(str, Enum): + asgi = "asgi" + executable = "executable" + + @dataclass class ConfigurationPayload: ip: Optional[str] @@ -51,6 +58,7 @@ class ConfigurationPayload: encoding: str entrypoint: str input_data: bytes + interface: Interface def as_msgpack(self) -> bytes: return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True) @@ -204,6 +212,9 @@ async def configure(self): code: bytes = load_file_content(self.resources.code_path) input_data: bytes = load_file_content(self.resources.data_path) + interface = Interface.asgi if ":" in self.resources.code_entrypoint \ + else Interface.executable + reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path) payload = ConfigurationPayload( ip=self.fvm.guest_ip if self.enable_networking else None, @@ -213,6 +224,7 @@ async def configure(self): encoding=self.resources.code_encoding, entrypoint=self.resources.code_entrypoint, input_data=input_data, + interface=interface, ) writer.write(b"CONNECT 52\n" + payload.as_msgpack()) await writer.drain() From 32ff04dd6afc4cd26077701985fdb24860e30a4e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 28 May 2021 16:50:57 +0200 Subject: [PATCH 061/990] Enhancement: Make FAKE_DATA example configurable --- vm_supervisor/conf.py | 1 + vm_supervisor/storage.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 100fc9fa0..66c3fbb70 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -84,6 +84,7 @@ class Settings(BaseSettings): DATA_CACHE: FilePath = FilePath(join(CACHE_ROOT, "data")) FAKE_DATA: bool = False + FAKE_DATA_EXAMPLE: str = "example_fastapi_2" def update(self, **kwargs): for key, value in kwargs.items(): diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index afee7be13..10b6c0ebe 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -60,9 +60,9 @@ async def get_message(ref: str) -> ProgramMessage: async def get_code_path(ref: str) -> FilePath: if settings.FAKE_DATA: root_dir = abspath(join(__file__, "../../examples/")) - archive_path = join(root_dir, "example_fastapi_2") + archive_path = join(root_dir, settings.FAKE_DATA_EXAMPLE) make_archive( - archive_path, "zip", root_dir=root_dir, base_dir="example_fastapi_2" + archive_path, "zip", root_dir=root_dir, base_dir=settings.FAKE_DATA_EXAMPLE ) return FilePath(f"{archive_path}.zip") From 13b7542924451adc2d2c0d93497f55dbd402b7b9 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 28 May 2021 16:51:34 +0200 Subject: [PATCH 062/990] Fix: Errors in Supervisor Docker image --- docker/vm_supervisor.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile index c45cf5fd2..4e3e5833a 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor.dockerfile @@ -17,14 +17,14 @@ RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/downl RUN ln /opt/firecracker/firecracker-v* /opt/firecracker/firecracker RUN ln /opt/firecracker/jailer-v* /opt/firecracker/jailer -RUN pip3 install typing-extensions +RUN pip3 install typing-extensions aleph-message pydantic RUN mkdir /srv/jailer ENV PYTHONPATH /mnt # Networking does not work in Docker containers -ENV ALLOW_VM_NETWORKING False +ENV ALEPH_VM_ALLOW_VM_NETWORKING False # Jailer does not work in Docker containers ENV ALEPH_VM_USE_JAILER False # Use fake test data From 375171cc3e1c34fc1206b4ef1371265c94d945e4 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 28 May 2021 16:56:27 +0200 Subject: [PATCH 063/990] Fix: Allow big setup data by specifying it's length --- runtimes/aleph-alpine-3.13-python/init1.py | 21 ++++++++++++++++++++- vm_supervisor/vm/firecracker_microvm.py | 4 +++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 1bd14db65..0dae8721e 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -278,14 +278,33 @@ def process_instruction(instruction: bytes, interface: Interface, application) - }) +def receive_data_length(client) -> int: + """Receive the length of the data to follow.""" + buffer = b"" + for _ in range(9): + byte = client.recv(1) + if byte == b"\n": + break + else: + buffer += byte + return int(buffer) + + def main(): client, addr = s.accept() - data = client.recv(1000_1000) + + logger.debug("Receiving setup...") + length = receive_data_length(client) + data = b"" + while len(data) < length: + data += client.recv(1024*1024) + msg_ = msgpack.loads(data, raw=False) config = ConfigurationPayload(**msg_) setup_network(config.ip, config.route, config.dns_servers) setup_input_data(config.input_data) + logger.debug("Setup finished") try: app: Union[ASGIApplication, subprocess.Popen] = setup_code( diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 6d8f711d3..1d2685faa 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -226,7 +226,9 @@ async def configure(self): input_data=input_data, interface=interface, ) - writer.write(b"CONNECT 52\n" + payload.as_msgpack()) + payload = config.as_msgpack() + length = f"{len(payload)}\n".encode() + writer.write(b"CONNECT 52\n" + length + payload) await writer.drain() await reader.readline() # Ignore the acknowledgement from the socket From 59da9caadaff0208847eea1c35bee2e58af2bd35 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 28 May 2021 16:57:50 +0200 Subject: [PATCH 064/990] Fix: Improve resource cleanup on error raised --- vm_supervisor/pool.py | 15 ++++++++---- vm_supervisor/vm/firecracker_microvm.py | 32 ++++++++++++++----------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index 8647d916d..5ebdc9916 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -48,11 +48,16 @@ async def create_a_vm(self, message_content: ProgramContent) -> AlephFirecracker enable_networking=message_content.environment.internet, hardware_resources=message_content.resources, ) - await vm.setup() - await vm.start() - await vm.configure() - await vm.start_guest_api() - return vm + try: + await vm.setup() + await vm.start() + await vm.configure() + await vm.start_guest_api() + return vm + except Exception: + await vm.teardown() + raise + async def get_a_vm(self, message: ProgramContent) -> AlephFirecrackerVM: """Provision a VM in the pool, then return the first VM from the pool.""" diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 1d2685faa..482f42478 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -176,19 +176,23 @@ async def setup(self): ) fvm.prepare_jailer() await fvm.start() - await fvm.socket_is_ready() - await fvm.set_boot_source( - self.resources.kernel_image_path, - enable_console=self.enable_console, - ) - await fvm.set_rootfs(self.resources.rootfs_path) - await fvm.set_vsock() - await fvm.set_resources(vcpus=self.hardware_resources.vcpus, - memory=self.hardware_resources.memory) - if self.enable_networking: - await fvm.set_network(interface=settings.NETWORK_INTERFACE) - logger.debug("setup done") - self.fvm = fvm + try: + await fvm.socket_is_ready() + await fvm.set_boot_source( + self.resources.kernel_image_path, + enable_console=self.enable_console, + ) + await fvm.set_rootfs(self.resources.rootfs_path) + await fvm.set_vsock() + await fvm.set_resources(vcpus=self.hardware_resources.vcpus, + memory=self.hardware_resources.memory) + if self.enable_networking: + await fvm.set_network(interface=settings.NETWORK_INTERFACE) + logger.debug("setup done") + self.fvm = fvm + except Exception: + await fvm.teardown() + raise async def start(self): logger.debug(f"starting vm {self.vm_id}") @@ -216,7 +220,7 @@ async def configure(self): else Interface.executable reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path) - payload = ConfigurationPayload( + config = ConfigurationPayload( ip=self.fvm.guest_ip if self.enable_networking else None, route=self.fvm.host_ip if self.enable_console else None, dns_servers=settings.DNS_NAMESERVERS, From 4ea093a0d22cc87b3c4ed35291df7f4492411928 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 28 May 2021 16:58:18 +0200 Subject: [PATCH 065/990] Enhancement: Better error when executable is missing --- runtimes/aleph-alpine-3.13-python/init1.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 0dae8721e..b081f1c2f 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -144,6 +144,9 @@ def setup_code_executable(code: bytes, encoding: Encoding, entrypoint: str) -> s logger.debug("Run unzip") os.system("unzip /opt/archive.zip -d /opt") path = f"/opt/{entrypoint}" + if not os.path.isfile(path): + os.system("find /opt") + raise FileNotFoundError(f"No such file: {path}") os.system(f"chmod +x {path}") elif encoding == Encoding.plain: path = f"/opt/executable {entrypoint}" From 1d02148aaea808ab02fd3dff0cba5f8b7cf04012 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 28 May 2021 17:05:56 +0200 Subject: [PATCH 066/990] Fix: Guest API process was not always present when errors happen --- vm_supervisor/vm/firecracker_microvm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 482f42478..26661683a 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -147,7 +147,7 @@ class AlephFirecrackerVM: enable_networking: bool hardware_resources: MachineResources fvm: MicroVM - guest_api_process: Process + guest_api_process: Optional[Process] = None def __init__( self, @@ -253,7 +253,8 @@ async def start_guest_api(self): logger.debug(f"started guest API for {self.vm_id}") async def stop_guest_api(self): - self.guest_api_process.terminate() + if self.guest_api_process: + self.guest_api_process.terminate() async def teardown(self): await self.fvm.teardown() From 0432115448b41000ec4aaba1cb6ee88d9428d339 Mon Sep 17 00:00:00 2001 From: Moshe Malawach Date: Thu, 3 Jun 2021 20:48:44 +0200 Subject: [PATCH 067/990] handle retry --- runtimes/aleph-alpine-3.13-python/init1.py | 44 +++++++++++++++------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index b081f1c2f..f3441d7cb 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -203,25 +203,41 @@ async def send(dico): return headers, body, output, output_data -async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[bytes]]: - logger.debug("Calling localhost") - - async with aiohttp.ClientSession(conn_timeout=2) as session: - async with session.request( +async def make_request(session, scope): + async with session.request( scope["method"], url="http://localhost:8080{}".format(scope["path"]), params=scope["query_string"], headers=[(a.decode('utf-8'), b.decode('utf-8')) for a, b in scope['headers']] - ) as resp: - headers = { - 'headers': [(a.encode('utf-8'), b.encode('utf-8')) - for a, b in resp.headers.items()], - 'status': resp.status - } - body = { - 'body': await resp.content.read() - } + ) as resp: + headers = { + 'headers': [(a.encode('utf-8'), b.encode('utf-8')) + for a, b in resp.headers.items()], + 'status': resp.status + } + body = { + 'body': await resp.content.read() + } + return headers, body + + +async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[bytes]]: + logger.debug("Calling localhost") + + tries = 0 + headers = None + body = None + + async with aiohttp.ClientSession(conn_timeout=2) as session: + while not body: + try: + tries += 1 + headers, body = await make_request(session, scope) + except aiohttp.ClientConnectorError: + if tries > 20: + raise + await asyncio.sleep(.05) output = "" output_data = None From 4b11268919b27646488e8bd2c728da2d2cb15005 Mon Sep 17 00:00:00 2001 From: Moshe Malawach Date: Thu, 3 Jun 2021 20:49:31 +0200 Subject: [PATCH 068/990] set hash and item_content based on content for fake_data --- vm_supervisor/storage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index 10b6c0ebe..da7acf744 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -5,6 +5,7 @@ In the future, it should connect to an Aleph node and retrieve the code from there. """ import json +import hashlib import logging import os from os.path import isfile, join, abspath @@ -54,6 +55,9 @@ async def get_message(ref: str) -> ProgramMessage: with open(cache_path, "r") as cache_file: msg = json.load(cache_file) + if settings.FAKE_DATA: + msg['item_content'] = json.dumps(msg['content']) + msg['item_hash'] = hashlib.sha256(msg['item_content'].encode('utf-8')).hexdigest() return ProgramMessage(**msg) From dc91631892c0c7fecbcebcd3507368823d48f364 Mon Sep 17 00:00:00 2001 From: Moshe Malawach Date: Fri, 4 Jun 2021 10:35:40 +0200 Subject: [PATCH 069/990] lower the connection timeout as we retry --- runtimes/aleph-alpine-3.13-python/init1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index f3441d7cb..3bbfb67f3 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -229,7 +229,7 @@ async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[by headers = None body = None - async with aiohttp.ClientSession(conn_timeout=2) as session: + async with aiohttp.ClientSession(conn_timeout=.05) as session: while not body: try: tries += 1 From d2eb7ee8f6fe7b150d3261535b7aec15424bab3f Mon Sep 17 00:00:00 2001 From: Moshe Malawach Date: Fri, 4 Jun 2021 10:35:58 +0200 Subject: [PATCH 070/990] adding node_modules to gitignore (for examples in javascript) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a51dd6bbe..1b735cbd7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ __pycache__ *.pyz *.tgz /pydantic/ +node_modules \ No newline at end of file From 59729b2eaf602049dbd28fe88f5c5e2a18a98ecb Mon Sep 17 00:00:00 2001 From: Moshe Malawach Date: Fri, 4 Jun 2021 10:40:37 +0200 Subject: [PATCH 071/990] pass body to the vm in the scope (work on #27) --- vm_supervisor/supervisor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index d536b6f52..9850dbc2a 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -44,13 +44,14 @@ async def try_get_message(ref: str) -> ProgramMessage: raise -def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: +async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: return { "type": "http", "path": path, "method": request.method, "query_string": request.query_string, "headers": request.raw_headers, + "body": await request.text() } @@ -73,7 +74,7 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res logger.debug(f"Using vm={vm.vm_id}") - scope: Dict = build_asgi_scope(path, request) + scope: Dict = await build_asgi_scope(path, request) try: result_raw: bytes = await vm.run_code(scope=scope) From ac65921903a45edb6f795150e23c10b23960c7dd Mon Sep 17 00:00:00 2001 From: Moshe Malawach Date: Fri, 4 Jun 2021 10:41:05 +0200 Subject: [PATCH 072/990] pass body from scope to http request --- runtimes/aleph-alpine-3.13-python/init1.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 3bbfb67f3..8fce8c7bc 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -209,7 +209,8 @@ async def make_request(session, scope): url="http://localhost:8080{}".format(scope["path"]), params=scope["query_string"], headers=[(a.decode('utf-8'), b.decode('utf-8')) - for a, b in scope['headers']] + for a, b in scope['headers']], + data=scope.get("body", None) ) as resp: headers = { 'headers': [(a.encode('utf-8'), b.encode('utf-8')) From 94f579e4f97d4f2907be4bc3e0ea0473e53b4c12 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 4 Jun 2021 17:49:43 +0200 Subject: [PATCH 073/990] Feature: VMs use item_hash as hostname --- runtimes/aleph-alpine-3.13-python/init1.py | 7 +++++++ vm_supervisor/pool.py | 7 ++++--- vm_supervisor/supervisor.py | 2 +- vm_supervisor/vm/firecracker_microvm.py | 5 +++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 8fce8c7bc..9ac15744a 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -51,6 +51,7 @@ class ConfigurationPayload: entrypoint: str input_data: bytes interface: Interface + vm_hash: str @dataclass @@ -74,6 +75,11 @@ class RunCodePayload: logger.debug("init1.py is launching") +def setup_hostname(hostname: str): + os.environ["ALEPH_ADDRESS_TO_USE"] = hostname + system(f"hostname {hostname}") + + def setup_network(ip: Optional[str], route: Optional[str], dns_servers: Optional[List[str]] = None): """Setup the system with info from the host.""" @@ -322,6 +328,7 @@ def main(): msg_ = msgpack.loads(data, raw=False) config = ConfigurationPayload(**msg_) + setup_hostname(config.vm_hash) setup_network(config.ip, config.route, config.dns_servers) setup_input_data(config.input_data) logger.debug("Setup finished") diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index 5ebdc9916..a12e858e1 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -37,13 +37,14 @@ def __init__(self): self.counter = settings.START_ID_INDEX self.started_vms_cache = {} - async def create_a_vm(self, message_content: ProgramContent) -> AlephFirecrackerVM: + async def create_a_vm(self, message_content: ProgramContent, vm_hash: str) -> AlephFirecrackerVM: """Create a new Aleph Firecracker VM from an Aleph function message.""" vm_resources = AlephFirecrackerResources(message_content) await vm_resources.download_all() self.counter += 1 vm = AlephFirecrackerVM( vm_id=self.counter, + vm_hash=vm_hash, resources=vm_resources, enable_networking=message_content.environment.internet, hardware_resources=message_content.resources, @@ -59,14 +60,14 @@ async def create_a_vm(self, message_content: ProgramContent) -> AlephFirecracker raise - async def get_a_vm(self, message: ProgramContent) -> AlephFirecrackerVM: + async def get_a_vm(self, message: ProgramContent, vm_hash: str, ) -> AlephFirecrackerVM: """Provision a VM in the pool, then return the first VM from the pool.""" try: started_vm = self.started_vms_cache.pop(message) started_vm.timeout_task.cancel() return started_vm.vm except KeyError: - return await self.create_a_vm(message) + return await self.create_a_vm(message_content=message, vm_hash=vm_hash) def keep_in_cache( self, vm: AlephFirecrackerVM, message: ProgramContent, timeout: float = 1.0 diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 9850dbc2a..c5720cedf 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -64,7 +64,7 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res message_content: ProgramContent = message.content try: - vm = await pool.get_a_vm(message_content) + vm = await pool.get_a_vm(message_content, vm_hash=message.item_hash) except ResourceDownloadError as error: logger.exception(error) raise HTTPBadRequest(reason="Code, runtime or data not available") diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 26661683a..1eedd8166 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -59,6 +59,7 @@ class ConfigurationPayload: entrypoint: str input_data: bytes interface: Interface + vm_hash: str def as_msgpack(self) -> bytes: return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True) @@ -142,6 +143,7 @@ class VmSetupError(Exception): class AlephFirecrackerVM: vm_id: int + vm_hash: str resources: AlephFirecrackerResources enable_console: bool enable_networking: bool @@ -152,12 +154,14 @@ class AlephFirecrackerVM: def __init__( self, vm_id: int, + vm_hash: str, resources: AlephFirecrackerResources, enable_networking: bool = False, enable_console: Optional[bool] = None, hardware_resources: MachineResources = MachineResources() ): self.vm_id = vm_id + self.vm_hash = vm_hash self.resources = resources self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING if enable_console is None: @@ -229,6 +233,7 @@ async def configure(self): entrypoint=self.resources.code_entrypoint, input_data=input_data, interface=interface, + vm_hash=self.vm_hash, ) payload = config.as_msgpack() length = f"{len(payload)}\n".encode() From 93b4f99b7084815a6016e1c901dabbfaae559032 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 7 Jun 2021 13:01:12 +0200 Subject: [PATCH 074/990] Feature: Allow users to use their own domain for hosted VMs Do do this, users need to create the following DNS records: A CNAME record to the server, for example: `hosted-on-aleph IN CNAME aleph.sh.` A TXT record to the VM hash with the prefix `_aleph-id`, for example: `_aleph-id.hosted-on-aleph 60 IN TXT "b34f193470c349b1d9b60903a6d172e8c335710736d4999ff05971692febe8bc"` --- vm_supervisor/README.md | 42 ++++++++++++++++++++++++++++++++++++- vm_supervisor/supervisor.py | 14 ++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index e5298604b..ca18e8283 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -56,7 +56,7 @@ when running the VM Supervisor. ```shell apt update -apt install -y git python3 python3-aiohttp python3-msgpack sudo acl curl systemd-container +apt install -y git python3 python3-aiohttp python3-msgpack python3-dnspython sudo acl curl systemd-container useradd jailman ``` @@ -219,6 +219,46 @@ vm.yourdomain.org:443 { EOL ``` +Optionally, you can allow users to host their website using their own domains using the following +configuration. Be careful about rate limits if you enable `on_demand` TLS, +see the [Caddy documentation on On-Demand TLS](https://caddyserver.com/docs/automatic-https#on-demand-tls). +```shell +cat >/etc/caddy/Caddyfile < bytes: return b16encode(hash_bytes).lower() +async def get_ref_from_dns(domain): + resolver = aiodns.DNSResolver() + record = await resolver.query(domain, 'TXT') + return record[0].text + + async def run_code_from_hostname(request: web.Request) -> web.Response: """Allow running an Aleph VM function from a hostname @@ -155,8 +162,13 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: else: try: message_ref = b32_to_b16(message_ref_base32).decode() + logger.debug(f"Using base32 message id from hostname to obtain '{message_ref}") except binascii.Error: - raise HTTPNotFound(reason="Invalid message reference") + try: + message_ref = await get_ref_from_dns(domain=f"_aleph-id.{request.host}") + logger.debug(f"Using DNS TXT record to obtain '{message_ref}'") + except aiodns.error.DNSError: + raise HTTPNotFound(reason="Invalid message reference") return await run_code(message_ref, path, request) From df2a64bed42bb5d802cdeeab71c6da18f1e22586 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 4 Jun 2021 17:49:43 +0200 Subject: [PATCH 075/990] Feature: VMs use item_hash as hostname --- vm_supervisor/vm/firecracker_microvm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 1eedd8166..5ec3324e5 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -250,7 +250,8 @@ async def configure(self): async def start_guest_api(self): logger.debug(f"starting guest API for {self.vm_id}") vsock_path = f"{self.fvm.vsock_path}_53" - self.guest_api_process = Process(target=run_guest_api, args=(vsock_path,)) + vm_hash = self.vm_hash + self.guest_api_process = Process(target=run_guest_api, args=(vsock_path, vm_hash)) self.guest_api_process.start() while not exists(vsock_path): await asyncio.sleep(0.01) From 7c000cf5482504f87e56142039a53322fe8d3aa8 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 4 Jun 2021 17:58:54 +0200 Subject: [PATCH 076/990] Feature: Allow VMs to POST messages signed by the host --- examples/example_fastapi_2/__init__.py | 34 ++++++++- guest_api/__main__.py | 88 ++++++++++++++++++++-- runtimes/aleph-alpine-3.13-python/init1.py | 2 + vm_connector/main.py | 67 ++++++++++++++-- 4 files changed, 175 insertions(+), 16 deletions(-) diff --git a/examples/example_fastapi_2/__init__.py b/examples/example_fastapi_2/__init__.py index 0efdc5de6..c20cd1695 100644 --- a/examples/example_fastapi_2/__init__.py +++ b/examples/example_fastapi_2/__init__.py @@ -1,12 +1,15 @@ import logging +from datetime import datetime + logger = logging.getLogger(__name__) logger.debug("import aiohttp") import aiohttp - logger.debug("import aleph_client") -from aleph_client.asynchronous import get_messages +from aleph_client.asynchronous import get_messages, create_post +from aleph_client.chains.remote import RemoteAccount + logger.debug("import fastapi") from fastapi import FastAPI logger.debug("imports done") @@ -38,3 +41,30 @@ async def read_internet(): async with session.get("https://aleph.im/") as resp: resp.raise_for_status() return {"result": resp.status, "headers": resp.headers} + + +@app.get("/post_a_message") +async def post_a_message(): + """Post a message on the Aleph network""" + + account = await RemoteAccount.from_crypto_host( + host="http://localhost", unix_socket="/tmp/socat-socket") + + content = { + "date": datetime.utcnow().isoformat(), + "test": True, + "answer": 42, + "something": "interesting", + } + response = await create_post( + account=account, + post_content=content, + post_type="test", + ref=None, + channel="TEST", + inline=True, + storage_engine="storage", + ) + return { + "response": response, + } diff --git a/guest_api/__main__.py b/guest_api/__main__.py index c05ac8743..ec8a9d83a 100644 --- a/guest_api/__main__.py +++ b/guest_api/__main__.py @@ -1,15 +1,20 @@ +import json +import logging +from typing import Optional + import aiohttp from aiohttp import web +logger = logging.getLogger(__name__) -ALEPH_API_SERVER = "https://api2.aleph.im/" +ALEPH_API_SERVER = "https://api2.aleph.im" +ALEPH_VM_CONNECTOR = "http://localhost:8000" async def proxy(request: web.Request): - - path = request.match_info.get('tail') + path = request.match_info.get('tail').lstrip('/') query_string = request.rel_url.query_string - url = f"{ALEPH_API_SERVER}{path}?{query_string}" + url = f"{ALEPH_API_SERVER}/{path}?{query_string}" async with aiohttp.ClientSession() as session: async with session.request(method=request.method, url=url) as response: @@ -19,13 +24,84 @@ async def proxy(request: web.Request): content_type=response.content_type) -def run_guest_api(unix_socket_path): +async def repost(request: web.Request): + logger.debug("REPOST") + data_raw = await request.json() + topic, message = data_raw["topic"], json.loads(data_raw["data"]) + + content = json.loads(message["item_content"]) + content["address"] = "VM on executor" + message["item_content"] = json.dumps(content) + + new_data = {"topic": topic, "data": json.dumps(message)} + + path = request.path + if request.rel_url.query_string: + query_string = request.rel_url.query_string + url = f"{ALEPH_VM_CONNECTOR}{path}?{query_string}" + else: + url = f"{ALEPH_VM_CONNECTOR}{path}" + + print('url', url) + + async with aiohttp.ClientSession() as session: + async with session.post(url=url, json=new_data) as response: + data = await response.read() + print('DT', data) + return web.Response(body=data, + status=response.status, + content_type=response.content_type) + + +# async def decrypt_secret(request: web.Request): +# Not implemented... + + +async def properties(request: web.Request): + logger.debug("Forwarding signing properties") + + url = f"{ALEPH_VM_CONNECTOR}/properties" + async with aiohttp.ClientSession() as session: + async with session.get(url=url) as response: + data = await response.read() + return web.Response(body=data, + status=response.status, + content_type=response.content_type) + + +async def sign(request: web.Request): + vm_hash = request.app.meta_vm_hash + message = await request.json() + + # Ensure that the hash of the VM is used as sending address + content = json.loads(message["item_content"]) + if content["address"] != vm_hash: + raise web.HTTPBadRequest(reason="Message address does not match VM item_hash") + + logger.info("Forwarding signing request to VM Connector") + + url = f"{ALEPH_VM_CONNECTOR}/sign" + async with aiohttp.ClientSession() as session: + async with session.post(url=url, json=message) as response: + signed_message = await response.read() + print('SIG', signed_message) + return web.Response(body=signed_message, + status=response.status, + content_type=response.content_type) + + +def run_guest_api(unix_socket_path, vm_hash: Optional[str] = None): app = web.Application() + app.meta_vm_hash = vm_hash + app.router.add_route(method='GET', path='/properties', handler=properties) + app.router.add_route(method='POST', path='/sign', handler=sign) app.router.add_route(method='GET', path='/{tail:.*}', handler=proxy) app.router.add_route(method='HEAD', path='/{tail:.*}', handler=proxy) app.router.add_route(method='OPTIONS', path='/{tail:.*}', handler=proxy) + app.router.add_route(method='POST', path='/api/v0/ipfs/pubsub/pub', handler=repost) + app.router.add_route(method='POST', path='/api/v0/p2p/pubsub/pub', handler=repost) web.run_app(app=app, path=unix_socket_path) if __name__ == '__main__': - run_guest_api("/tmp/guest-api") + run_guest_api("/tmp/guest-api", vm_hash='vm') diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 9ac15744a..51d04ac8d 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -71,6 +71,8 @@ class RunCodePayload: # Configure aleph-client to use the guest API os.environ["ALEPH_API_UNIX_SOCKET"] = "/tmp/socat-socket" +os.environ["ALEPH_REMOTE_CRYPTO_HOST"] = "http://localhost" +os.environ["ALEPH_REMOTE_CRYPTO_UNIX_SOCKET"] = "/tmp/socat-socket" logger.debug("init1.py is launching") diff --git a/vm_connector/main.py b/vm_connector/main.py index b4879ace9..5b178bff6 100644 --- a/vm_connector/main.py +++ b/vm_connector/main.py @@ -3,12 +3,17 @@ import os.path from typing import Optional, Dict, Union -# from aleph_client.chains.common import get_fallback_private_key -# from aleph_client.asynchronous import get_posts +from aleph_client.asynchronous import get_posts, create_post +from aleph_client.chains.common import get_fallback_private_key +from aleph_client.chains.ethereum import ETHAccount + import aiohttp from fastapi import FastAPI from fastapi.responses import StreamingResponse, Response, FileResponse +from fastapi import Request +from pydantic import BaseModel + from .conf import settings logger = logging.getLogger(__file__) @@ -159,11 +164,57 @@ async def download_runtime( return StreamingResponse(stream_url_chunks(url), media_type="application/ext4") -@app.post("/publish/data/") -async def publish_data(encoding: str): +class PostBody(BaseModel): + topic: str + data: str + + +@app.post("/api/v0/ipfs/pubsub/pub") +@app.post("/api/v0/p2p/pubsub/pub") +async def publish_data(body: PostBody): """ - Publish a new state on the Aleph Network. - :param encoding: - :return: + Publish a new POST message on the Aleph Network. """ - raise NotImplementedError() + private_key = get_fallback_private_key() + account: ETHAccount = ETHAccount(private_key=private_key) + + message = json.loads(body.data) + content = json.loads(message["item_content"]) + content_content = content["content"] + + result = await create_post( + account=account, + post_content=content_content, + post_type=content["type"], + address=content["address"], + ref=None, + channel=message["channel"], + inline=True, + storage_engine="storage", + ) + return {"status": "success"} + + +@app.get("/properties") +async def properties(request: Request): + """Get signing key properties""" + private_key = get_fallback_private_key() + account: ETHAccount = ETHAccount(private_key=private_key) + + return { + "chain": account.CHAIN, + "curve": account.CURVE, + "address": account.get_address(), + "public_key": account.get_public_key(), + } + + +@app.post("/sign") +async def sign_message(request: Request): + """Sign a message""" + private_key = get_fallback_private_key() + account: ETHAccount = ETHAccount(private_key=private_key) + + message = await request.json() + message = account.sign_message(message) + return message From f4f6900e3443d7c6125a6f87cc4830b6ff8b4ebc Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 4 Jun 2021 18:01:22 +0200 Subject: [PATCH 077/990] Fix: Benchmark was broken by 59729b2eaf602049dbd28fe88f5c5e2a18a98ecb --- vm_supervisor/__main__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index a34da948a..a9fc805dc 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -3,6 +3,7 @@ import logging import sys import time +from asyncio import coroutine from statistics import mean from typing import List @@ -108,6 +109,7 @@ class FakeRequest: pass fake_request.query_string = "" fake_request.headers = [] fake_request.raw_headers = [] + fake_request.text = coroutine(lambda: None) logger.info("--- Start benchmark ---") From bdf0f598795797a660188a0bc6fed6cbf2e3e8bb Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 4 Jun 2021 18:02:09 +0200 Subject: [PATCH 078/990] Enhancement: Log exception on configuration error --- vm_supervisor/vm/firecracker_microvm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 5ec3324e5..dae208d1f 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -245,6 +245,7 @@ async def configure(self): response = ConfigurationResponse( **msgpack.loads(response_raw, raw=False)) if response.success is False: + logger.exception(response.traceback) raise VmSetupError(response.error) async def start_guest_api(self): From 91d94a0125328e1ef6e20bb26465cd17edc55b1f Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 4 Jun 2021 18:13:47 +0200 Subject: [PATCH 079/990] Chore: Require aleph-client >= 0.2.2 --- runtimes/aleph-alpine-3.13-python/create_disk_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index 691a053ab..2b961a101 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -26,7 +26,7 @@ apk add py3-aiohttp py3-msgpack pip install fastapi apk add git pkgconf gcc py3-wheel python3-dev musl-dev py3-cffi libffi-dev autoconf automake libtool make -pip install git+https://github.com/aleph-im/aleph-client coincurve==15.0.0 +pip install aleph-client>=0.2.2 coincurve==15.0.0 # Compile all Python bytecode python3 -m compileall -f /usr/lib/python3.8/site-packages From f217671b054450f99f1b6c192f1970fdd1ff73df Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 7 Jun 2021 15:39:27 +0200 Subject: [PATCH 080/990] Fix: Unzip logs in init could be too long --- runtimes/aleph-alpine-3.13-python/init1.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 51d04ac8d..d1d35435f 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -119,7 +119,7 @@ def setup_input_data(input_data: bytes): if not os.path.exists("/opt/input.zip"): open("/opt/input.zip", "wb").write(input_data) os.makedirs("/data", exist_ok=True) - os.system("unzip /opt/input.zip -d /data") + os.system("unzip -q /opt/input.zip -d /data") def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplication: @@ -129,7 +129,7 @@ def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApp if not os.path.exists("/opt/archive.zip"): open("/opt/archive.zip", "wb").write(code) logger.debug("Run unzip") - os.system("unzip /opt/archive.zip -d /opt") + os.system("unzip -q /opt/archive.zip -d /opt") sys.path.append("/opt") module_name, app_name = entrypoint.split(":", 1) logger.debug("import module") From 963303b20cc781c85e17117ab50b63d2a5943438 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 8 Jun 2021 17:13:32 +0200 Subject: [PATCH 081/990] Refactor: Rename last_amend -> use_latest --- vm_connector/main.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/vm_connector/main.py b/vm_connector/main.py index 5b178bff6..bfd9911e5 100644 --- a/vm_connector/main.py +++ b/vm_connector/main.py @@ -58,14 +58,14 @@ async def stream_url_chunks(url): @app.get("/download/message/{ref}") async def download_message( - ref: str, last_amend: Optional[bool] = True + ref: str, use_latest: Optional[bool] = True ) -> Union[Dict, Response]: """ Fetch on Aleph and return a VM function message, after checking its validity. Used by the VM Supervisor run the code. - +K :param ref: item_hash of the code file - :param last_amend: should the last amend to the code be used + :param use_latest: should the last amend to the code be used :return: a file containing the code file """ @@ -83,14 +83,14 @@ async def download_message( @app.get("/download/code/{ref}") async def download_code( - ref: str, last_amend: Optional[bool] = True + ref: str, use_latest: Optional[bool] = True ) -> Union[StreamingResponse, Response]: """ Fetch on Aleph and return a VM code file, after checking its validity. Used by the VM Supervisor to download function source code. :param ref: item_hash of the code file - :param last_amend: should the last amend to the code be used + :param use_latest: should the last amend to the code be used :return: a file containing the code file """ @@ -112,14 +112,14 @@ async def download_code( @app.get("/download/data/{ref}") async def download_data( - ref: str, last_amend: Optional[bool] = True + ref: str, use_latest: Optional[bool] = True ) -> Union[StreamingResponse, Response]: """ Fetch on Aleph and return a VM data file, after checking its validity. Used by the VM Supervisor to download state data. :param ref: item_hash of the data - :param last_amend: should the last amend to the data be used + :param use_latest: should the last amend to the data be used :return: a file containing the data """ @@ -139,14 +139,14 @@ async def download_data( @app.get("/download/runtime/{ref}") async def download_runtime( - ref: str, last_amend: Optional[bool] = True + ref: str, use_latest: Optional[bool] = True ) -> Union[StreamingResponse, Response]: """ Fetch on Aleph and return a VM runtime, after checking its validity. Used by the VM Supervisor to download a runtime. :param ref: item_hash of the runtime - :param last_amend: should the last amend to the runtime be used + :param use_latest: should the last amend to the runtime be used :return: a file containing the runtime """ From d303082ee3cf70112f675abfe30fb44d2ec72afa Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 10 Jun 2021 09:31:34 +0200 Subject: [PATCH 082/990] Feature: Add support for amend on runtimes --- vm_connector/main.py | 29 +++++++++++++++++++++++++++++ vm_supervisor/pool.py | 2 +- vm_supervisor/storage.py | 13 +++++++++++++ vm_supervisor/supervisor.py | 20 +++++++++++++++++++- 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/vm_connector/main.py b/vm_connector/main.py index bfd9911e5..585f593b2 100644 --- a/vm_connector/main.py +++ b/vm_connector/main.py @@ -32,6 +32,19 @@ class Encoding: zip = "zip" +async def get_latest_message_amend(ref: str, sender: str) -> Optional[Dict]: + async with aiohttp.ClientSession() as session: + url = f"{settings.ALEPH_SERVER}/api/v0/messages.json?msgType=STORE&sort_order=-1" \ + f"&refs={ref}&addresses={sender}" + resp = await session.get(url) + resp.raise_for_status() + resp_data = await resp.json() + if resp_data["messages"]: + return resp_data["messages"][0] + else: + return None + + async def get_message(hash_: str) -> Optional[Dict]: async with aiohttp.ClientSession() as session: url = f"{settings.ALEPH_SERVER}/api/v0/messages.json?hashes={hash_}" @@ -164,6 +177,22 @@ async def download_runtime( return StreamingResponse(stream_url_chunks(url), media_type="application/ext4") +@app.get("/compute/latest_amend/{item_hash}") +async def compute_latest_amend(item_hash: str) -> str: + msg = await get_message(hash_=item_hash) + sender = msg['sender'] + latest_amend = await get_latest_message_amend(ref=item_hash, sender=sender) + if latest_amend: + # Validation + assert latest_amend['sender'] == sender + assert latest_amend['content']['ref'] == item_hash + + return latest_amend['item_hash'] + else: + # Original message is the latest + return item_hash + + class PostBody(BaseModel): topic: str data: str diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index a12e858e1..82b8481ce 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -60,7 +60,7 @@ async def create_a_vm(self, message_content: ProgramContent, vm_hash: str) -> Al raise - async def get_a_vm(self, message: ProgramContent, vm_hash: str, ) -> AlephFirecrackerVM: + async def get_a_vm(self, message: ProgramContent, vm_hash: str) -> AlephFirecrackerVM: """Provision a VM in the pool, then return the first VM from the pool.""" try: started_vm = self.started_vms_cache.pop(message) diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index da7acf744..8147ae38d 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -43,6 +43,19 @@ async def download_file(url: str, local_path: FilePath) -> None: raise +async def get_latest_amend(item_hash: str) -> str: + if settings.FAKE_DATA: + return item_hash + else: + url = f"{settings.CONNECTOR_URL}/compute/latest_amend/{item_hash}" + async with aiohttp.ClientSession() as session: + resp = await session.get(url) + resp.raise_for_status() + result: str = await resp.json() + assert isinstance(result, str) + return result or item_hash + + async def get_message(ref: str) -> ProgramMessage: if settings.FAKE_DATA: cache_path = os.path.abspath( diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 9a2978f38..d646ff419 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -20,7 +20,7 @@ from aleph_message.models import ProgramMessage, ProgramContent from .conf import settings from .pool import VmPool -from .storage import get_message +from .storage import get_message, get_latest_amend from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError logger = logging.getLogger(__name__) @@ -45,6 +45,18 @@ async def try_get_message(ref: str) -> ProgramMessage: raise +async def get_latest_ref(item_hash: str) -> str: + try: + return await get_latest_amend(item_hash) + except ClientConnectorError: + raise HTTPServiceUnavailable(reason="Aleph Connector unavailable") + except ClientResponseError as error: + if error.status == 404: + raise HTTPNotFound(reason="Hash not found") + else: + raise + + async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: return { "type": "http", @@ -64,6 +76,12 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res message: ProgramMessage = await try_get_message(message_ref) message_content: ProgramContent = message.content + # Load amends + if message_content.runtime.use_latest: + message_content.runtime.ref = await get_latest_ref(message_content.runtime.ref) + + # TODO: Cache message content after amends + try: vm = await pool.get_a_vm(message_content, vm_hash=message.item_hash) except ResourceDownloadError as error: From 04c26a538d253ce26e5125ea258416264a97e243 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 10 Jun 2021 15:20:25 +0200 Subject: [PATCH 083/990] Feature: Add a cache per VM The cache is available even after a VM has been teared down, and is shared between different instances of a VM. --- examples/example_fastapi_2/__init__.py | 21 ++++++++ guest_api/__main__.py | 52 ++++++++++++++++++- .../create_disk_image.sh | 2 +- runtimes/aleph-alpine-3.13-python/init1.py | 1 + vm_supervisor/README.md | 2 +- 5 files changed, 75 insertions(+), 3 deletions(-) diff --git a/examples/example_fastapi_2/__init__.py b/examples/example_fastapi_2/__init__.py index c20cd1695..d2abb8a3a 100644 --- a/examples/example_fastapi_2/__init__.py +++ b/examples/example_fastapi_2/__init__.py @@ -9,12 +9,14 @@ logger.debug("import aleph_client") from aleph_client.asynchronous import get_messages, create_post from aleph_client.chains.remote import RemoteAccount +from aleph_client.vm.cache import VmCache logger.debug("import fastapi") from fastapi import FastAPI logger.debug("imports done") app = FastAPI() +cache = VmCache() @app.get("/") @@ -68,3 +70,22 @@ async def post_a_message(): return { "response": response, } + + +@app.get("/cache/get/{key}") +async def get_from_cache(key: str): + """Get data in the VM cache""" + return await cache.get(key) + + +@app.get("/cache/set/{key}/{value}") +async def store_in_cache(key: str, value: str): + """Store data in the VM cache""" + return await cache.set(key, value) + + +@app.get("/cache/remove/{key}") +async def remove_from_cache(key: str): + """Store data in the VM cache""" + result = await cache.delete(key) + return result == 1 diff --git a/guest_api/__main__.py b/guest_api/__main__.py index ec8a9d83a..c77f011c7 100644 --- a/guest_api/__main__.py +++ b/guest_api/__main__.py @@ -1,14 +1,17 @@ import json import logging +import re from typing import Optional import aiohttp from aiohttp import web +import aioredis logger = logging.getLogger(__name__) ALEPH_API_SERVER = "https://api2.aleph.im" ALEPH_VM_CONNECTOR = "http://localhost:8000" +CACHE_EXPIRES_AFTER = 7 * 24 * 3600 # Seconds async def proxy(request: web.Request): @@ -90,16 +93,63 @@ async def sign(request: web.Request): content_type=response.content_type) +async def get_from_cache(request: web.Request): + prefix: str = request.app.meta_vm_hash + key: str = request.match_info.get('key') + if not re.match(r'^\w+$', key): + return web.HTTPBadRequest(text="Invalid key") + + redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost") + body = await redis.get(f"{prefix}:{key}") + if body: + return web.Response(body=body, status=200) + else: + return web.Response(text="No such key in cache", status=404) + + +async def put_in_cache(request: web.Request): + prefix: str = request.app.meta_vm_hash + key: str = request.match_info.get('key') + if not re.match(r'^\w+$', key): + return web.HTTPBadRequest(text="Invalid key") + + value: bytes = await request.read() + + redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost") + return web.json_response(await redis.set(f"{prefix}:{key}", value, + expire=CACHE_EXPIRES_AFTER)) + + +async def delete_from_cache(request: web.Request): + prefix: str = request.app.meta_vm_hash + key: str = request.match_info.get('key') + if not re.match(r'^\w+$', key): + return web.HTTPBadRequest(text="Invalid key") + + redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost") + logger.debug("DEL", f"{prefix}:{key}") + result = await redis.delete(f"{prefix}:{key}") + return web.json_response(result) + + def run_guest_api(unix_socket_path, vm_hash: Optional[str] = None): app = web.Application() - app.meta_vm_hash = vm_hash + app.meta_vm_hash = vm_hash or '_' + app.router.add_route(method='GET', path='/properties', handler=properties) app.router.add_route(method='POST', path='/sign', handler=sign) + + app.router.add_route(method='GET', path='/cache/{key:.*}', handler=get_from_cache) + app.router.add_route(method='PUT', path='/cache/{key:.*}', handler=put_in_cache) + app.router.add_route(method='DELETE', path='/cache/{key:.*}', handler=delete_from_cache) + app.router.add_route(method='GET', path='/{tail:.*}', handler=proxy) app.router.add_route(method='HEAD', path='/{tail:.*}', handler=proxy) app.router.add_route(method='OPTIONS', path='/{tail:.*}', handler=proxy) + app.router.add_route(method='POST', path='/api/v0/ipfs/pubsub/pub', handler=repost) app.router.add_route(method='POST', path='/api/v0/p2p/pubsub/pub', handler=repost) + web.run_app(app=app, path=unix_socket_path) diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index 2b961a101..9a090f9a3 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -26,7 +26,7 @@ apk add py3-aiohttp py3-msgpack pip install fastapi apk add git pkgconf gcc py3-wheel python3-dev musl-dev py3-cffi libffi-dev autoconf automake libtool make -pip install aleph-client>=0.2.2 coincurve==15.0.0 +pip install aleph-client>=0.2.4 coincurve==15.0.0 # Compile all Python bytecode python3 -m compileall -f /usr/lib/python3.8/site-packages diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index d1d35435f..bbad7c7d7 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -70,6 +70,7 @@ class RunCodePayload: s0.close() # Configure aleph-client to use the guest API +os.environ["ALEPH_API_HOST"] = "http://localhost" os.environ["ALEPH_API_UNIX_SOCKET"] = "/tmp/socat-socket" os.environ["ALEPH_REMOTE_CRYPTO_HOST"] = "http://localhost" os.environ["ALEPH_REMOTE_CRYPTO_UNIX_SOCKET"] = "/tmp/socat-socket" diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index ca18e8283..16877cb58 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -56,7 +56,7 @@ when running the VM Supervisor. ```shell apt update -apt install -y git python3 python3-aiohttp python3-msgpack python3-dnspython sudo acl curl systemd-container +apt install -y git python3 python3-aiohttp python3-msgpack python3-dnspython redis python3-aioredis sudo acl curl systemd-container useradd jailman ``` From 615c97fa10d9ead84e3d011abac27b0ec2900876 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 11 Jun 2021 13:27:35 +0200 Subject: [PATCH 084/990] WIP: Add mount of extra volumes on VMs This allows VM publishers to specify other volumes to mount on the VM. Volumes must be formatted as SquashFS partitions. --- .gitignore | 3 +- examples/example_fastapi_2/__init__.py | 6 +- examples/message_from_aleph.json | 112 +++++++++++------- examples/volumes/Dockerfile | 11 ++ examples/volumes/build_squashfs.sh | 6 + firecracker/microvm.py | 27 ++++- .../create_disk_image.sh | 2 +- runtimes/aleph-alpine-3.13-python/init0.sh | 3 + runtimes/aleph-alpine-3.13-python/init1.py | 10 ++ vm_supervisor/README.md | 2 +- vm_supervisor/storage.py | 11 ++ vm_supervisor/vm/firecracker_microvm.py | 28 ++++- 12 files changed, 168 insertions(+), 53 deletions(-) create mode 100644 examples/volumes/Dockerfile create mode 100644 examples/volumes/build_squashfs.sh diff --git a/.gitignore b/.gitignore index 1b735cbd7..d36d1361d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ __pycache__ *.pyz *.tgz /pydantic/ -node_modules \ No newline at end of file +node_modules +*.squashfs diff --git a/examples/example_fastapi_2/__init__.py b/examples/example_fastapi_2/__init__.py index d2abb8a3a..7d798e501 100644 --- a/examples/example_fastapi_2/__init__.py +++ b/examples/example_fastapi_2/__init__.py @@ -1,5 +1,6 @@ import logging from datetime import datetime +from os import listdir logger = logging.getLogger(__name__) @@ -23,7 +24,10 @@ async def index(): return { "Example": "example_fastapi_2", - "endpoints": ["/messages", "/internet"], + "endpoints": ["/messages", "/internet", "/post_a_message"], + "files_in_volumes": { + "/opt/venv": list(listdir("/opt/venv")) + }, } diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index aef5e583a..7a26b9958 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -1,49 +1,71 @@ { - "_id": { - "$oid": "60ae674509e67ee2839b67cb" - }, - "chain": "ETH", - "item_hash": "2197b6d13fbeecee029807b2de5d3576e71364b94e9379f500ac26eb0d529ae9", - "sender": "0x0bE24CB9568dA8ec4d33c8E2aA25Fb841550e607", - "type": "PROGRAM", - "channel": "TEST", - "confirmed": false, - "content": { - "address": "0x0bE24CB9568dA8ec4d33c8E2aA25Fb841550e607", - "time": 1622042437.3603787, - "type": "vm-function", - "allow_amend": false, - "code": { - "encoding": "zip", - "entrypoint": "example_fastapi_2:app", - "ref": "78eda627a1a51cb783197cda49c24b66864c4f18843adfec6d9675a5337eb48a", - "use_latest": false + "_id": { + "$oid": "6080402d7f44efefd611dc1e" }, - "data": null, - "export": null, - "on": { - "http": true + "chain": "ETH", + "item_hash": "787fb143b2ac74c6cc348b3fc10bb571d41f372156ab2f54b0e41494b58b1a1e", + "sender": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", + "type": "PROGRAM", + "channel": "Fun-dApps", + "confirmed": true, + "content": { + "type": "vm-function", + "address": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", + "allow_amend": false, + "code": { + "encoding": "zip", + "entrypoint": "example_fastapi_2:app", + "ref": "7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003", + "use_latest": false + }, + "on": { + "http": true + }, + "environment": { + "reproducible": true, + "internet": false, + "aleph_api": false + }, + "resources": { + "vcpus": 1, + "memory": 128, + "seconds": 30 + }, + "runtime": { + "ref": "5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51", + "use_latest": false, + "comment": "Aleph Alpine Linux with Python 3.8" + }, + "volumes": [ + { + "mount": "/opt/venv", + "ref": "5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51", + "use_latest": false + } + ], + "data": { + "encoding": "zip", + "mount": "/data", + "ref": "7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003", + "use_latest": false + }, + "export": { + "encoding": "zip", + "mount": "/data" + }, + "replaces": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", + "time": 1619017773.8950517 }, - "environment": { - "reproducible": false, - "internet": true, - "aleph_api": true - }, - "resources": { - "vcpus": 1, - "memory": 128, - "seconds": 30 - }, - "runtime": { - "ref": "c05879dd0229e686eeaaf3e1a55cf7d1503a3d9ae11854a2cf6e5a3913e1a080", - "use_latest": true, - "comment": "Aleph Alpine Linux with Python 3.8" - }, - "replaces": null - }, - "item_content": "{\"address\":\"0x0bE24CB9568dA8ec4d33c8E2aA25Fb841550e607\",\"time\":1622042437.3603787,\"type\":\"vm-function\",\"allow_amend\":false,\"code\":{\"encoding\":\"zip\",\"entrypoint\":\"example_fastapi_2:app\",\"ref\":\"78eda627a1a51cb783197cda49c24b66864c4f18843adfec6d9675a5337eb48a\",\"use_latest\":false},\"data\":null,\"export\":null,\"on\":{\"http\":true},\"environment\":{\"reproducible\":false,\"internet\":true,\"aleph_api\":true},\"resources\":{\"vcpus\":1,\"memory\":128,\"seconds\":30},\"runtime\":{\"ref\":\"c05879dd0229e686eeaaf3e1a55cf7d1503a3d9ae11854a2cf6e5a3913e1a080\",\"use_latest\":true,\"comment\":\"Aleph Alpine Linux with Python 3.8\"},\"replaces\":null}", - "item_type": "inline", - "signature": "0x6d26aed5d9968d32e5536e058f558d3c5ef9524686a17bfe0e944517276cd308578f746a403a911c7ef07bd60833bb5bfbc42e48c5f3a800a58971da81ab15d61b", - "size": 609, - "time": 1622042437.3610504 + "item_content": "{\"type\": \"vm-function\", \"address\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"allow_amend\": false, \"code\": {\"encoding\": \"zip\", \"entrypoint\": \"example_fastapi_2:app\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"on\": {\"http\": true}, \"environment\": {\"reproducible\": true, \"internet\": false, \"aleph_api\": false}, \"resources\": {\"vcpus\": 1, \"memory\": 128, \"seconds\": 30}, \"runtime\": {\"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false, \"comment\": \"Aleph Alpine Linux with Python 3.8\"}, \"volumes\": [{\"mount\": \"/opt/venv\", \"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false}], \"data\": {\"encoding\": \"zip\", \"mount\": \"/data\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"export\": {\"encoding\": \"zip\", \"mount\": \"/data\"}, \"replaces\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"time\": 1619017773.8950517}", + "item_type": "inline", + "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b", + "size": 749, + "time": 1619017773.8950577, + "confirmations": [ + { + "chain": "ETH", + "height": 12284734, + "hash": "0x67f2f3cde5e94e70615c92629c70d22dc959a118f46e9411b29659c2fce87cdc" + } + ] } diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile new file mode 100644 index 000000000..8ecd44529 --- /dev/null +++ b/examples/volumes/Dockerfile @@ -0,0 +1,11 @@ +FROM debian:buster + +RUN apt-get update && apt-get -y upgrade && apt-get install -y \ + python3-venv \ + squashfs-tools \ + && rm -rf /var/lib/apt/lists/* + +RUN python3 -m venv /opt/venv +RUN /opt/venv/bin/pip install aleph-message + +CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs diff --git a/examples/volumes/build_squashfs.sh b/examples/volumes/build_squashfs.sh new file mode 100644 index 000000000..4f662ee8f --- /dev/null +++ b/examples/volumes/build_squashfs.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +set -euf + +podman build -t aleph-vm-build-squashfs . +podman run --rm -ti -v "$( dirname "$0" )":/mnt aleph-vm-build-squashfs diff --git a/firecracker/microvm.py b/firecracker/microvm.py index fe13e5cd7..e3404c5cb 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -2,16 +2,19 @@ import json import logging import os.path +import string from asyncio import Task from enum import Enum from os import getuid from pathlib import Path from pwd import getpwnam -from typing import Optional, Tuple +from typing import Optional, Tuple, Dict import aiohttp from aiohttp import ClientResponse +from vm_supervisor.models import FilePath + logger = logging.getLogger(__name__) @@ -231,6 +234,28 @@ async def set_rootfs(self, path_on_host: str): response = await session.put("http://localhost/drives/rootfs", json=data) response.raise_for_status() + async def mount(self, volume_paths: Dict[str, FilePath]): + counter = 1 + for path, partition_path in volume_paths.items(): + device_name = f"vd{string.ascii_lowercase[counter]}" + if self.use_jailer: + partition_filename = Path(partition_path).name + jailer_path_on_host = f"/opt/{partition_filename}" + os.link(partition_path, f"{self.jailer_path}/{jailer_path_on_host}") + partition_path = jailer_path_on_host + + data = { + "drive_id": device_name, + "path_on_host": partition_path, + "is_root_device": False, + "is_read_only": True, + } + async with self.get_session() as session: + response = await session.put(f"http://localhost/drives/{device_name}", json=data) + response.raise_for_status() + counter += 1 + + async def set_vsock(self): data = { "vsock_id": "1", diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index 9a090f9a3..38d1e5be5 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -26,7 +26,7 @@ apk add py3-aiohttp py3-msgpack pip install fastapi apk add git pkgconf gcc py3-wheel python3-dev musl-dev py3-cffi libffi-dev autoconf automake libtool make -pip install aleph-client>=0.2.4 coincurve==15.0.0 +pip install aleph-client>=0.2.5 coincurve==15.0.0 # Compile all Python bytecode python3 -m compileall -f /usr/lib/python3.8/site-packages diff --git a/runtimes/aleph-alpine-3.13-python/init0.sh b/runtimes/aleph-alpine-3.13-python/init0.sh index 7860b9399..fe9d23bfd 100644 --- a/runtimes/aleph-alpine-3.13-python/init0.sh +++ b/runtimes/aleph-alpine-3.13-python/init0.sh @@ -33,6 +33,9 @@ mount -t tmpfs run /run -o mode=0755,nosuid,nodev mount -t devpts devpts /dev/pts -o mode=0620,gid=5,nosuid,noexec mount -t tmpfs shm /dev/shm -omode=1777,nosuid,nodev +# List block devices +lsblk + #cat /proc/sys/kernel/random/entropy_avail # TODO: Move in init1 diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index bbad7c7d7..60303484a 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -52,6 +52,7 @@ class ConfigurationPayload: input_data: bytes interface: Interface vm_hash: str + volumes: Dict[str, str] @dataclass @@ -123,6 +124,14 @@ def setup_input_data(input_data: bytes): os.system("unzip -q /opt/input.zip -d /data") +def setup_volumes(volumes: Dict[str, str]): + for path, device in volumes.items(): + logger.debug(f"Mounting /dev/{device} on {path}") + os.makedirs(path, exist_ok=True) + system(f"mount -t squashfs -o ro /dev/{device} {path}") + system("mount") + + def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplication: logger.debug("Extracting code") if encoding == Encoding.zip: @@ -332,6 +341,7 @@ def main(): config = ConfigurationPayload(**msg_) setup_hostname(config.vm_hash) + setup_volumes(config.volumes) setup_network(config.ip, config.route, config.dns_servers) setup_input_data(config.input_data) logger.debug("Setup finished") diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index 16877cb58..752516453 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -87,7 +87,7 @@ is used to parse and validate Aleph messages. ```shell apt install -y --no-install-recommends --no-install-suggests python3-pip pip3 install pydantic[dotenv] -pip3 install aleph-message>=0.1.6 +pip3 install aleph-message>=0.1.7 ``` ### 2.f. Create the jailer working directory: diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index 8147ae38d..ebc481db2 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -113,3 +113,14 @@ async def get_runtime_path(ref: str) -> FilePath: url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" await download_file(url, cache_path) return cache_path + + +async def get_volume_path(ref: str) -> FilePath: + if settings.FAKE_DATA: + data_dir = abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs")) + return FilePath(data_dir) + + cache_path = FilePath(join(settings.DATA_CACHE, ref)) + url = f"{settings.CONNECTOR_URL}/download/data/{ref}" + await download_file(url, cache_path) + return cache_path diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index dae208d1f..d0a9ba18c 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -1,6 +1,7 @@ import asyncio import dataclasses import logging +import string from dataclasses import dataclass from enum import Enum from multiprocessing import Process, set_start_method @@ -12,12 +13,12 @@ from aiohttp import ClientResponseError from aleph_message.models import ProgramContent -from aleph_message.models.program import MachineResources +from aleph_message.models.program import MachineResources, MachineVolume from firecracker.microvm import MicroVM, setfacl, Encoding from guest_api.__main__ import run_guest_api from ..conf import settings from ..models import FilePath -from ..storage import get_code_path, get_runtime_path, get_data_path +from ..storage import get_code_path, get_runtime_path, get_data_path, get_volume_path logger = logging.getLogger(__name__) set_start_method("spawn") @@ -60,6 +61,7 @@ class ConfigurationPayload: input_data: bytes interface: Interface vm_hash: str + volumes: Dict[str, str] def as_msgpack(self) -> bytes: return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True) @@ -89,12 +91,15 @@ class AlephFirecrackerResources: code_encoding: Encoding code_entrypoint: str rootfs_path: FilePath + volumes: List[MachineVolume] + volume_paths: Dict[str, FilePath] data_path: Optional[FilePath] def __init__(self, message_content: ProgramContent): self.message_content = message_content self.code_encoding = message_content.code.encoding self.code_entrypoint = message_content.code.entrypoint + self.volumes = message_content.volumes async def download_kernel(self): # Assumes kernel is already present on the host @@ -128,11 +133,19 @@ async def download_data(self): else: self.data_path = None + async def download_volumes(self): + volume_paths = {} + # TODO: Download in parallel + for volume in self.volumes: + volume_paths[volume.mount] = await get_volume_path(volume.ref) + self.volume_paths = volume_paths + async def download_all(self): await asyncio.gather( self.download_kernel(), self.download_code(), self.download_runtime(), + self.download_volumes(), self.download_data(), ) @@ -148,7 +161,7 @@ class AlephFirecrackerVM: enable_console: bool enable_networking: bool hardware_resources: MachineResources - fvm: MicroVM + fvm: MicroVM = None guest_api_process: Optional[Process] = None def __init__( @@ -187,6 +200,8 @@ async def setup(self): enable_console=self.enable_console, ) await fvm.set_rootfs(self.resources.rootfs_path) + await fvm.mount(self.resources.volume_paths) + await fvm.set_vsock() await fvm.set_resources(vcpus=self.hardware_resources.vcpus, memory=self.hardware_resources.memory) @@ -223,6 +238,12 @@ async def configure(self): interface = Interface.asgi if ":" in self.resources.code_entrypoint \ else Interface.executable + # Start at vdb since vda is already used by the root filesystem + volumes: Dict[str, str] = { + volume.mount: f"vd{string.ascii_lowercase[index+1]}" + for index, volume in enumerate(self.resources.volumes) + } + reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path) config = ConfigurationPayload( ip=self.fvm.guest_ip if self.enable_networking else None, @@ -234,6 +255,7 @@ async def configure(self): input_data=input_data, interface=interface, vm_hash=self.vm_hash, + volumes=volumes, ) payload = config.as_msgpack() length = f"{len(payload)}\n".encode() From 2f10edfbe97a8bbe9b6dfb28f4e42e194bef9361 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 11 Jun 2021 13:27:53 +0200 Subject: [PATCH 085/990] Cleanup: Remove deprecated comment --- runtimes/aleph-alpine-3.13-python/init0.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init0.sh b/runtimes/aleph-alpine-3.13-python/init0.sh index fe9d23bfd..a114e8a3b 100644 --- a/runtimes/aleph-alpine-3.13-python/init0.sh +++ b/runtimes/aleph-alpine-3.13-python/init0.sh @@ -20,10 +20,6 @@ pivot_root /mnt /mnt/rom mount --move /rom/proc /proc mount --move /rom/dev /dev -#echo "Mounts" -#ls / -#ls /dev - mkdir -p /dev/pts mkdir -p /dev/shm From eab5e6db9de54b6ae8c48a05ab1226caec4ac518 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 11 Jun 2021 13:28:24 +0200 Subject: [PATCH 086/990] Fix: Variable is not always defined --- vm_supervisor/vm/firecracker_microvm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index d0a9ba18c..2fb8cb8ff 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -286,7 +286,8 @@ async def stop_guest_api(self): self.guest_api_process.terminate() async def teardown(self): - await self.fvm.teardown() + if self.fvm: + await self.fvm.teardown() await self.stop_guest_api() async def run_code( From 2c02525630e2ee9acf0b5ec19ceb2638a549816e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 11 Jun 2021 13:28:49 +0200 Subject: [PATCH 087/990] Chore: Upgrade runtime Alpine version to 3.13.5 --- runtimes/aleph-alpine-3.13-python/create_disk_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index 38d1e5be5..5c82dcd7b 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -4,7 +4,7 @@ umount /mnt/rootfs set -euf -curl -fsSL -o ./alpine-miniroot.tgz https://dl-cdn.alpinelinux.org/alpine/v3.13/releases/x86_64/alpine-minirootfs-3.13.3-x86_64.tar.gz +curl -fsSL -o ./alpine-miniroot.tgz https://dl-cdn.alpinelinux.org/alpine/v3.13/releases/x86_64/alpine-minirootfs-3.13.5-x86_64.tar.gz dd if=/dev/zero of=./rootfs.ext4 bs=1M count=500 mkfs.ext4 ./rootfs.ext4 From 05be382392a553451b932ccb912c11c8a825442c Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 11 Jun 2021 14:26:42 +0200 Subject: [PATCH 088/990] Cleanup: Use Volume dataclass instead of dict --- firecracker/microvm.py | 6 ++---- runtimes/aleph-alpine-3.13-python/init1.py | 22 +++++++++++++++------- vm_supervisor/vm/firecracker_microvm.py | 16 +++++++++++----- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/firecracker/microvm.py b/firecracker/microvm.py index e3404c5cb..187ddea8d 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -235,9 +235,8 @@ async def set_rootfs(self, path_on_host: str): response.raise_for_status() async def mount(self, volume_paths: Dict[str, FilePath]): - counter = 1 - for path, partition_path in volume_paths.items(): - device_name = f"vd{string.ascii_lowercase[counter]}" + for index, (path, partition_path) in enumerate(volume_paths.items()): + device_name = f"vd{string.ascii_lowercase[index + 1]}" if self.use_jailer: partition_filename = Path(partition_path).name jailer_path_on_host = f"/opt/{partition_filename}" @@ -253,7 +252,6 @@ async def mount(self, volume_paths: Dict[str, FilePath]): async with self.get_session() as session: response = await session.put(f"http://localhost/drives/{device_name}", json=data) response.raise_for_status() - counter += 1 async def set_vsock(self): diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 60303484a..c2ef64dd3 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -41,6 +41,12 @@ class Interface(str, Enum): executable = "executable" +@dataclass +class Volume: + mount: str + device: str + + @dataclass class ConfigurationPayload: ip: Optional[str] @@ -52,7 +58,7 @@ class ConfigurationPayload: input_data: bytes interface: Interface vm_hash: str - volumes: Dict[str, str] + volumes: List[Volume] @dataclass @@ -124,11 +130,11 @@ def setup_input_data(input_data: bytes): os.system("unzip -q /opt/input.zip -d /data") -def setup_volumes(volumes: Dict[str, str]): - for path, device in volumes.items(): - logger.debug(f"Mounting /dev/{device} on {path}") - os.makedirs(path, exist_ok=True) - system(f"mount -t squashfs -o ro /dev/{device} {path}") +def setup_volumes(volumes: List[Volume]): + for volume in volumes: + logger.debug(f"Mounting /dev/{volume.device} on {volume.mount}") + os.makedirs(volume.mount, exist_ok=True) + system(f"mount -t squashfs -o ro /dev/{volume.device} {volume.mount}") system("mount") @@ -338,8 +344,10 @@ def main(): data += client.recv(1024*1024) msg_ = msgpack.loads(data, raw=False) - + msg_['volumes'] = [Volume(**volume_dict) + for volume_dict in msg_.get('volumes')] config = ConfigurationPayload(**msg_) + setup_hostname(config.vm_hash) setup_volumes(config.volumes) setup_network(config.ip, config.route, config.dns_servers) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 2fb8cb8ff..8961fb9af 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -50,18 +50,24 @@ class Interface(str, Enum): executable = "executable" +@dataclass +class Volume: + mount: str + device: str + + @dataclass class ConfigurationPayload: ip: Optional[str] route: Optional[str] dns_servers: List[str] code: bytes - encoding: str + encoding: Encoding entrypoint: str input_data: bytes interface: Interface vm_hash: str - volumes: Dict[str, str] + volumes: List[Volume] def as_msgpack(self) -> bytes: return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True) @@ -239,10 +245,10 @@ async def configure(self): else Interface.executable # Start at vdb since vda is already used by the root filesystem - volumes: Dict[str, str] = { - volume.mount: f"vd{string.ascii_lowercase[index+1]}" + volumes: List[Volume] = [ + Volume(mount=volume.mount, device=f"vd{string.ascii_lowercase[index+1]}") for index, volume in enumerate(self.resources.volumes) - } + ] reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path) config = ConfigurationPayload( From 11cc6e2ff2213c383a948dd69c3b41634ae4543e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 14 Jun 2021 12:01:55 +0200 Subject: [PATCH 089/990] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..9bda95264 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main ] + schedule: + - cron: '15 16 * * 0' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 500dfcc7d233def6ad602d4baf678e0d619d3853 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 14 Jun 2021 13:10:35 +0200 Subject: [PATCH 090/990] Fix: Improve script syntax and name container --- docker/run_vm_connector.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docker/run_vm_connector.sh b/docker/run_vm_connector.sh index 500f604e3..5a4811dd5 100755 --- a/docker/run_vm_connector.sh +++ b/docker/run_vm_connector.sh @@ -1,7 +1,11 @@ #!/bin/sh +set -euf + docker build -t aleph-connector -f docker/vm_connector.dockerfile . + docker run -ti --rm -p 8000:8000/tcp \ - -v $(pwd)/kernels:/opt/kernels:ro \ - -v $(pwd)/vm_connector:/opt/vm_connector:ro \ - aleph-connector $@ + -v "$(pwd)/kernels:/opt/kernels:ro" \ + -v "$(pwd)/vm_connector:/opt/vm_connector:ro" \ + --name aleph-connector \ + aleph-connector "$@" From 88179b48b796686373de66c01bfe754806e062fa Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 14 Jun 2021 13:01:41 +0200 Subject: [PATCH 091/990] Fix: VM Connector sign_message crashed --- vm_connector/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_connector/main.py b/vm_connector/main.py index 585f593b2..d16248621 100644 --- a/vm_connector/main.py +++ b/vm_connector/main.py @@ -245,5 +245,5 @@ async def sign_message(request: Request): account: ETHAccount = ETHAccount(private_key=private_key) message = await request.json() - message = account.sign_message(message) + message = await account.sign_message(message) return message From 082071b366cad1811101c57a41023bbbc9bfb998 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 11 Jun 2021 16:12:40 +0200 Subject: [PATCH 092/990] Fix: Add support for Docker in SquashFS build --- examples/volumes/build_squashfs.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/examples/volumes/build_squashfs.sh b/examples/volumes/build_squashfs.sh index 4f662ee8f..a48e133a0 100644 --- a/examples/volumes/build_squashfs.sh +++ b/examples/volumes/build_squashfs.sh @@ -2,5 +2,13 @@ set -euf -podman build -t aleph-vm-build-squashfs . -podman run --rm -ti -v "$( dirname "$0" )":/mnt aleph-vm-build-squashfs +# Use Podman if installed, else use Docker +if hash podman 2> /dev/null +then + DOCKER_COMMAND=podman +else + DOCKER_COMMAND=docker +fi + +$DOCKER_COMMAND build -t aleph-vm-build-squashfs . +$DOCKER_COMMAND run --rm -v "$(pwd)":/mnt aleph-vm-build-squashfs From 9721b356996f52311c4c60fac481f036acadd851 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 14 Jun 2021 13:16:45 +0200 Subject: [PATCH 093/990] Fix: Wrong Python library specified in README --- vm_supervisor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index 752516453..bd30fd335 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -56,7 +56,7 @@ when running the VM Supervisor. ```shell apt update -apt install -y git python3 python3-aiohttp python3-msgpack python3-dnspython redis python3-aioredis sudo acl curl systemd-container +apt install -y git python3 python3-aiohttp python3-msgpack python3-aiodns redis python3-aioredis sudo acl curl systemd-container useradd jailman ``` From 8c2f37391fb9016be9580274ff4f0e66ad3b70af Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 14 Jun 2021 13:17:51 +0200 Subject: [PATCH 094/990] Clean: Log when pip starts in disk image creation --- runtimes/aleph-alpine-3.13-python/create_disk_image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index 5c82dcd7b..4560498c0 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -26,6 +26,7 @@ apk add py3-aiohttp py3-msgpack pip install fastapi apk add git pkgconf gcc py3-wheel python3-dev musl-dev py3-cffi libffi-dev autoconf automake libtool make +echo "Pip installing aleph-client" pip install aleph-client>=0.2.5 coincurve==15.0.0 # Compile all Python bytecode From c91dce0b5aa1ad36cab18db00a4b31ff6b31cfe3 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 11 Jun 2021 15:23:45 +0200 Subject: [PATCH 095/990] Feature: Add self-hosted CI builds These builds run the entrypoints from the example example_fastapi_2 to check that the code is working properly. --- .../workflows/test-integration-fakedata.yml | 48 +++++++++++++++++++ examples/message_from_aleph.json | 6 +-- vm_supervisor/__main__.py | 16 ++++++- 3 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test-integration-fakedata.yml diff --git a/.github/workflows/test-integration-fakedata.yml b/.github/workflows/test-integration-fakedata.yml new file mode 100644 index 000000000..8e4edc1fc --- /dev/null +++ b/.github/workflows/test-integration-fakedata.yml @@ -0,0 +1,48 @@ +name: Run VM Supervisor +on: [push] +jobs: + Run-VM-Supervisor-Fake-Data: + runs-on: self-hosted + timeout-minutes: 5 + env: + ALEPH_VM_FAKE_DATA: true + ALEPH_VM_LINUX_PATH: /opt/vmlinux.bin + + steps: + - name: Check out repository code + uses: actions/checkout@v2 + + - name: Upgrade aleph-message + run: pip3 install --upgrade aleph-message + + - name: Build the example squashfs + run: | + cd examples/volumes + bash build_squashfs.sh + + - name: Build the rootfs + run: | + cd runtimes/aleph-alpine-3.13-python/ + cp /var/tmp/rootfs.ext4 ./ + bash update_inits.sh +# bash ./create_disk_image.sh + + - name: Build VM Connector + run: | + docker build -t aleph-connector -f docker/vm_connector.dockerfile . + + - name: Run the VM Connector + run: | + docker stop aleph-connector || true + docker run -d --rm -p 8000:8000/tcp \ + -v $(pwd)/kernels:/opt/kernels:ro \ + -v $(pwd)/vm_connector:/opt/vm_connector:ro \ + --name aleph-connector \ + aleph-connector $@ + + - name: Run the main entrypoint + run: python3 -m vm_supervisor -p -vv --profile --print-settings --system-logs --benchmark=1 + + - name: Stop the VM Connector + run: | + docker stop aleph-connector diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index 7a26b9958..caf9c035b 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -23,8 +23,8 @@ }, "environment": { "reproducible": true, - "internet": false, - "aleph_api": false + "internet": true, + "aleph_api": true }, "resources": { "vcpus": 1, @@ -56,7 +56,7 @@ "replaces": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", "time": 1619017773.8950517 }, - "item_content": "{\"type\": \"vm-function\", \"address\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"allow_amend\": false, \"code\": {\"encoding\": \"zip\", \"entrypoint\": \"example_fastapi_2:app\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"on\": {\"http\": true}, \"environment\": {\"reproducible\": true, \"internet\": false, \"aleph_api\": false}, \"resources\": {\"vcpus\": 1, \"memory\": 128, \"seconds\": 30}, \"runtime\": {\"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false, \"comment\": \"Aleph Alpine Linux with Python 3.8\"}, \"volumes\": [{\"mount\": \"/opt/venv\", \"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false}], \"data\": {\"encoding\": \"zip\", \"mount\": \"/data\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"export\": {\"encoding\": \"zip\", \"mount\": \"/data\"}, \"replaces\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"time\": 1619017773.8950517}", + "item_content": "{\"type\": \"vm-function\", \"address\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"allow_amend\": false, \"code\": {\"encoding\": \"zip\", \"entrypoint\": \"example_fastapi_2:app\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"on\": {\"http\": true}, \"environment\": {\"reproducible\": true, \"internet\": true, \"aleph_api\": true}, \"resources\": {\"vcpus\": 1, \"memory\": 128, \"seconds\": 30}, \"runtime\": {\"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false, \"comment\": \"Aleph Alpine Linux with Python 3.8\"}, \"volumes\": [{\"mount\": \"/opt/venv\", \"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false}], \"data\": {\"encoding\": \"zip\", \"mount\": \"/data\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"export\": {\"encoding\": \"zip\", \"mount\": \"/data\"}, \"replaces\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"time\": 1619017773.8950517}", "item_type": "inline", "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b", "size": 749, diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index a9fc805dc..506d5b959 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -115,10 +115,24 @@ class FakeRequest: pass bench: List[float] = [] + # First test all methods + settings.REUSE_TIMEOUT = 0.1 + for path in ("/", "/messages", "/internet", "/post_a_message", + "/cache/set/foo/bar", "/cache/get/foo"): + fake_request.match_info["suffix"] = path + response: Response = await supervisor.run_code(message_ref=ref, + path=path, + request=fake_request) + assert response.status == 200 + + # Disable VM timeout to exit benchmark properly + settings.REUSE_TIMEOUT = 0 if runs == 1 else 0.1 + path = "/" for run in range(runs): t0 = time.time() + fake_request.match_info["suffix"] = path response: Response = await supervisor.run_code(message_ref=ref, - path="/", + path=path, request=fake_request) assert response.status == 200 bench.append(time.time() - t0) From ff2025d850cccf6bcb0b3fa843cf452b630cec73 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 15 Jun 2021 12:56:08 +0200 Subject: [PATCH 096/990] WIP: Refactor: Configure Firecracker using JSON file --- firecracker/__init__.py | 3 +- firecracker/config.py | 57 +++++++ firecracker/microvm.py | 207 +++++++++++------------- vm_supervisor/vm/firecracker_microvm.py | 57 ++++--- 4 files changed, 188 insertions(+), 136 deletions(-) create mode 100644 firecracker/config.py diff --git a/firecracker/__init__.py b/firecracker/__init__.py index e213855fe..e24f31fd8 100644 --- a/firecracker/__init__.py +++ b/firecracker/__init__.py @@ -1 +1,2 @@ -from firecracker.microvm import MicroVM +from .microvm import MicroVM +from .config import FirecrackerConfig diff --git a/firecracker/config.py b/firecracker/config.py new file mode 100644 index 000000000..ad9bd97de --- /dev/null +++ b/firecracker/config.py @@ -0,0 +1,57 @@ +from typing import List, Optional + +from pydantic import BaseModel, PositiveInt +from vm_supervisor.models import FilePath + +VSOCK_PATH = "/tmp/v.sock" + + +class BootSource(BaseModel): + kernel_image_path: FilePath = "vmlinux.bin" + boot_args: str = "console=ttyS0 reboot=k panic=1 pci=off " \ + "ro noapic nomodules random.trust_cpu=on" + + @staticmethod + def args(enable_console: bool = True): + default = "reboot=k panic=1 pci=off ro noapic nomodules random.trust_cpu=on" + if enable_console: + return "console=ttyS0 " + default + else: + return default + + +class Drive(BaseModel): + drive_id: str = "rootfs" + path_on_host: FilePath = "./runtimes/aleph-alpine-3.13-python/rootfs.ext4" + is_root_device: bool = True + is_read_only: bool = True + + +class MachineConfig(BaseModel): + vcpu_count: PositiveInt = 1 + mem_size_mib: PositiveInt = 128 + ht_enabled: bool = False + + +class Vsock(BaseModel): + vsock_id: str = "1" + guest_cid: PositiveInt = 3 + uds_path: str = VSOCK_PATH + + +class NetworkInterface(BaseModel): + iface_id: str = "eth0" + guest_mac: str = "AA:FC:00:00:00:01" + host_dev_name: str + + +class FirecrackerConfig(BaseModel): + boot_source: BootSource + drives: List[Drive] + machine_config: MachineConfig + vsock: Optional[Vsock] + network_interfaces: Optional[List[NetworkInterface]] + + class Config: + allow_population_by_field_name = True + alias_generator = lambda x: x.replace('_', '-') diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 187ddea8d..edcb27c53 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -8,12 +8,15 @@ from os import getuid from pathlib import Path from pwd import getpwnam -from typing import Optional, Tuple, Dict +from tempfile import NamedTemporaryFile +from typing import Optional, Tuple, Dict, List import aiohttp from aiohttp import ClientResponse +from firecracker.config import FirecrackerConfig from vm_supervisor.models import FilePath +from .config import Drive logger = logging.getLogger(__name__) @@ -70,6 +73,8 @@ class MicroVM: network_interface: Optional[str] = None stdout_task: Optional[Task] = None stderr_task: Optional[Task] = None + config_file = None + drives: List[Drive] = None @property def jailer_path(self): @@ -109,10 +114,7 @@ def __init__( self.use_jailer = use_jailer self.firecracker_bin_path = firecracker_bin_path self.jailer_bin_path = jailer_bin_path - - def get_session(self) -> aiohttp.ClientSession: - conn = aiohttp.UnixConnector(path=self.socket_path) - return aiohttp.ClientSession(connector=conn) + self.drives = [] def prepare_jailer(self): system(f"rm -fr {self.jailer_path}") @@ -132,35 +134,54 @@ def prepare_jailer(self): # system(f"cp disks/rootfs.ext4 {self.jailer_path}/opt") # system(f"cp hello-vmlinux.bin {self.jailer_path}/opt") - async def start(self) -> asyncio.subprocess.Process: + async def start(self, config: FirecrackerConfig) -> asyncio.subprocess.Process: if self.use_jailer: - return await self.start_jailed_firecracker() + return await self.start_jailed_firecracker(config) else: - return await self.start_firecracker() + return await self.start_firecracker(config) + + async def start_firecracker(self, config: FirecrackerConfig) -> asyncio.subprocess.Process: - async def start_firecracker(self) -> asyncio.subprocess.Process: - logger.debug( - " ".join((self.firecracker_bin_path, "--api-sock", self.socket_path)) - ) if os.path.exists(VSOCK_PATH): os.remove(VSOCK_PATH) if os.path.exists(self.socket_path): os.remove(self.socket_path) + + config_file = NamedTemporaryFile() + config_file.write(config.json(by_alias=True, exclude_none=True, indent=4).encode()) + config_file.flush() + self.config_file = config_file + print(self.config_file) + + logger.debug( + " ".join((self.firecracker_bin_path, "--api-sock", self.socket_path, + "--config-file", config_file.name)) + ) + self.proc = await asyncio.create_subprocess_exec( self.firecracker_bin_path, "--api-sock", self.socket_path, + "--config-file", + config_file.name, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) return self.proc - async def start_jailed_firecracker(self) -> asyncio.subprocess.Process: + async def start_jailed_firecracker(self, config: FirecrackerConfig) -> asyncio.subprocess.Process: if not self.jailer_bin_path: raise ValueError("Jailer binary path is missing") uid = str(getpwnam("jailman").pw_uid) gid = str(getpwnam("jailman").pw_gid) + + config_file = NamedTemporaryFile(dir=f"{self.jailer_path}/tmp/", suffix='.json') + config_file.write(config.json(by_alias=True, exclude_none=True, indent=4).encode()) + config_file.flush() + os.chmod(config_file.name, 0o644) + self.config_file = config_file + logger.debug( " ".join( ( @@ -173,9 +194,13 @@ async def start_jailed_firecracker(self) -> asyncio.subprocess.Process: uid, "--gid", gid, + "--", + "--config-file", + "/tmp/" + os.path.basename(config_file.name), ) ) ) + self.proc = await asyncio.create_subprocess_exec( self.jailer_bin_path, "--id", @@ -186,138 +211,90 @@ async def start_jailed_firecracker(self) -> asyncio.subprocess.Process: uid, "--gid", gid, + "--", + "--config-file", + "/tmp/" + os.path.basename(config_file.name), stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) return self.proc - async def socket_is_ready(self, delay=0.01): - while not os.path.exists(self.socket_path): - await asyncio.sleep(delay) + def enable_kernel(self, kernel_image_path: str) -> str: + """Make a kernel available to the VM. - async def set_boot_source( - self, kernel_image_path: str, enable_console: bool = False - ): + Creates a symlink to the kernel file if jailer is in use. + """ if self.use_jailer: kernel_filename = Path(kernel_image_path).name jailer_kernel_image_path = f"/opt/{kernel_filename}" os.link(kernel_image_path, f"{self.jailer_path}{jailer_kernel_image_path}") kernel_image_path = jailer_kernel_image_path + return kernel_image_path - console = "console=ttyS0" if enable_console else "" - data = { - "kernel_image_path": kernel_image_path, - # Add console=ttyS0 for debugging, but it makes the boot twice slower - "boot_args": f"{console} reboot=k panic=1 pci=off ro noapic nomodules random.trust_cpu=on", - } - async with self.get_session() as session: - response: ClientResponse = await session.put( - "http://localhost/boot-source", json=data - ) - response.raise_for_status() + def enable_rootfs(self, path_on_host: str) -> str: + """Make a rootfs available to the VM. - async def set_rootfs(self, path_on_host: str): + Creates a symlink to the rootfs file if jailer is in use. + """ if self.use_jailer: rootfs_filename = Path(path_on_host).name jailer_path_on_host = f"/opt/{rootfs_filename}" os.link(path_on_host, f"{self.jailer_path}/{jailer_path_on_host}") - path_on_host = jailer_path_on_host - - data = { - "drive_id": "rootfs", - "path_on_host": path_on_host, - "is_root_device": True, - "is_read_only": True, - } - async with self.get_session() as session: - response = await session.put("http://localhost/drives/rootfs", json=data) - response.raise_for_status() - - async def mount(self, volume_paths: Dict[str, FilePath]): - for index, (path, partition_path) in enumerate(volume_paths.items()): - device_name = f"vd{string.ascii_lowercase[index + 1]}" - if self.use_jailer: - partition_filename = Path(partition_path).name - jailer_path_on_host = f"/opt/{partition_filename}" - os.link(partition_path, f"{self.jailer_path}/{jailer_path_on_host}") - partition_path = jailer_path_on_host - - data = { - "drive_id": device_name, - "path_on_host": partition_path, - "is_root_device": False, - "is_read_only": True, - } - async with self.get_session() as session: - response = await session.put(f"http://localhost/drives/{device_name}", json=data) - response.raise_for_status() - - - async def set_vsock(self): - data = { - "vsock_id": "1", - "guest_cid": 3, - "uds_path": VSOCK_PATH, - } - async with self.get_session() as session: - response = await session.put("http://localhost/vsock", json=data) - response.raise_for_status() - - async def set_network(self, interface: str = "eth0"): - """Configure the host network with a tap interface to the VM.""" - logger.debug("Network setup") + return jailer_path_on_host + else: + return path_on_host + + def compute_device_name(self, index: int) -> str: + return f"vd{string.ascii_lowercase[index + 1]}" + + def enable_drive(self, drive_path: str) -> Drive: + """Make a volume available to the VM. + + Creates a symlink to the volume file if jailer is in use. + """ + index = len(self.drives) + device_name = self.compute_device_name(index) + if self.use_jailer: + drive_filename = Path(drive_path).name + jailer_path_on_host = f"/opt/{drive_filename}" + os.link(drive_path, f"{self.jailer_path}/{jailer_path_on_host}") + drive_path = jailer_path_on_host + + drive = Drive( + drive_id=device_name, + path_on_host=FilePath(drive_path), + is_root_device=False, + is_read_only=True, + ) + self.drives.append(drive) + return drive + + async def create_network_interface(self, interface: str = "eth0") -> str: + logger.debug("Create network interface") + + assert self.network_interface is None # Only one is supported at the moment + assert self.network_tap is None self.network_interface = interface - name = f"vmtap{self.vm_id}" - self.network_tap = name + host_dev_name = f"vmtap{self.vm_id}" + self.network_tap = host_dev_name - system(f"ip tuntap add {name} mode tap") + system(f"ip tuntap add {host_dev_name} mode tap") system( - f"ip addr add {self.host_ip}/24 dev {name}" + f"ip addr add {self.host_ip}/24 dev {host_dev_name}" ) - system(f"ip link set {name} up") + system(f"ip link set {host_dev_name} up") system('sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"') # TODO: Don't fill iptables with duplicate rules; purge rules on delete system(f"iptables -t nat -A POSTROUTING -o {interface} -j MASQUERADE") system( "iptables -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT" ) - system(f"iptables -A FORWARD -i {name} -o {interface} -j ACCEPT") - - data = { - "iface_id": "eth0", - "guest_mac": f"AA:FC:00:00:00:01", - "host_dev_name": name, - } - async with self.get_session() as session: - response = await session.put( - "http://localhost/network-interfaces/eth0", json=data - ) - logger.debug(response) - logger.debug(await response.text()) - response.raise_for_status() - - async def set_resources(self, vcpus: int = 1, memory: int = 128, - ht_enabled: bool = False): - """Set machine resources (number of CPU cores, memory)""" - data = { - "vcpu_count": vcpus, - "mem_size_mib": memory, - "ht_enabled": ht_enabled, - } - async with self.get_session() as session: - response = await session.put("http://localhost/machine-config", json=data) - response.raise_for_status() - - async def start_instance(self): - data = { - "action_type": "InstanceStart", - } - async with self.get_session() as session: - response = await session.put("http://localhost/actions", json=data) - response.raise_for_status() + system(f"iptables -A FORWARD -i {host_dev_name} -o {interface} -j ACCEPT") + + return host_dev_name async def print_logs(self): while not self.proc: diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 8961fb9af..3ce29941a 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -1,7 +1,6 @@ import asyncio import dataclasses import logging -import string from dataclasses import dataclass from enum import Enum from multiprocessing import Process, set_start_method @@ -14,6 +13,8 @@ from aleph_message.models import ProgramContent from aleph_message.models.program import MachineResources, MachineVolume +from firecracker.config import BootSource, Drive, MachineConfig, FirecrackerConfig, Vsock, \ + NetworkInterface from firecracker.microvm import MicroVM, setfacl, Encoding from guest_api.__main__ import run_guest_api from ..conf import settings @@ -198,21 +199,40 @@ async def setup(self): jailer_bin_path=settings.JAILER_PATH, ) fvm.prepare_jailer() - await fvm.start() + + config = FirecrackerConfig( + boot_source=BootSource( + kernel_image_path=FilePath(fvm.enable_kernel(self.resources.kernel_image_path)), + boot_args=BootSource.args(enable_console=self.enable_console), + ), + drives=[ + Drive( + drive_id="rootfs", + path_on_host=FilePath(fvm.enable_rootfs(self.resources.rootfs_path)), + is_root_device=True, + is_read_only=True, + ), + ] + [ + fvm.enable_drive(volume) + for volume in self.resources.volume_paths.values() + ], + machine_config=MachineConfig( + vcpu_count=self.hardware_resources.vcpus, + mem_size_mib=self.hardware_resources.memory, + ), + vsock=Vsock(), + network_interfaces = [ + NetworkInterface( + iface_id="eth0", + host_dev_name=await fvm.create_network_interface(interface="eth0"), + ) + ] if self.enable_networking else [], + ) + + logger.debug(config.json(by_alias=True, exclude_none=True, indent=4)) + try: - await fvm.socket_is_ready() - await fvm.set_boot_source( - self.resources.kernel_image_path, - enable_console=self.enable_console, - ) - await fvm.set_rootfs(self.resources.rootfs_path) - await fvm.mount(self.resources.volume_paths) - - await fvm.set_vsock() - await fvm.set_resources(vcpus=self.hardware_resources.vcpus, - memory=self.hardware_resources.memory) - if self.enable_networking: - await fvm.set_network(interface=settings.NETWORK_INTERFACE) + await fvm.start(config) logger.debug("setup done") self.fvm = fvm except Exception: @@ -229,10 +249,7 @@ async def start(self): if self.enable_console: fvm.start_printing_logs() - await asyncio.gather( - fvm.start_instance(), - fvm.wait_for_init(), - ) + await fvm.wait_for_init() logger.debug(f"started fvm {self.vm_id}") async def configure(self): @@ -246,7 +263,7 @@ async def configure(self): # Start at vdb since vda is already used by the root filesystem volumes: List[Volume] = [ - Volume(mount=volume.mount, device=f"vd{string.ascii_lowercase[index+1]}") + Volume(mount=volume.mount, device=self.fvm.drives[index].drive_id) for index, volume in enumerate(self.resources.volumes) ] From 9b1724566661f2fee09a5794e3cbad2c9e36c91f Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 15 Jun 2021 12:51:07 +0200 Subject: [PATCH 097/990] Fix: VM Supervisor Dockerfile missed dependencies --- docker/vm_supervisor.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile index 4e3e5833a..9b3495c6e 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor.dockerfile @@ -4,8 +4,8 @@ FROM debian:buster RUN apt-get update && apt-get -y upgrade && apt-get install -y \ sudo acl curl systemd-container \ - python3 python3-aiohttp python3-msgpack python3-pip \ - && rm -rf /var/lib/apt/lists/* + python3 python3-aiohttp python3-msgpack python3-pip python3-aiodns python3-aioredis \ + && rm -rf /var/lib/apt/lists/* RUN useradd jailman From d3c8ac8dd2c2f9c7789b88b936cd1c81a9c30220 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 15 Jun 2021 15:57:50 +0200 Subject: [PATCH 098/990] Feature: Add support for code on Squashfs --- docker/vm_supervisor.dockerfile | 3 ++- examples/message_from_aleph.json | 6 ++--- firecracker/microvm.py | 11 +-------- runtimes/aleph-alpine-3.13-python/init1.py | 17 +++++++++++-- vm_connector/main.py | 4 ---- vm_supervisor/README.md | 3 ++- vm_supervisor/storage.py | 19 +++++++++++---- vm_supervisor/vm/firecracker_microvm.py | 28 +++++++++++++++------- 8 files changed, 57 insertions(+), 34 deletions(-) diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile index 9b3495c6e..597e303f1 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor.dockerfile @@ -5,6 +5,7 @@ FROM debian:buster RUN apt-get update && apt-get -y upgrade && apt-get install -y \ sudo acl curl systemd-container \ python3 python3-aiohttp python3-msgpack python3-pip python3-aiodns python3-aioredis \ + squashfs-tools \ && rm -rf /var/lib/apt/lists/* RUN useradd jailman @@ -17,7 +18,7 @@ RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/downl RUN ln /opt/firecracker/firecracker-v* /opt/firecracker/firecracker RUN ln /opt/firecracker/jailer-v* /opt/firecracker/jailer -RUN pip3 install typing-extensions aleph-message pydantic +RUN pip3 install typing-extensions aleph-message>=0.1.8 pydantic RUN mkdir /srv/jailer diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index caf9c035b..47e36a784 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -13,8 +13,8 @@ "address": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", "allow_amend": false, "code": { - "encoding": "zip", - "entrypoint": "example_fastapi_2:app", + "encoding": "squashfs", + "entrypoint": "__init__:app", "ref": "7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003", "use_latest": false }, @@ -56,7 +56,7 @@ "replaces": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", "time": 1619017773.8950517 }, - "item_content": "{\"type\": \"vm-function\", \"address\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"allow_amend\": false, \"code\": {\"encoding\": \"zip\", \"entrypoint\": \"example_fastapi_2:app\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"on\": {\"http\": true}, \"environment\": {\"reproducible\": true, \"internet\": true, \"aleph_api\": true}, \"resources\": {\"vcpus\": 1, \"memory\": 128, \"seconds\": 30}, \"runtime\": {\"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false, \"comment\": \"Aleph Alpine Linux with Python 3.8\"}, \"volumes\": [{\"mount\": \"/opt/venv\", \"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false}], \"data\": {\"encoding\": \"zip\", \"mount\": \"/data\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"export\": {\"encoding\": \"zip\", \"mount\": \"/data\"}, \"replaces\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"time\": 1619017773.8950517}", + "item_content": "{\"type\": \"vm-function\", \"address\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"allow_amend\": false, \"code\": {\"encoding\": \"zip\", \"entrypoint\": \"__init__:app\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"on\": {\"http\": true}, \"environment\": {\"reproducible\": true, \"internet\": true, \"aleph_api\": true}, \"resources\": {\"vcpus\": 1, \"memory\": 128, \"seconds\": 30}, \"runtime\": {\"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false, \"comment\": \"Aleph Alpine Linux with Python 3.8\"}, \"volumes\": [{\"mount\": \"/opt/venv\", \"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false}], \"data\": {\"encoding\": \"zip\", \"mount\": \"/data\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"export\": {\"encoding\": \"zip\", \"mount\": \"/data\"}, \"replaces\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"time\": 1619017773.8950517}", "item_type": "inline", "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b", "size": 749, diff --git a/firecracker/microvm.py b/firecracker/microvm.py index edcb27c53..7f5179215 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -9,10 +9,7 @@ from pathlib import Path from pwd import getpwnam from tempfile import NamedTemporaryFile -from typing import Optional, Tuple, Dict, List - -import aiohttp -from aiohttp import ClientResponse +from typing import Optional, Tuple, List from firecracker.config import FirecrackerConfig from vm_supervisor.models import FilePath @@ -20,12 +17,6 @@ logger = logging.getLogger(__name__) - -class Encoding(str, Enum): - plain = "plain" - zip = "zip" - - VSOCK_PATH = "/tmp/v.sock" diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index c2ef64dd3..befa25590 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -34,6 +34,7 @@ class Encoding(str, Enum): plain = "plain" zip = "zip" + squashfs = "squashfs" class Interface(str, Enum): @@ -140,7 +141,13 @@ def setup_volumes(volumes: List[Volume]): def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplication: logger.debug("Extracting code") - if encoding == Encoding.zip: + if encoding == Encoding.squashfs: + sys.path.append("/opt/code") + module_name, app_name = entrypoint.split(":", 1) + logger.debug("import module") + module = __import__(module_name) + app: ASGIApplication = getattr(module, app_name) + elif encoding == Encoding.zip: # Unzip in /opt and import the entrypoint from there if not os.path.exists("/opt/archive.zip"): open("/opt/archive.zip", "wb").write(code) @@ -163,7 +170,13 @@ def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApp def setup_code_executable(code: bytes, encoding: Encoding, entrypoint: str) -> subprocess.Popen: logger.debug("Extracting code") - if encoding == Encoding.zip: + if encoding == Encoding.squashfs: + path = f"/opt/code/{entrypoint}" + if not os.path.isfile(path): + os.system("find /opt/code/") + raise FileNotFoundError(f"No such file: {path}") + os.system(f"chmod +x {path}") + elif encoding == Encoding.zip: open("/opt/archive.zip", "wb").write(code) logger.debug("Run unzip") os.system("unzip /opt/archive.zip -d /opt") diff --git a/vm_connector/main.py b/vm_connector/main.py index d16248621..e7bb468e0 100644 --- a/vm_connector/main.py +++ b/vm_connector/main.py @@ -27,10 +27,6 @@ def read_root(): return {"Server": "Aleph.im VM Connector"} -class Encoding: - plain = "plain" - zip = "zip" - async def get_latest_message_amend(ref: str, sender: str) -> Optional[Dict]: async with aiohttp.ClientSession() as session: diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index bd30fd335..7498831af 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -56,7 +56,8 @@ when running the VM Supervisor. ```shell apt update -apt install -y git python3 python3-aiohttp python3-msgpack python3-aiodns redis python3-aioredis sudo acl curl systemd-container +apt install -y git python3 python3-aiohttp python3-msgpack python3-aiodns redis python3-aioredis \ + sudo acl curl systemd-container squashfs-tools useradd jailman ``` diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index ebc481db2..1a8a1c54d 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -14,6 +14,7 @@ import aiohttp from aleph_message.models import ProgramMessage +from aleph_message.models.program import Encoding from .conf import settings from .models import FilePath @@ -78,10 +79,20 @@ async def get_code_path(ref: str) -> FilePath: if settings.FAKE_DATA: root_dir = abspath(join(__file__, "../../examples/")) archive_path = join(root_dir, settings.FAKE_DATA_EXAMPLE) - make_archive( - archive_path, "zip", root_dir=root_dir, base_dir=settings.FAKE_DATA_EXAMPLE - ) - return FilePath(f"{archive_path}.zip") + + encoding: Encoding = (await get_message(ref="fake-message")).content.code.encoding + if encoding == Encoding.squashfs: + if os.path.exists(f"{archive_path}.squashfs"): + os.remove(f"{archive_path}.squashfs") + os.system(f"mksquashfs {archive_path} {archive_path}.squashfs") + return FilePath(f"{archive_path}.squashfs") + elif encoding == Encoding.zip: + make_archive( + archive_path, "zip", root_dir=archive_path) + return FilePath(f"{archive_path}.zip") + else: + raise ValueError(f"Unsupported encoding: {encoding}") + cache_path = FilePath(join(settings.CODE_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/code/{ref}" diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 3ce29941a..510f75664 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -12,10 +12,10 @@ from aiohttp import ClientResponseError from aleph_message.models import ProgramContent -from aleph_message.models.program import MachineResources, MachineVolume +from aleph_message.models.program import MachineResources, MachineVolume, Encoding from firecracker.config import BootSource, Drive, MachineConfig, FirecrackerConfig, Vsock, \ NetworkInterface -from firecracker.microvm import MicroVM, setfacl, Encoding +from firecracker.microvm import MicroVM, setfacl from guest_api.__main__ import run_guest_api from ..conf import settings from ..models import FilePath @@ -212,7 +212,10 @@ async def setup(self): is_root_device=True, is_read_only=True, ), - ] + [ + ] + ( + [fvm.enable_drive(self.resources.code_path)] + if self.resources.code_encoding == Encoding.squashfs else [] + ) + [ fvm.enable_drive(volume) for volume in self.resources.volume_paths.values() ], @@ -255,17 +258,24 @@ async def start(self): async def configure(self): """Configure the VM by sending configuration info to it's init""" - code: bytes = load_file_content(self.resources.code_path) input_data: bytes = load_file_content(self.resources.data_path) interface = Interface.asgi if ":" in self.resources.code_entrypoint \ else Interface.executable - # Start at vdb since vda is already used by the root filesystem - volumes: List[Volume] = [ - Volume(mount=volume.mount, device=self.fvm.drives[index].drive_id) - for index, volume in enumerate(self.resources.volumes) - ] + volumes: List[Volume] + if self.resources.code_encoding == Encoding.squashfs: + code = b'' + volumes = [Volume(mount="/opt/code", device="vdb")] + [ + Volume(mount=volume.mount, device=self.fvm.drives[index+1].drive_id) + for index, volume in enumerate(self.resources.volumes) + ] + else: + code: bytes = load_file_content(self.resources.code_path) + volumes = [ + Volume(mount=volume.mount, device=self.fvm.drives[index].drive_id) + for index, volume in enumerate(self.resources.volumes) + ] reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path) config = ConfigurationPayload( From 039cf2aeb26374c83017b839283932a0cd763b6b Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 16 Jun 2021 10:57:59 +0200 Subject: [PATCH 099/990] Feature: Allow amend for code, data and volumes Closes #43 --- vm_supervisor/supervisor.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index d646ff419..fb6515a12 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -5,6 +5,7 @@ At it's core, it is currently an asynchronous HTTP server using aiohttp, but this may evolve in the future. """ +import asyncio import binascii import logging from base64 import b32decode, b16encode @@ -57,6 +58,18 @@ async def get_latest_ref(item_hash: str) -> str: raise +async def update_with_latest_ref(obj): + """ + Update the reference `ref` inplace if a newer version is available. + + Useful to update references in parallel with asyncio.gather. + """ + if obj is None: + return obj + if obj.use_latest: + obj.ref = await get_latest_ref(obj.ref) + + async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: return { "type": "http", @@ -77,8 +90,15 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res message_content: ProgramContent = message.content # Load amends - if message_content.runtime.use_latest: - message_content.runtime.ref = await get_latest_ref(message_content.runtime.ref) + await asyncio.gather( + update_with_latest_ref(message_content.runtime), + update_with_latest_ref(message_content.code), + update_with_latest_ref(message_content.data), + *( + update_with_latest_ref(volume) + for volume in (message_content.volumes or []) + ), + ) # TODO: Cache message content after amends From cd974f7047bcaf05da9be70d42814aee457ded31 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 16 Jun 2021 11:10:26 +0200 Subject: [PATCH 100/990] Fix: Add shared_cache in message example --- examples/message_from_aleph.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index 47e36a784..225ab5ce5 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -24,7 +24,8 @@ "environment": { "reproducible": true, "internet": true, - "aleph_api": true + "aleph_api": true, + "shared_cache": false }, "resources": { "vcpus": 1, @@ -56,7 +57,7 @@ "replaces": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", "time": 1619017773.8950517 }, - "item_content": "{\"type\": \"vm-function\", \"address\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"allow_amend\": false, \"code\": {\"encoding\": \"zip\", \"entrypoint\": \"__init__:app\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"on\": {\"http\": true}, \"environment\": {\"reproducible\": true, \"internet\": true, \"aleph_api\": true}, \"resources\": {\"vcpus\": 1, \"memory\": 128, \"seconds\": 30}, \"runtime\": {\"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false, \"comment\": \"Aleph Alpine Linux with Python 3.8\"}, \"volumes\": [{\"mount\": \"/opt/venv\", \"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false}], \"data\": {\"encoding\": \"zip\", \"mount\": \"/data\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"export\": {\"encoding\": \"zip\", \"mount\": \"/data\"}, \"replaces\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"time\": 1619017773.8950517}", + "item_content": "{\"type\": \"vm-function\", \"address\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"allow_amend\": false, \"code\": {\"encoding\": \"zip\", \"entrypoint\": \"__init__:app\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"on\": {\"http\": true}, \"environment\": {\"reproducible\": true, \"internet\": true, \"aleph_api\": true, \"shared_cache\": false}, \"resources\": {\"vcpus\": 1, \"memory\": 128, \"seconds\": 30}, \"runtime\": {\"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false, \"comment\": \"Aleph Alpine Linux with Python 3.8\"}, \"volumes\": [{\"mount\": \"/opt/venv\", \"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false}], \"data\": {\"encoding\": \"zip\", \"mount\": \"/data\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"export\": {\"encoding\": \"zip\", \"mount\": \"/data\"}, \"replaces\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"time\": 1619017773.8950517}", "item_type": "inline", "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b", "size": 749, From 827d5c2e6d40ea6cc2c2fdfe03dc8e7a1b0fcc0b Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 16 Jun 2021 12:41:18 +0200 Subject: [PATCH 101/990] Feature: Add Debian based runtime --- .../create_disk_image.sh | 81 +++++++++++++++++++ runtimes/aleph-debian-11-python/init0.sh | 1 + runtimes/aleph-debian-11-python/init1.py | 1 + .../aleph-debian-11-python/update_inits.sh | 1 + vm_supervisor/README.md | 2 +- 5 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 runtimes/aleph-debian-11-python/create_disk_image.sh create mode 120000 runtimes/aleph-debian-11-python/init0.sh create mode 120000 runtimes/aleph-debian-11-python/init1.py create mode 120000 runtimes/aleph-debian-11-python/update_inits.sh diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh new file mode 100644 index 000000000..b87ee0594 --- /dev/null +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -0,0 +1,81 @@ +#!/bin/sh + +umount /mnt/rootfs + +set -euf + +dd if=/dev/zero of=./rootfs.ext4 bs=1M count=1000 +mkfs.ext4 ./rootfs.ext4 +mkdir -p /mnt/rootfs +mount ./rootfs.ext4 /mnt/rootfs + +debootstrap --variant=minbase bullseye /mnt/rootfs http://deb.debian.org/debian/ + +chroot /mnt/rootfs /bin/sh <=0.2.5' 'coincurve==15.0.0' + +# Compile all Python bytecode +python3 -m compileall -f /usr/local/lib/python3.9 + +#echo -e "toor\ntoor" | passwd root + +mkdir -p /overlay + +# Set up a login terminal on the serial console (ttyS0): +ln -s agetty /etc/init.d/agetty.ttyS0 +echo ttyS0 > /etc/securetty +EOT + +echo "PermitRootLogin yes" >> /mnt/rootfs/etc/ssh/sshd_config + +# Generate SSH host keys +#systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key +#systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key +#systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key +#systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t ed25519 -f /etc/ssh/ssh_host_ed25519_key + +cat < /mnt/rootfs/etc/inittab +# /etc/inittab + +::sysinit:/sbin/init sysinit +::sysinit:/sbin/init boot +::wait:/sbin/init default + +# Set up a couple of getty's +tty1::respawn:/sbin/getty 38400 tty1 +tty2::respawn:/sbin/getty 38400 tty2 +tty3::respawn:/sbin/getty 38400 tty3 +tty4::respawn:/sbin/getty 38400 tty4 +tty5::respawn:/sbin/getty 38400 tty5 +tty6::respawn:/sbin/getty 38400 tty6 + +# Put a getty on the serial port +ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100 + +# Stuff to do for the 3-finger salute +::ctrlaltdel:/sbin/reboot + +# Stuff to do before rebooting +::shutdown:/sbin/init shutdown +EOT + +# Custom init +cp ./init0.sh /mnt/rootfs/sbin/init +cp ./init1.py /mnt/rootfs/root/init1.py +chmod +x /mnt/rootfs/sbin/init +chmod +x /mnt/rootfs/root/init1.py + +umount /mnt/rootfs diff --git a/runtimes/aleph-debian-11-python/init0.sh b/runtimes/aleph-debian-11-python/init0.sh new file mode 120000 index 000000000..4315744b7 --- /dev/null +++ b/runtimes/aleph-debian-11-python/init0.sh @@ -0,0 +1 @@ +../aleph-alpine-3.13-python/init0.sh \ No newline at end of file diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py new file mode 120000 index 000000000..529895e71 --- /dev/null +++ b/runtimes/aleph-debian-11-python/init1.py @@ -0,0 +1 @@ +../aleph-alpine-3.13-python/init1.py \ No newline at end of file diff --git a/runtimes/aleph-debian-11-python/update_inits.sh b/runtimes/aleph-debian-11-python/update_inits.sh new file mode 120000 index 000000000..8bd2ec9e1 --- /dev/null +++ b/runtimes/aleph-debian-11-python/update_inits.sh @@ -0,0 +1 @@ +../aleph-alpine-3.13-python/update_inits.sh \ No newline at end of file diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index 7498831af..ce7a5e542 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -57,7 +57,7 @@ when running the VM Supervisor. ```shell apt update apt install -y git python3 python3-aiohttp python3-msgpack python3-aiodns redis python3-aioredis \ - sudo acl curl systemd-container squashfs-tools + sudo acl curl systemd-container squashfs-tools debootstrap useradd jailman ``` From 3e84d602d95d11b6dda0fb38595ab7c33439d504 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 16 Jun 2021 13:02:25 +0200 Subject: [PATCH 102/990] Feature: Switch rootfs to Squashfs --- .../workflows/test-integration-fakedata.yml | 6 ++-- docker/run_vm_supervisor.sh | 4 +-- docker/vm_supervisor.dockerfile | 2 +- .../create_disk_image.sh | 34 ++++++++----------- .../aleph-debian-11-python/update_inits.sh | 15 +++++++- vm_supervisor/storage.py | 2 +- 6 files changed, 36 insertions(+), 27 deletions(-) mode change 120000 => 100644 runtimes/aleph-debian-11-python/update_inits.sh diff --git a/.github/workflows/test-integration-fakedata.yml b/.github/workflows/test-integration-fakedata.yml index 8e4edc1fc..e4be242a2 100644 --- a/.github/workflows/test-integration-fakedata.yml +++ b/.github/workflows/test-integration-fakedata.yml @@ -20,10 +20,10 @@ jobs: cd examples/volumes bash build_squashfs.sh - - name: Build the rootfs + - name: Update the rootfs run: | - cd runtimes/aleph-alpine-3.13-python/ - cp /var/tmp/rootfs.ext4 ./ + cd runtimes/aleph-debian-11-python/ + cp -pr /var/tmp/rootfs-debian ./rootfs bash update_inits.sh # bash ./create_disk_image.sh diff --git a/docker/run_vm_supervisor.sh b/docker/run_vm_supervisor.sh index 015d5f476..a7c7a69cb 100755 --- a/docker/run_vm_supervisor.sh +++ b/docker/run_vm_supervisor.sh @@ -1,7 +1,7 @@ #!/bin/sh -docker build -ti -t aleph-vm-supervisor -f docker/vm_supervisor.dockerfile . -docker run -ti --rm \ +podman build -ti -t aleph-vm-supervisor -f docker/vm_supervisor.dockerfile . +podman run -ti --rm \ -v $(pwd):/root/aleph-vm \ --device /dev/kvm \ aleph-vm-supervisor \ diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile index 597e303f1..0cef72d33 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor.dockerfile @@ -18,7 +18,7 @@ RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/downl RUN ln /opt/firecracker/firecracker-v* /opt/firecracker/firecracker RUN ln /opt/firecracker/jailer-v* /opt/firecracker/jailer -RUN pip3 install typing-extensions aleph-message>=0.1.8 pydantic +RUN pip3 install typing-extensions aleph-message>=0.1.10 pydantic RUN mkdir /srv/jailer diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index b87ee0594..6be8b66af 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -1,17 +1,13 @@ #!/bin/sh -umount /mnt/rootfs - set -euf -dd if=/dev/zero of=./rootfs.ext4 bs=1M count=1000 -mkfs.ext4 ./rootfs.ext4 -mkdir -p /mnt/rootfs -mount ./rootfs.ext4 /mnt/rootfs +rm -fr ./rootfs +mkdir ./rootfs -debootstrap --variant=minbase bullseye /mnt/rootfs http://deb.debian.org/debian/ +debootstrap --variant=minbase bullseye ./rootfs http://deb.debian.org/debian/ -chroot /mnt/rootfs /bin/sh < /etc/securetty EOT -echo "PermitRootLogin yes" >> /mnt/rootfs/etc/ssh/sshd_config +echo "PermitRootLogin yes" >> ./rootfs/etc/ssh/sshd_config # Generate SSH host keys -#systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key -#systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key -#systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -#systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t ed25519 -f /etc/ssh/ssh_host_ed25519_key +#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key +#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key +#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key +#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -cat < /mnt/rootfs/etc/inittab +cat < ./rootfs/etc/inittab # /etc/inittab ::sysinit:/sbin/init sysinit @@ -73,9 +69,9 @@ ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100 EOT # Custom init -cp ./init0.sh /mnt/rootfs/sbin/init -cp ./init1.py /mnt/rootfs/root/init1.py -chmod +x /mnt/rootfs/sbin/init -chmod +x /mnt/rootfs/root/init1.py +cp ./init0.sh ./rootfs/sbin/init +cp ./init1.py ./rootfs/root/init1.py +chmod +x ./rootfs/sbin/init +chmod +x ./rootfs/root/init1.py -umount /mnt/rootfs +mksquashfs ./rootfs/ ./rootfs.squashfs diff --git a/runtimes/aleph-debian-11-python/update_inits.sh b/runtimes/aleph-debian-11-python/update_inits.sh deleted file mode 120000 index 8bd2ec9e1..000000000 --- a/runtimes/aleph-debian-11-python/update_inits.sh +++ /dev/null @@ -1 +0,0 @@ -../aleph-alpine-3.13-python/update_inits.sh \ No newline at end of file diff --git a/runtimes/aleph-debian-11-python/update_inits.sh b/runtimes/aleph-debian-11-python/update_inits.sh new file mode 100644 index 000000000..55a1c99b1 --- /dev/null +++ b/runtimes/aleph-debian-11-python/update_inits.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +rm ./rootfs.squashfs + +set -euf + +cp ./init0.sh ./rootfs/sbin/init +cp ./init1.py ./rootfs/root/init1.py +chmod +x ./rootfs/sbin/init +chmod +x ./rootfs/root/init1.py + +mksquashfs ./rootfs/ ./rootfs.squashfs + +echo "OK" diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index 1a8a1c54d..49a8f0042 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -116,7 +116,7 @@ async def get_runtime_path(ref: str) -> FilePath: if settings.FAKE_DATA: return FilePath( os.path.abspath( - join(__file__, "../../runtimes/aleph-alpine-3.13-python/rootfs.ext4") + join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs") ) ) From 86538615ab766c84efa92f0396c5d6a8cbcd1a35 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 16 Jun 2021 14:18:39 +0200 Subject: [PATCH 103/990] Feature: Optimize Debian squashfs filesize --- runtimes/aleph-debian-11-python/create_disk_image.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index 6be8b66af..adaefa6a0 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -1,5 +1,7 @@ #!/bin/sh +rm ./rootfs.squashfs + set -euf rm -fr ./rootfs @@ -15,7 +17,7 @@ apt-get install -y --no-install-recommends --no-install-suggests \ \ python3-aiohttp python3-msgpack \ python3-setuptools \ - python3-pip python3-cytoolz \ + python3-pip python3-cytoolz python3-pydantic \ iproute2 unzip pip3 install fastapi @@ -68,6 +70,13 @@ ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100 ::shutdown:/sbin/init shutdown EOT +# Reduce size +rm -fr ./rootfs/root/.cache +rm -fr ./rootfs/var/cache +rm -fr ./rootfs/usr/share/doc +rm -fr ./rootfs/usr/share/man +rm -fr ./rootfs/var/lib/apt/lists/ + # Custom init cp ./init0.sh ./rootfs/sbin/init cp ./init1.py ./rootfs/root/init1.py From 03a574820616a730b39093b091bf390260834df1 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 16 Jun 2021 14:39:39 +0200 Subject: [PATCH 104/990] Feature: Add NodeJS in new default runtime --- runtimes/aleph-debian-11-python/create_disk_image.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index adaefa6a0..6d81effff 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -18,7 +18,8 @@ apt-get install -y --no-install-recommends --no-install-suggests \ python3-aiohttp python3-msgpack \ python3-setuptools \ python3-pip python3-cytoolz python3-pydantic \ - iproute2 unzip + iproute2 unzip \ + nodejs pip3 install fastapi From 8f3f760b8a0054a2b457f19f9014befcecf5ec22 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 16 Jun 2021 15:49:32 +0200 Subject: [PATCH 105/990] Fix: A VM corouting kept hanging when the rootfs was invalid Fixes #30 --- .../workflows/test-integration-fakedata.yml | 1 + firecracker/microvm.py | 37 +++++++++++++++---- vm_supervisor/conf.py | 1 + vm_supervisor/supervisor.py | 4 ++ vm_supervisor/vm/firecracker_microvm.py | 1 + 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-integration-fakedata.yml b/.github/workflows/test-integration-fakedata.yml index e4be242a2..06b11c3a4 100644 --- a/.github/workflows/test-integration-fakedata.yml +++ b/.github/workflows/test-integration-fakedata.yml @@ -7,6 +7,7 @@ jobs: env: ALEPH_VM_FAKE_DATA: true ALEPH_VM_LINUX_PATH: /opt/vmlinux.bin + ALEPH_VM_INIT_TIMEOUT: 20 steps: - name: Check out repository code diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 7f5179215..a9f4be1f6 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -11,7 +11,7 @@ from tempfile import NamedTemporaryFile from typing import Optional, Tuple, List -from firecracker.config import FirecrackerConfig +from .config import FirecrackerConfig from vm_supervisor.models import FilePath from .config import Drive @@ -20,6 +20,10 @@ VSOCK_PATH = "/tmp/v.sock" +class MicroVMFailedInit(Exception): + pass + + # extend the json.JSONEncoder class to support bytes class JSONBytesEncoder(json.JSONEncoder): @@ -66,6 +70,7 @@ class MicroVM: stderr_task: Optional[Task] = None config_file = None drives: List[Drive] = None + init_timeout: float @property def jailer_path(self): @@ -100,12 +105,14 @@ def __init__( firecracker_bin_path: str, use_jailer: bool = True, jailer_bin_path: Optional[str] = None, + init_timeout: float = 5., ): self.vm_id = vm_id self.use_jailer = use_jailer self.firecracker_bin_path = firecracker_bin_path self.jailer_bin_path = jailer_bin_path self.drives = [] + self.init_timeout = init_timeout def prepare_jailer(self): system(f"rm -fr {self.jailer_path}") @@ -167,7 +174,8 @@ async def start_jailed_firecracker(self, config: FirecrackerConfig) -> asyncio.s uid = str(getpwnam("jailman").pw_uid) gid = str(getpwnam("jailman").pw_gid) - config_file = NamedTemporaryFile(dir=f"{self.jailer_path}/tmp/", suffix='.json') + # config_file = NamedTemporaryFile(dir=f"{self.jailer_path}/tmp/", suffix='.json') + config_file = open(f"{self.jailer_path}/tmp/config.json", 'wb') config_file.write(config.json(by_alias=True, exclude_none=True, indent=4).encode()) config_file.flush() os.chmod(config_file.name, 0o644) @@ -325,13 +333,20 @@ async def unix_client_connected(*_): unix_client_connected, path=f"{self.vsock_path}_52" ) os.system(f"chown jailman:jailman {self.vsock_path}_52") - await queue.get() - logger.debug("...signal from init received") + try: + await asyncio.wait_for(queue.get(), timeout=self.init_timeout) + logger.debug("...signal from init received") + except asyncio.TimeoutError: + logger.warning("Never received signal from init") + raise MicroVMFailedInit() async def stop(self): if self.proc: - self.proc.terminate() - self.proc.kill() + try: + self.proc.terminate() + self.proc.kill() + except ProcessLookupError: + pass self.proc = None async def teardown(self): @@ -361,5 +376,11 @@ async def teardown(self): def __del__(self): - loop = asyncio.get_running_loop() - loop.create_task(self.teardown()) + try: + loop = asyncio.get_running_loop() + loop.create_task(self.teardown()) + except RuntimeError as error: + if error.args == ('no running event loop',): + return + else: + raise diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 66c3fbb70..716c70e6c 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -74,6 +74,7 @@ class Settings(BaseSettings): FIRECRACKER_PATH: str = "/opt/firecracker/firecracker" JAILER_PATH: str = "/opt/firecracker/jailer" LINUX_PATH: str = os.path.abspath("./kernels/vmlinux.bin") + INIT_TIMEOUT: float = 20 CONNECTOR_URL: Url = Url("http://localhost:8000") diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index fb6515a12..52f0e11c9 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -19,6 +19,7 @@ from msgpack import UnpackValueError from aleph_message.models import ProgramMessage, ProgramContent +from firecracker.microvm import MicroVMFailedInit from .conf import settings from .pool import VmPool from .storage import get_message, get_latest_amend @@ -110,6 +111,9 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res except VmSetupError as error: logger.exception(error) raise HTTPInternalServerError(reason="Error during program initialisation") + except MicroVMFailedInit as error: + logger.exception(error) + raise HTTPInternalServerError(reason="Error during runtime initialisation") logger.debug(f"Using vm={vm.vm_id}") diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 510f75664..053201404 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -197,6 +197,7 @@ async def setup(self): firecracker_bin_path=settings.FIRECRACKER_PATH, use_jailer=settings.USE_JAILER, jailer_bin_path=settings.JAILER_PATH, + init_timeout=settings.INIT_TIMEOUT, ) fvm.prepare_jailer() From fb7b71d86c96ea851550b2bb2f7bfbac5dc94c64 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 16 Jun 2021 18:02:15 +0200 Subject: [PATCH 106/990] Feature: Add API to list cache keys --- examples/example_fastapi_2/__init__.py | 5 +++++ guest_api/__main__.py | 18 +++++++++++++++++- .../create_disk_image.sh | 2 +- .../create_disk_image.sh | 2 +- vm_supervisor/__main__.py | 2 +- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/examples/example_fastapi_2/__init__.py b/examples/example_fastapi_2/__init__.py index 7d798e501..bf7750d1a 100644 --- a/examples/example_fastapi_2/__init__.py +++ b/examples/example_fastapi_2/__init__.py @@ -93,3 +93,8 @@ async def remove_from_cache(key: str): """Store data in the VM cache""" result = await cache.delete(key) return result == 1 + +@app.get("/cache/keys") +async def keys_from_cache(pattern: str = '*'): + """List keys from the VM cache""" + return await cache.keys(pattern) diff --git a/guest_api/__main__.py b/guest_api/__main__.py index c77f011c7..2ad39c41e 100644 --- a/guest_api/__main__.py +++ b/guest_api/__main__.py @@ -127,11 +127,25 @@ async def delete_from_cache(request: web.Request): return web.HTTPBadRequest(text="Invalid key") redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost") - logger.debug("DEL", f"{prefix}:{key}") result = await redis.delete(f"{prefix}:{key}") return web.json_response(result) +async def list_keys_from_cache(request: web.Request): + prefix: str = request.app.meta_vm_hash + pattern: str = request.rel_url.query.get('pattern', '*') + if not re.match(r'^[\w?*^\-]+$', pattern): + return web.HTTPBadRequest(text="Invalid key") + + redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost") + result = await redis.keys(f"{prefix}:{pattern}") + keys = [ + key.decode()[len(prefix)+1:] + for key in result + ] + return web.json_response(keys) + + def run_guest_api(unix_socket_path, vm_hash: Optional[str] = None): app = web.Application() app.meta_vm_hash = vm_hash or '_' @@ -139,6 +153,7 @@ def run_guest_api(unix_socket_path, vm_hash: Optional[str] = None): app.router.add_route(method='GET', path='/properties', handler=properties) app.router.add_route(method='POST', path='/sign', handler=sign) + app.router.add_route(method='GET', path='/cache/', handler=list_keys_from_cache) app.router.add_route(method='GET', path='/cache/{key:.*}', handler=get_from_cache) app.router.add_route(method='PUT', path='/cache/{key:.*}', handler=put_in_cache) app.router.add_route(method='DELETE', path='/cache/{key:.*}', handler=delete_from_cache) @@ -150,6 +165,7 @@ def run_guest_api(unix_socket_path, vm_hash: Optional[str] = None): app.router.add_route(method='POST', path='/api/v0/ipfs/pubsub/pub', handler=repost) app.router.add_route(method='POST', path='/api/v0/p2p/pubsub/pub', handler=repost) + # web.run_app(app=app, port=9000) web.run_app(app=app, path=unix_socket_path) diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index 4560498c0..9c3ccb92f 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -27,7 +27,7 @@ pip install fastapi apk add git pkgconf gcc py3-wheel python3-dev musl-dev py3-cffi libffi-dev autoconf automake libtool make echo "Pip installing aleph-client" -pip install aleph-client>=0.2.5 coincurve==15.0.0 +pip install aleph-client>=0.2.7 coincurve==15.0.0 # Compile all Python bytecode python3 -m compileall -f /usr/lib/python3.8/site-packages diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index 6d81effff..d6bc8846a 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -24,7 +24,7 @@ apt-get install -y --no-install-recommends --no-install-suggests \ pip3 install fastapi echo "Pip installing aleph-client" -pip3 install 'aleph-client>=0.2.5' 'coincurve==15.0.0' +pip3 install 'aleph-client>=0.2.7' 'coincurve==15.0.0' # Compile all Python bytecode python3 -m compileall -f /usr/local/lib/python3.9 diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 506d5b959..574918431 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -118,7 +118,7 @@ class FakeRequest: pass # First test all methods settings.REUSE_TIMEOUT = 0.1 for path in ("/", "/messages", "/internet", "/post_a_message", - "/cache/set/foo/bar", "/cache/get/foo"): + "/cache/set/foo/bar", "/cache/get/foo", "/cache/keys"): fake_request.match_info["suffix"] = path response: Response = await supervisor.run_code(message_ref=ref, path=path, From 204d199d6b0459a60205c12ccc7ec9c455777d2c Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 18 Jun 2021 15:40:26 +0200 Subject: [PATCH 107/990] Fix: Apt failed in VM due to cache directory removed --- runtimes/aleph-debian-11-python/create_disk_image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index d6bc8846a..ec242c2b6 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -74,6 +74,7 @@ EOT # Reduce size rm -fr ./rootfs/root/.cache rm -fr ./rootfs/var/cache +mkdir -p ./rootfs/var/cache/apt/archives/partial rm -fr ./rootfs/usr/share/doc rm -fr ./rootfs/usr/share/man rm -fr ./rootfs/var/lib/apt/lists/ From 75a47d1f34f8a372d108503f299ca3c26012b471 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 18 Jun 2021 15:35:29 +0200 Subject: [PATCH 108/990] Feature: Add persistent volumes as ext4 partitions Warning: Concurrent access to the same partition is not prevented --- examples/example_fastapi_2/__init__.py | 14 ++++++ examples/message_from_aleph.json | 8 +++- firecracker/microvm.py | 8 ++-- .../create_disk_image.sh | 2 +- runtimes/aleph-alpine-3.13-python/init1.py | 7 ++- vm_supervisor/conf.py | 2 + vm_supervisor/pool.py | 2 +- vm_supervisor/storage.py | 46 +++++++++++++++---- vm_supervisor/supervisor.py | 6 +-- vm_supervisor/vm/firecracker_microvm.py | 44 ++++++++++++------ 10 files changed, 106 insertions(+), 33 deletions(-) diff --git a/examples/example_fastapi_2/__init__.py b/examples/example_fastapi_2/__init__.py index bf7750d1a..74babfee2 100644 --- a/examples/example_fastapi_2/__init__.py +++ b/examples/example_fastapi_2/__init__.py @@ -1,3 +1,4 @@ +import json import logging from datetime import datetime from os import listdir @@ -98,3 +99,16 @@ async def remove_from_cache(key: str): async def keys_from_cache(pattern: str = '*'): """List keys from the VM cache""" return await cache.keys(pattern) + +@app.get("/state/increment") +async def increment(): + path = "/var/lib/sqlite/mydb" + try: + with open(path) as fd: + data = json.load(fd) + data["counter"] += 1 + except: + data = {"counter": 0} + with open(path, 'w') as fd: + json.dump(data, fd) + return data diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index 225ab5ce5..2a53f78b1 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -42,6 +42,12 @@ "mount": "/opt/venv", "ref": "5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51", "use_latest": false + }, + { + "comment": "Working data persisted on the VM supervisor, not available on other nodes", + "mount": "/var/lib/sqlite", + "name": "database", + "persistence": "host" } ], "data": { @@ -57,7 +63,7 @@ "replaces": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", "time": 1619017773.8950517 }, - "item_content": "{\"type\": \"vm-function\", \"address\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"allow_amend\": false, \"code\": {\"encoding\": \"zip\", \"entrypoint\": \"__init__:app\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"on\": {\"http\": true}, \"environment\": {\"reproducible\": true, \"internet\": true, \"aleph_api\": true, \"shared_cache\": false}, \"resources\": {\"vcpus\": 1, \"memory\": 128, \"seconds\": 30}, \"runtime\": {\"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false, \"comment\": \"Aleph Alpine Linux with Python 3.8\"}, \"volumes\": [{\"mount\": \"/opt/venv\", \"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false}], \"data\": {\"encoding\": \"zip\", \"mount\": \"/data\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"export\": {\"encoding\": \"zip\", \"mount\": \"/data\"}, \"replaces\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"time\": 1619017773.8950517}", + "item_content": "{\"type\": \"vm-function\", \"address\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"allow_amend\": false, \"code\": {\"encoding\": \"squashfs\", \"entrypoint\": \"__init__:app\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"on\": {\"http\": true}, \"environment\": {\"reproducible\": true, \"internet\": true, \"aleph_api\": true, \"shared_cache\": false}, \"resources\": {\"vcpus\": 1, \"memory\": 128, \"seconds\": 30}, \"runtime\": {\"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false, \"comment\": \"Aleph Alpine Linux with Python 3.8\"}, \"volumes\": [{\"mount\": \"/opt/venv\", \"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false}, {\"comment\": \"Working data persisted on the VM supervisor, not available on other nodes\", \"mount\": \"/var/lib/sqlite\", \"name\": \"database\", \"persistence\": \"host\"}], \"data\": {\"encoding\": \"zip\", \"mount\": \"/data\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"export\": {\"encoding\": \"zip\", \"mount\": \"/data\"}, \"replaces\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"time\": 1619017773.8950517}", "item_type": "inline", "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b", "size": 749, diff --git a/firecracker/microvm.py b/firecracker/microvm.py index a9f4be1f6..30d79834d 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -4,7 +4,6 @@ import os.path import string from asyncio import Task -from enum import Enum from os import getuid from pathlib import Path from pwd import getpwnam @@ -244,10 +243,11 @@ def enable_rootfs(self, path_on_host: str) -> str: else: return path_on_host - def compute_device_name(self, index: int) -> str: + @staticmethod + def compute_device_name(index: int) -> str: return f"vd{string.ascii_lowercase[index + 1]}" - def enable_drive(self, drive_path: str) -> Drive: + def enable_drive(self, drive_path: str, read_only: bool = True) -> Drive: """Make a volume available to the VM. Creates a symlink to the volume file if jailer is in use. @@ -264,7 +264,7 @@ def enable_drive(self, drive_path: str) -> Drive: drive_id=device_name, path_on_host=FilePath(drive_path), is_root_device=False, - is_read_only=True, + is_read_only=read_only, ) self.drives.append(drive) return drive diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh index 9c3ccb92f..9144227f2 100644 --- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh +++ b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh @@ -27,7 +27,7 @@ pip install fastapi apk add git pkgconf gcc py3-wheel python3-dev musl-dev py3-cffi libffi-dev autoconf automake libtool make echo "Pip installing aleph-client" -pip install aleph-client>=0.2.7 coincurve==15.0.0 +pip install 'aleph-client>=0.2.7' 'coincurve==15.0.0' # Compile all Python bytecode python3 -m compileall -f /usr/lib/python3.8/site-packages diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index befa25590..5f32237fb 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -46,6 +46,7 @@ class Interface(str, Enum): class Volume: mount: str device: str + read_only: bool @dataclass @@ -135,7 +136,11 @@ def setup_volumes(volumes: List[Volume]): for volume in volumes: logger.debug(f"Mounting /dev/{volume.device} on {volume.mount}") os.makedirs(volume.mount, exist_ok=True) - system(f"mount -t squashfs -o ro /dev/{volume.device} {volume.mount}") + if volume.read_only: + system(f"mount -t squashfs -o ro /dev/{volume.device} {volume.mount}") + else: + system(f"mount -o rw /dev/{volume.device} {volume.mount}") + system("mount") diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 716c70e6c..d46e7c34d 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -84,6 +84,8 @@ class Settings(BaseSettings): RUNTIME_CACHE: FilePath = FilePath(join(CACHE_ROOT, "runtime")) DATA_CACHE: FilePath = FilePath(join(CACHE_ROOT, "data")) + PERSISTENT_VOLUMES_DIR: FilePath = FilePath(join("/var/tmp/aleph", "volumes", "persistent")) + FAKE_DATA: bool = False FAKE_DATA_EXAMPLE: str = "example_fastapi_2" diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index 82b8481ce..10895db8b 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -39,7 +39,7 @@ def __init__(self): async def create_a_vm(self, message_content: ProgramContent, vm_hash: str) -> AlephFirecrackerVM: """Create a new Aleph Firecracker VM from an Aleph function message.""" - vm_resources = AlephFirecrackerResources(message_content) + vm_resources = AlephFirecrackerResources(message_content, vm_hash) await vm_resources.download_all() self.counter += 1 vm = AlephFirecrackerVM( diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index 49a8f0042..a48780302 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -4,17 +4,20 @@ In this prototype, it returns a hardcoded example. In the future, it should connect to an Aleph node and retrieve the code from there. """ +import asyncio import json import hashlib import logging import os +import re from os.path import isfile, join, abspath from shutil import make_archive import aiohttp from aleph_message.models import ProgramMessage -from aleph_message.models.program import Encoding +from aleph_message.models.program import Encoding, MachineVolume, ImmutableVolume, PersistentVolume, \ + VolumePersistence from .conf import settings from .models import FilePath @@ -126,12 +129,37 @@ async def get_runtime_path(ref: str) -> FilePath: return cache_path -async def get_volume_path(ref: str) -> FilePath: - if settings.FAKE_DATA: - data_dir = abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs")) - return FilePath(data_dir) +def create_ext4(path: FilePath) -> bool: + if os.path.isfile(path): + return False + tmp_path = f"{path}.tmp" + os.system(f"dd if=/dev/zero of={tmp_path} bs=1M count=500") + os.system(f"mkfs.ext4 {tmp_path}") + if settings.USE_JAILER: + os.system(f"chown jailman:jailman {tmp_path}") + os.rename(tmp_path, path) + return True - cache_path = FilePath(join(settings.DATA_CACHE, ref)) - url = f"{settings.CONNECTOR_URL}/download/data/{ref}" - await download_file(url, cache_path) - return cache_path + +async def get_volume_path(volume: MachineVolume, vm_hash: str) -> FilePath: + if isinstance(volume, ImmutableVolume): + ref = volume.ref + if settings.FAKE_DATA: + data_dir = abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs")) + return FilePath(data_dir) + + cache_path = FilePath(join(settings.DATA_CACHE, ref)) + url = f"{settings.CONNECTOR_URL}/download/data/{ref}" + await download_file(url, cache_path) + return cache_path + elif isinstance(volume, PersistentVolume): + if volume.persistence != VolumePersistence.host: + raise NotImplementedError("Only 'host' persistence is supported") + if not re.match(r'^[\w\-_/]+$', volume.name): + raise ValueError(f"Invalid value for volume name: {volume.name}") + os.makedirs(join(settings.PERSISTENT_VOLUMES_DIR, vm_hash), exist_ok=True) + volume_path = FilePath(join(settings.PERSISTENT_VOLUMES_DIR, vm_hash, f"{volume.name}.ext4")) + await asyncio.get_event_loop().run_in_executor(None, create_ext4, volume_path) + return volume_path + else: + raise NotImplementedError("Only immutable volumes are supported") diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 52f0e11c9..0dff832b4 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -65,10 +65,10 @@ async def update_with_latest_ref(obj): Useful to update references in parallel with asyncio.gather. """ - if obj is None: - return obj - if obj.use_latest: + if hasattr(obj, 'use_latest') and obj.use_latest: obj.ref = await get_latest_ref(obj.ref) + else: + return obj async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 053201404..392efd7b2 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -12,7 +12,7 @@ from aiohttp import ClientResponseError from aleph_message.models import ProgramContent -from aleph_message.models.program import MachineResources, MachineVolume, Encoding +from aleph_message.models.program import MachineResources, Encoding from firecracker.config import BootSource, Drive, MachineConfig, FirecrackerConfig, Vsock, \ NetworkInterface from firecracker.microvm import MicroVM, setfacl @@ -55,6 +55,14 @@ class Interface(str, Enum): class Volume: mount: str device: str + read_only: bool + + +@dataclass +class HostVolume: + mount: str + path_on_host: str + read_only: bool @dataclass @@ -98,15 +106,16 @@ class AlephFirecrackerResources: code_encoding: Encoding code_entrypoint: str rootfs_path: FilePath - volumes: List[MachineVolume] + volumes: List[HostVolume] volume_paths: Dict[str, FilePath] data_path: Optional[FilePath] + vm_hash: str - def __init__(self, message_content: ProgramContent): + def __init__(self, message_content: ProgramContent, vm_hash: str): self.message_content = message_content self.code_encoding = message_content.code.encoding self.code_entrypoint = message_content.code.entrypoint - self.volumes = message_content.volumes + self.vm_hash = vm_hash async def download_kernel(self): # Assumes kernel is already present on the host @@ -141,11 +150,18 @@ async def download_data(self): self.data_path = None async def download_volumes(self): - volume_paths = {} + volumes = [] # TODO: Download in parallel - for volume in self.volumes: - volume_paths[volume.mount] = await get_volume_path(volume.ref) - self.volume_paths = volume_paths + for volume in self.message_content.volumes: + volumes.append(HostVolume( + mount=volume.mount, + path_on_host=(await get_volume_path( + volume=volume, vm_hash=self.vm_hash)), + + read_only=volume.is_read_only(), + )) + self.volumes = volumes + async def download_all(self): await asyncio.gather( @@ -217,8 +233,8 @@ async def setup(self): [fvm.enable_drive(self.resources.code_path)] if self.resources.code_encoding == Encoding.squashfs else [] ) + [ - fvm.enable_drive(volume) - for volume in self.resources.volume_paths.values() + fvm.enable_drive(volume.path_on_host, read_only=volume.read_only) + for volume in self.resources.volumes ], machine_config=MachineConfig( vcpu_count=self.hardware_resources.vcpus, @@ -267,14 +283,16 @@ async def configure(self): volumes: List[Volume] if self.resources.code_encoding == Encoding.squashfs: code = b'' - volumes = [Volume(mount="/opt/code", device="vdb")] + [ - Volume(mount=volume.mount, device=self.fvm.drives[index+1].drive_id) + volumes = [Volume(mount="/opt/code", device="vdb", read_only=True)] + [ + Volume(mount=volume.mount, device=self.fvm.drives[index+1].drive_id, + read_only=volume.read_only) for index, volume in enumerate(self.resources.volumes) ] else: code: bytes = load_file_content(self.resources.code_path) volumes = [ - Volume(mount=volume.mount, device=self.fvm.drives[index].drive_id) + Volume(mount=volume.mount, device=self.fvm.drives[index].drive_id, + read_only=volume.read_only) for index, volume in enumerate(self.resources.volumes) ] From a04499938f298b249cbd6807c10f9f20e7b0d085 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 30 Jun 2021 10:26:59 +0200 Subject: [PATCH 109/990] Fix: Redirect all requests to the same running VM This should solve issues regarding concurrent access to the same partition --- examples/example_fastapi_2/__init__.py | 2 +- vm_supervisor/__main__.py | 4 +- vm_supervisor/models.py | 1 + vm_supervisor/pool.py | 82 +++++++++++++++----------- vm_supervisor/supervisor.py | 62 ++++++++++--------- 5 files changed, 87 insertions(+), 64 deletions(-) diff --git a/examples/example_fastapi_2/__init__.py b/examples/example_fastapi_2/__init__.py index 74babfee2..a766be004 100644 --- a/examples/example_fastapi_2/__init__.py +++ b/examples/example_fastapi_2/__init__.py @@ -25,7 +25,7 @@ async def index(): return { "Example": "example_fastapi_2", - "endpoints": ["/messages", "/internet", "/post_a_message"], + "endpoints": ["/messages", "/internet", "/post_a_message", "/state/increment"], "files_in_volumes": { "/opt/venv": list(listdir("/opt/venv")) }, diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 574918431..ede9a8808 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -9,6 +9,7 @@ from aiohttp.web import Response +from vm_supervisor.models import VmHash from . import supervisor from .conf import settings @@ -99,7 +100,7 @@ async def benchmark(runs: int): """Measure performance by immediately running the supervisor with fake requests. """ - ref = "fe488a08a7bed020515f069ce9a52847092af468beca79c66c8c0108bdab98a1" + ref = VmHash("652da037fe24ae9376946da6e50079881212cd6bb6274d043f34d74dc3339ab4") class FakeRequest: pass @@ -109,6 +110,7 @@ class FakeRequest: pass fake_request.query_string = "" fake_request.headers = [] fake_request.raw_headers = [] + # noinspection PyDeprecation fake_request.text = coroutine(lambda: None) logger.info("--- Start benchmark ---") diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index b79743982..7b84ef554 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -1,3 +1,4 @@ from typing import NewType FilePath = NewType("FilePath", str) +VmHash = NewType("VmHash", str) diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index 10895db8b..6fdd44f98 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -1,9 +1,12 @@ import asyncio import logging + +from dataclasses import dataclass from typing import Dict, Optional from aleph_message.models import ProgramContent from vm_supervisor.conf import settings +from vm_supervisor.models import VmHash from vm_supervisor.vm.firecracker_microvm import ( AlephFirecrackerVM, AlephFirecrackerResources, @@ -12,13 +15,11 @@ logger = logging.getLogger(__name__) +@dataclass class StartedVM: vm: AlephFirecrackerVM - timeout_task: Optional[asyncio.Task] - - def __init__(self, vm: AlephFirecrackerVM): - self.vm = vm - self.timeout_task = None + program: ProgramContent + timeout_task: Optional[asyncio.Task] = None class VmPool: @@ -31,23 +32,25 @@ class VmPool: """ counter: int # Used to provide distinct ids to network interfaces - started_vms_cache: Dict[ProgramContent, StartedVM] + starting_vms: Dict[VmHash, bool] + started_vms: Dict[VmHash, StartedVM] def __init__(self): self.counter = settings.START_ID_INDEX - self.started_vms_cache = {} + self.starting_vms = {} + self.started_vms = {} - async def create_a_vm(self, message_content: ProgramContent, vm_hash: str) -> AlephFirecrackerVM: + async def create_a_vm(self, program: ProgramContent, vm_hash: VmHash) -> AlephFirecrackerVM: """Create a new Aleph Firecracker VM from an Aleph function message.""" - vm_resources = AlephFirecrackerResources(message_content, vm_hash) + vm_resources = AlephFirecrackerResources(program, vm_hash) await vm_resources.download_all() self.counter += 1 vm = AlephFirecrackerVM( vm_id=self.counter, vm_hash=vm_hash, resources=vm_resources, - enable_networking=message_content.environment.internet, - hardware_resources=message_content.resources, + enable_networking=program.environment.internet, + hardware_resources=program.resources, ) try: await vm.setup() @@ -59,36 +62,49 @@ async def create_a_vm(self, message_content: ProgramContent, vm_hash: str) -> Al await vm.teardown() raise - - async def get_a_vm(self, message: ProgramContent, vm_hash: str) -> AlephFirecrackerVM: + async def get_or_create(self, program: ProgramContent, vm_hash: VmHash) -> AlephFirecrackerVM: """Provision a VM in the pool, then return the first VM from the pool.""" - try: - started_vm = self.started_vms_cache.pop(message) + # Wait for a VM already starting to be available + while self.starting_vms.get(vm_hash): + await asyncio.sleep(0.01) + + started_vm = self.started_vms.get(vm_hash) + if started_vm: + if started_vm.timeout_task: + started_vm.timeout_task.cancel() + return started_vm.vm + else: + self.starting_vms[vm_hash] = True + try: + vm = await self.create_a_vm(program=program, vm_hash=vm_hash) + self.started_vms[vm_hash] = StartedVM(vm=vm, program=program) + return vm + finally: + del self.starting_vms[vm_hash] + + async def get(self, vm_hash: VmHash) -> Optional[AlephFirecrackerVM]: + started_vm = self.started_vms.get(vm_hash) + if started_vm: started_vm.timeout_task.cancel() return started_vm.vm - except KeyError: - return await self.create_a_vm(message_content=message, vm_hash=vm_hash) + else: + return None - def keep_in_cache( - self, vm: AlephFirecrackerVM, message: ProgramContent, timeout: float = 1.0 - ) -> None: + def stop_after_timeout(self, vm_hash: VmHash, timeout: float = 1.0) -> None: """Keep a VM running for `timeout` seconds in case another query comes by.""" + print('SS', self.started_vms) + started_vm = self.started_vms[vm_hash] - if message in self.started_vms_cache: - logger.warning("VM already in keep_in_cache, not caching") - return - - started_vm = StartedVM(vm=vm) - self.started_vms_cache[message] = started_vm + if started_vm.timeout_task: + logger.debug("VM already has a timeout. Extending it.") + started_vm.timeout_task.cancel() loop = asyncio.get_event_loop() - started_vm.timeout_task = loop.create_task(self.expire(vm, message, timeout)) + started_vm.timeout_task = loop.create_task(self.expire(vm_hash, timeout)) - async def expire( - self, vm: AlephFirecrackerVM, message: ProgramContent, timeout: float - ): + async def expire(self, vm_hash: VmHash, timeout: float) -> None: """Coroutine that will stop the VM after 'timeout' seconds.""" await asyncio.sleep(timeout) - assert self.started_vms_cache[message].vm is vm - del self.started_vms_cache[message] - await vm.teardown() + started_vm = self.started_vms[vm_hash] + del self.started_vms[vm_hash] + await started_vm.vm.teardown() diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 0dff832b4..d326fab09 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -21,9 +21,10 @@ from aleph_message.models import ProgramMessage, ProgramContent from firecracker.microvm import MicroVMFailedInit from .conf import settings +from .models import VmHash from .pool import VmPool from .storage import get_message, get_latest_amend -from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError +from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError, AlephFirecrackerVM logger = logging.getLogger(__name__) pool = VmPool() @@ -82,38 +83,41 @@ async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: } -async def run_code(message_ref: str, path: str, request: web.Request) -> web.Response: +async def run_code(message_ref: VmHash, path: str, request: web.Request) -> web.Response: """ Execute the code corresponding to the 'code id' in the path. """ - message: ProgramMessage = await try_get_message(message_ref) - message_content: ProgramContent = message.content - - # Load amends - await asyncio.gather( - update_with_latest_ref(message_content.runtime), - update_with_latest_ref(message_content.code), - update_with_latest_ref(message_content.data), - *( - update_with_latest_ref(volume) - for volume in (message_content.volumes or []) - ), - ) + vm: AlephFirecrackerVM = await pool.get(vm_hash=message_ref) + if not vm: + message: ProgramMessage = await try_get_message(message_ref) + message_content: ProgramContent = message.content + + # Load amends + await asyncio.gather( + update_with_latest_ref(message_content.runtime), + update_with_latest_ref(message_content.code), + update_with_latest_ref(message_content.data), + *( + update_with_latest_ref(volume) + for volume in (message_content.volumes or []) + ), + ) - # TODO: Cache message content after amends + # TODO: Cache message content after amends + # TODO: Update VM in case a new version has been released - try: - vm = await pool.get_a_vm(message_content, vm_hash=message.item_hash) - except ResourceDownloadError as error: - logger.exception(error) - raise HTTPBadRequest(reason="Code, runtime or data not available") - except VmSetupError as error: - logger.exception(error) - raise HTTPInternalServerError(reason="Error during program initialisation") - except MicroVMFailedInit as error: - logger.exception(error) - raise HTTPInternalServerError(reason="Error during runtime initialisation") + try: + vm = await pool.get_or_create(message_content, vm_hash=VmHash(message.item_hash)) + except ResourceDownloadError as error: + logger.exception(error) + raise HTTPBadRequest(reason="Code, runtime or data not available") + except VmSetupError as error: + logger.exception(error) + raise HTTPInternalServerError(reason="Error during program initialisation") + except MicroVMFailedInit as error: + logger.exception(error) + raise HTTPInternalServerError(reason="Error during runtime initialisation") logger.debug(f"Using vm={vm.vm_id}") @@ -153,7 +157,7 @@ async def run_code(message_ref: str, path: str, request: web.Request) -> web.Res return web.Response(status=502, reason="Invalid response from VM") finally: if settings.REUSE_TIMEOUT > 0: - pool.keep_in_cache(vm, message_content, timeout=settings.REUSE_TIMEOUT) + pool.stop_after_timeout(vm_hash=message_ref, timeout=settings.REUSE_TIMEOUT) else: await vm.teardown() @@ -167,7 +171,7 @@ def run_code_from_path(request: web.Request) -> Awaitable[web.Response]: path = request.match_info["suffix"] path = path if path.startswith("/") else f"/{path}" - message_ref: str = request.match_info["ref"] + message_ref: VmHash = request.match_info["ref"] return run_code(message_ref, path, request) From f41934147e1984cdd7219883256b272cb1498d56 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 30 Jun 2021 13:11:38 +0200 Subject: [PATCH 110/990] Feature: Support persistent disk size specification --- docker/vm_supervisor.dockerfile | 2 +- examples/message_from_aleph.json | 3 ++- examples/volumes/Dockerfile | 2 +- vm_supervisor/README.md | 2 +- vm_supervisor/__main__.py | 2 +- vm_supervisor/storage.py | 7 ++++--- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile index 0cef72d33..f5ed2f3d4 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor.dockerfile @@ -18,7 +18,7 @@ RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/downl RUN ln /opt/firecracker/firecracker-v* /opt/firecracker/firecracker RUN ln /opt/firecracker/jailer-v* /opt/firecracker/jailer -RUN pip3 install typing-extensions aleph-message>=0.1.10 pydantic +RUN pip3 install typing-extensions 'aleph-message>=0.1.12' pydantic RUN mkdir /srv/jailer diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index 2a53f78b1..f3f6eb6a3 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -47,7 +47,8 @@ "comment": "Working data persisted on the VM supervisor, not available on other nodes", "mount": "/var/lib/sqlite", "name": "database", - "persistence": "host" + "persistence": "host", + "size_mib": 5 } ], "data": { diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile index 8ecd44529..b23ab639a 100644 --- a/examples/volumes/Dockerfile +++ b/examples/volumes/Dockerfile @@ -6,6 +6,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \ && rm -rf /var/lib/apt/lists/* RUN python3 -m venv /opt/venv -RUN /opt/venv/bin/pip install aleph-message +RUN /opt/venv/bin/pip install 'aleph-message>=0.1.12' CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index ce7a5e542..ef3737bdd 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -88,7 +88,7 @@ is used to parse and validate Aleph messages. ```shell apt install -y --no-install-recommends --no-install-suggests python3-pip pip3 install pydantic[dotenv] -pip3 install aleph-message>=0.1.7 +pip3 install aleph-message>=0.1.12 ``` ### 2.f. Create the jailer working directory: diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index ede9a8808..6adc7791c 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -100,7 +100,7 @@ async def benchmark(runs: int): """Measure performance by immediately running the supervisor with fake requests. """ - ref = VmHash("652da037fe24ae9376946da6e50079881212cd6bb6274d043f34d74dc3339ab4") + ref = VmHash("9b1ef4d969e393c871cef25bab345c8eaabfe81d1fc6536f287be4f6bb7c852a") class FakeRequest: pass diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index a48780302..9c51ea737 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -129,11 +129,11 @@ async def get_runtime_path(ref: str) -> FilePath: return cache_path -def create_ext4(path: FilePath) -> bool: +def create_ext4(path: FilePath, size_mib: int) -> bool: if os.path.isfile(path): return False tmp_path = f"{path}.tmp" - os.system(f"dd if=/dev/zero of={tmp_path} bs=1M count=500") + os.system(f"dd if=/dev/zero of={tmp_path} bs=1M count={size_mib}") os.system(f"mkfs.ext4 {tmp_path}") if settings.USE_JAILER: os.system(f"chown jailman:jailman {tmp_path}") @@ -159,7 +159,8 @@ async def get_volume_path(volume: MachineVolume, vm_hash: str) -> FilePath: raise ValueError(f"Invalid value for volume name: {volume.name}") os.makedirs(join(settings.PERSISTENT_VOLUMES_DIR, vm_hash), exist_ok=True) volume_path = FilePath(join(settings.PERSISTENT_VOLUMES_DIR, vm_hash, f"{volume.name}.ext4")) - await asyncio.get_event_loop().run_in_executor(None, create_ext4, volume_path) + await asyncio.get_event_loop().run_in_executor( + None, create_ext4, volume_path, volume.size_mib) return volume_path else: raise NotImplementedError("Only immutable volumes are supported") From eb94565d0bee342b7b84a87f9b00b7c57f81f678 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 1 Jul 2021 16:57:45 +0200 Subject: [PATCH 111/990] Fix: ASGI mandates lowercase header names --- vm_supervisor/supervisor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index d326fab09..0560ed747 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -73,12 +73,15 @@ async def update_with_latest_ref(obj): async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: + # ASGI mandates lowercase header names + headers = tuple((name.lower(), value) + for name, value in request.raw_headers) return { "type": "http", "path": path, "method": request.method, "query_string": request.query_string, - "headers": request.raw_headers, + "headers": headers, "body": await request.text() } From 4c680043afd2457a2717a3f897f7c9846baebec4 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 1 Jul 2021 16:59:52 +0200 Subject: [PATCH 112/990] Fix: POST body was not passed to ASGI apps --- runtimes/aleph-alpine-3.13-python/init1.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 5f32237fb..c86b7c050 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -218,8 +218,14 @@ async def run_python_code_http(application: ASGIApplication, scope: dict logger.debug("Running code") with StringIO() as buf, redirect_stdout(buf): # Execute in the same process, saves ~20ms than a subprocess + + # The body should not be part of the ASGI scope itself + body: bytes = scope.pop('body') + async def receive(): - pass + return {'type': 'http.request', + 'body': body, + 'more_body': False} send_queue: asyncio.Queue = asyncio.Queue() From 3b271790f055ba1a09cd960a5d599b08eea0b4b6 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 1 Jul 2021 17:00:20 +0200 Subject: [PATCH 113/990] Fix: ASGI did not support submodules --- runtimes/aleph-alpine-3.13-python/init1.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index c86b7c050..bae778765 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -151,6 +151,8 @@ def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApp module_name, app_name = entrypoint.split(":", 1) logger.debug("import module") module = __import__(module_name) + for level in module_name.split('.')[1:]: + module = getattr(module, level) app: ASGIApplication = getattr(module, app_name) elif encoding == Encoding.zip: # Unzip in /opt and import the entrypoint from there From 750dd7b8c4efff339dac4dac0155d8c468136174 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 1 Jul 2021 17:01:01 +0200 Subject: [PATCH 114/990] Fix: Pass body to in ASGI scope as binary --- vm_supervisor/supervisor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 0560ed747..febb9ea56 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -82,7 +82,7 @@ async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: "method": request.method, "query_string": request.query_string, "headers": headers, - "body": await request.text() + "body": await request.read() } From 5bb27f266b9018a9b2144aa5562f4c19d0483359 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 1 Jul 2021 18:46:07 +0200 Subject: [PATCH 115/990] Fix: Hash access failed on failed hashes with FAKE_DATA --- vm_supervisor/pool.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index 6fdd44f98..2a8f96b1a 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -93,6 +93,10 @@ async def get(self, vm_hash: VmHash) -> Optional[AlephFirecrackerVM]: def stop_after_timeout(self, vm_hash: VmHash, timeout: float = 1.0) -> None: """Keep a VM running for `timeout` seconds in case another query comes by.""" print('SS', self.started_vms) + + if settings.FAKE_DATA: + vm_hash = list(self.started_vms.keys())[0] + started_vm = self.started_vms[vm_hash] if started_vm.timeout_task: From 2da5ab8cabbced26e0532dd40872a0c1fcec0612 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 1 Jul 2021 18:46:32 +0200 Subject: [PATCH 116/990] Doc: Document the use of pool properties --- vm_supervisor/pool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index 2a8f96b1a..48536e9aa 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -32,8 +32,8 @@ class VmPool: """ counter: int # Used to provide distinct ids to network interfaces - starting_vms: Dict[VmHash, bool] - started_vms: Dict[VmHash, StartedVM] + starting_vms: Dict[VmHash, bool] # Lock containing hash of VMs being started + started_vms: Dict[VmHash, StartedVM] # Shared pool of VMs already started def __init__(self): self.counter = settings.START_ID_INDEX From eefce5d315b5211789c86bc58630375849df94ec Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 2 Jul 2021 11:12:13 +0200 Subject: [PATCH 117/990] Fix: CI timeout of 5 minutes was cancelling the tests --- .github/workflows/test-integration-fakedata.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-integration-fakedata.yml b/.github/workflows/test-integration-fakedata.yml index 06b11c3a4..de7fd7bfe 100644 --- a/.github/workflows/test-integration-fakedata.yml +++ b/.github/workflows/test-integration-fakedata.yml @@ -3,7 +3,7 @@ on: [push] jobs: Run-VM-Supervisor-Fake-Data: runs-on: self-hosted - timeout-minutes: 5 + timeout-minutes: 10 env: ALEPH_VM_FAKE_DATA: true ALEPH_VM_LINUX_PATH: /opt/vmlinux.bin From a6ad93ef3060cb7d43c5f3dd6d4ccdedd140d361 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 2 Jul 2021 11:28:34 +0200 Subject: [PATCH 118/990] Fix: Benchmark fake request headers were incorrect --- vm_supervisor/__main__.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 6adc7791c..8ea52da16 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -5,7 +5,7 @@ import time from asyncio import coroutine from statistics import mean -from typing import List +from typing import List, Tuple, Dict from aiohttp.web import Response @@ -102,16 +102,26 @@ async def benchmark(runs: int): """ ref = VmHash("9b1ef4d969e393c871cef25bab345c8eaabfe81d1fc6536f287be4f6bb7c852a") - class FakeRequest: pass + class FakeRequest: + headers: Dict[str, str] + raw_headers: List[Tuple[bytes, bytes]] fake_request = FakeRequest() fake_request.match_info = {"ref": ref, "suffix": "/"} fake_request.method = "GET" fake_request.query_string = "" - fake_request.headers = [] - fake_request.raw_headers = [] + + fake_request.headers = { + 'host': '127.0.0.1', + 'content-type': 'application/json' + } + fake_request.raw_headers = [ + (name.encode(), value.encode()) + for name, value in fake_request.headers.items() + ] + # noinspection PyDeprecation - fake_request.text = coroutine(lambda: None) + fake_request.read = coroutine(lambda: b'') logger.info("--- Start benchmark ---") From 1b577a5242758c5d86d425dbd710e40459cd5307 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 2 Jul 2021 13:40:52 +0200 Subject: [PATCH 119/990] Chore: Use coherent fixed fake hash --- examples/message_from_aleph.json | 2 +- vm_supervisor/__main__.py | 6 +++--- vm_supervisor/supervisor.py | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index f3f6eb6a3..064e6d2e6 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -3,7 +3,7 @@ "$oid": "6080402d7f44efefd611dc1e" }, "chain": "ETH", - "item_hash": "787fb143b2ac74c6cc348b3fc10bb571d41f372156ab2f54b0e41494b58b1a1e", + "item_hash": "TEST_HASH", "sender": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", "type": "PROGRAM", "channel": "Fun-dApps", diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 8ea52da16..f03f2c465 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -100,7 +100,7 @@ async def benchmark(runs: int): """Measure performance by immediately running the supervisor with fake requests. """ - ref = VmHash("9b1ef4d969e393c871cef25bab345c8eaabfe81d1fc6536f287be4f6bb7c852a") + ref = VmHash("TEST_HASH") class FakeRequest: headers: Dict[str, str] @@ -132,7 +132,7 @@ class FakeRequest: for path in ("/", "/messages", "/internet", "/post_a_message", "/cache/set/foo/bar", "/cache/get/foo", "/cache/keys"): fake_request.match_info["suffix"] = path - response: Response = await supervisor.run_code(message_ref=ref, + response: Response = await supervisor.run_code(vm_hash=ref, path=path, request=fake_request) assert response.status == 200 @@ -143,7 +143,7 @@ class FakeRequest: for run in range(runs): t0 = time.time() fake_request.match_info["suffix"] = path - response: Response = await supervisor.run_code(message_ref=ref, + response: Response = await supervisor.run_code(vm_hash=ref, path=path, request=fake_request) assert response.status == 200 diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index febb9ea56..7ae38a998 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -86,14 +86,14 @@ async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: } -async def run_code(message_ref: VmHash, path: str, request: web.Request) -> web.Response: +async def run_code(vm_hash: VmHash, path: str, request: web.Request) -> web.Response: """ Execute the code corresponding to the 'code id' in the path. """ - vm: AlephFirecrackerVM = await pool.get(vm_hash=message_ref) + vm: AlephFirecrackerVM = await pool.get(vm_hash=vm_hash) if not vm: - message: ProgramMessage = await try_get_message(message_ref) + message: ProgramMessage = await try_get_message(vm_hash) message_content: ProgramContent = message.content # Load amends @@ -111,7 +111,7 @@ async def run_code(message_ref: VmHash, path: str, request: web.Request) -> web. # TODO: Update VM in case a new version has been released try: - vm = await pool.get_or_create(message_content, vm_hash=VmHash(message.item_hash)) + vm = await pool.get_or_create(message_content, vm_hash=vm_hash) except ResourceDownloadError as error: logger.exception(error) raise HTTPBadRequest(reason="Code, runtime or data not available") @@ -160,7 +160,7 @@ async def run_code(message_ref: VmHash, path: str, request: web.Request) -> web. return web.Response(status=502, reason="Invalid response from VM") finally: if settings.REUSE_TIMEOUT > 0: - pool.stop_after_timeout(vm_hash=message_ref, timeout=settings.REUSE_TIMEOUT) + pool.stop_after_timeout(vm_hash=vm_hash, timeout=settings.REUSE_TIMEOUT) else: await vm.teardown() @@ -207,7 +207,7 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: message_ref_base32 = request.host.split(".")[0] if settings.FAKE_DATA: - message_ref = "test" + message_ref = "TEST_HASH" else: try: message_ref = b32_to_b16(message_ref_base32).decode() From c18289a34c41b2bd909dfb2a9702ab7b870b3b4e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 2 Jul 2021 13:49:12 +0200 Subject: [PATCH 120/990] Refactor: Rename example with fastapi --- examples/example_fastapi_2/{__init__.py => main.py} | 12 ++++++++++++ examples/message_from_aleph.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) rename examples/example_fastapi_2/{__init__.py => main.py} (93%) diff --git a/examples/example_fastapi_2/__init__.py b/examples/example_fastapi_2/main.py similarity index 93% rename from examples/example_fastapi_2/__init__.py rename to examples/example_fastapi_2/main.py index a766be004..b27cd756d 100644 --- a/examples/example_fastapi_2/__init__.py +++ b/examples/example_fastapi_2/main.py @@ -2,6 +2,8 @@ import logging from datetime import datetime from os import listdir +from fastapi import Request +from pydantic import BaseModel logger = logging.getLogger(__name__) @@ -112,3 +114,13 @@ async def increment(): with open(path, 'w') as fd: json.dump(data, fd) return data + + +class Data(BaseModel): + text: str + number: int + + +@app.post("/post") +async def receive_post(data: Data): + return str(data) diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index 064e6d2e6..26a4f253d 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -14,7 +14,7 @@ "allow_amend": false, "code": { "encoding": "squashfs", - "entrypoint": "__init__:app", + "entrypoint": "main:app", "ref": "7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003", "use_latest": false }, From bf51736e8d81e7624c8cc03459c44ac2b8accac7 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 2 Jul 2021 11:56:58 +0200 Subject: [PATCH 121/990] Feature: Add example using Django --- .dockerignore | 1 + .gitignore | 1 + examples/example_django/blog/__init__.py | 0 examples/example_django/blog/admin.py | 6 + examples/example_django/blog/apps.py | 6 + examples/example_django/blog/forms.py | 11 ++ .../blog/migrations/0001_initial.py | 42 ++++++ .../blog/migrations/__init__.py | 0 examples/example_django/blog/models.py | 21 +++ .../blog/templates/blog/article_list.html | 62 ++++++++ .../blog/templates/blog/comment.html | 5 + examples/example_django/blog/urls.py | 10 ++ examples/example_django/blog/views.py | 25 ++++ .../example_django/example_django/__init__.py | 0 .../example_django/example_django/asgi.py | 16 ++ .../example_django/example_django/settings.py | 138 ++++++++++++++++++ .../example_django/example_django/urls.py | 22 +++ .../example_django/example_django/wsgi.py | 16 ++ examples/example_django/manage.py | 22 +++ .../create_disk_image.sh | 2 +- vm_supervisor/conf.py | 1 + 21 files changed, 406 insertions(+), 1 deletion(-) create mode 100644 examples/example_django/blog/__init__.py create mode 100644 examples/example_django/blog/admin.py create mode 100644 examples/example_django/blog/apps.py create mode 100644 examples/example_django/blog/forms.py create mode 100644 examples/example_django/blog/migrations/0001_initial.py create mode 100644 examples/example_django/blog/migrations/__init__.py create mode 100644 examples/example_django/blog/models.py create mode 100644 examples/example_django/blog/templates/blog/article_list.html create mode 100644 examples/example_django/blog/templates/blog/comment.html create mode 100644 examples/example_django/blog/urls.py create mode 100644 examples/example_django/blog/views.py create mode 100644 examples/example_django/example_django/__init__.py create mode 100644 examples/example_django/example_django/asgi.py create mode 100644 examples/example_django/example_django/settings.py create mode 100644 examples/example_django/example_django/urls.py create mode 100644 examples/example_django/example_django/wsgi.py create mode 100755 examples/example_django/manage.py diff --git a/.dockerignore b/.dockerignore index e41ff7a8b..72ede1e30 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,7 @@ **/*.pyc **/__pycache__ +**/*.sqlite3 **/*.bin **/*.ext4 **/*.zip diff --git a/.gitignore b/.gitignore index d36d1361d..0c1b40102 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.pyc __pycache__ +*.sqlite3 *.bin *.ext4 *.zip diff --git a/examples/example_django/blog/__init__.py b/examples/example_django/blog/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/example_django/blog/admin.py b/examples/example_django/blog/admin.py new file mode 100644 index 000000000..54601b564 --- /dev/null +++ b/examples/example_django/blog/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin + +from .models import Article, Comment + +admin.site.register(Article) +admin.site.register(Comment) diff --git a/examples/example_django/blog/apps.py b/examples/example_django/blog/apps.py new file mode 100644 index 000000000..6be26c734 --- /dev/null +++ b/examples/example_django/blog/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class BlogConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "blog" diff --git a/examples/example_django/blog/forms.py b/examples/example_django/blog/forms.py new file mode 100644 index 000000000..e07413eff --- /dev/null +++ b/examples/example_django/blog/forms.py @@ -0,0 +1,11 @@ +from django import forms +from django.forms import ModelForm + +from .models import Comment + + +class CommentForm(ModelForm): + class Meta: + model = Comment + fields = ["text", "article"] + widgets = {"article": forms.HiddenInput()} diff --git a/examples/example_django/blog/migrations/0001_initial.py b/examples/example_django/blog/migrations/0001_initial.py new file mode 100644 index 000000000..214f95af3 --- /dev/null +++ b/examples/example_django/blog/migrations/0001_initial.py @@ -0,0 +1,42 @@ +# Generated by Django 3.2.4 on 2021-07-02 09:35 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Article", + fields=[ + ("date", models.DateTimeField(auto_created=True)), + ("id", models.UUIDField(primary_key=True, serialize=False)), + ( + "title", + models.CharField( + help_text="Title of the blog article", max_length=256 + ), + ), + ("body", models.TextField(help_text="Body of the blog article")), + ], + ), + migrations.CreateModel( + name="Comment", + fields=[ + ("date", models.DateTimeField(auto_created=True, auto_now_add=True)), + ("id", models.UUIDField(primary_key=True, serialize=False)), + ("text", models.CharField(max_length=1024)), + ( + "article", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="blog.article" + ), + ), + ], + ), + ] diff --git a/examples/example_django/blog/migrations/__init__.py b/examples/example_django/blog/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/example_django/blog/models.py b/examples/example_django/blog/models.py new file mode 100644 index 000000000..8a73e3842 --- /dev/null +++ b/examples/example_django/blog/models.py @@ -0,0 +1,21 @@ +from django.db import models + + +class Article(models.Model): + id = models.UUIDField(primary_key=True) + title = models.CharField(max_length=256, help_text="Title of the blog article") + body = models.TextField(help_text="Body of the blog article") + date = models.DateTimeField(auto_created=True) + + def __str__(self): + return f"Blog article '{self.title}'" + + +class Comment(models.Model): + id = models.UUIDField(primary_key=True) + text = models.CharField(max_length=1024) + article = models.ForeignKey(to=Article, on_delete=models.CASCADE) + date = models.DateTimeField(auto_created=True, auto_now_add=True) + + def __str__(self): + return f"Comment on {self.article.title}" diff --git a/examples/example_django/blog/templates/blog/article_list.html b/examples/example_django/blog/templates/blog/article_list.html new file mode 100644 index 000000000..362732ea2 --- /dev/null +++ b/examples/example_django/blog/templates/blog/article_list.html @@ -0,0 +1,62 @@ + + + + My Django Blog + + + +

My Django Blog

+ +{% for article in object_list %} +
+

{{ article.title }}

+
Published on
+

+ {{ article.body }} +

+
+ {% for comment in article.comment_set.all %} +

{{ comment.text }}

+ {% endfor %} + +
+ {% csrf_token %} + {{ form }} + + +
+
+
+{% empty %} +
  • No articles yet.
    • +{% endfor %} + + + + diff --git a/examples/example_django/blog/templates/blog/comment.html b/examples/example_django/blog/templates/blog/comment.html new file mode 100644 index 000000000..58409efe3 --- /dev/null +++ b/examples/example_django/blog/templates/blog/comment.html @@ -0,0 +1,5 @@ +
    + {% csrf_token %} + {{ form }} + +
    diff --git a/examples/example_django/blog/urls.py b/examples/example_django/blog/urls.py new file mode 100644 index 000000000..1d5ef5927 --- /dev/null +++ b/examples/example_django/blog/urls.py @@ -0,0 +1,10 @@ +from django.urls import path +from django.views.decorators.csrf import csrf_exempt + +from .views import ArticleListView, CommentFormView, test_view + +urlpatterns = [ + path("", ArticleListView.as_view(), name="article-list"), + path("comment", csrf_exempt(CommentFormView.as_view()), name="comment"), + path("post", csrf_exempt(test_view), name="test-post"), +] diff --git a/examples/example_django/blog/views.py b/examples/example_django/blog/views.py new file mode 100644 index 000000000..7d3d3e7f8 --- /dev/null +++ b/examples/example_django/blog/views.py @@ -0,0 +1,25 @@ +import os + +from django.http import JsonResponse +from django.views.generic import ListView, FormView, CreateView + +from .forms import CommentForm +from .models import Article + + +class ArticleListView(ListView): + model = Article + ordering = "-date" + + extra_context = {"form": CommentForm} + + +class CommentFormView(CreateView): + template_name = "blog/comment.html" + form_class = CommentForm + success_url = "/" + + +def test_view(request): + print(request.POST) + return JsonResponse(request.POST) diff --git a/examples/example_django/example_django/__init__.py b/examples/example_django/example_django/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/example_django/example_django/asgi.py b/examples/example_django/example_django/asgi.py new file mode 100644 index 000000000..bb25cd738 --- /dev/null +++ b/examples/example_django/example_django/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for example_django project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example_django.settings") + +application = get_asgi_application() diff --git a/examples/example_django/example_django/settings.py b/examples/example_django/example_django/settings.py new file mode 100644 index 000000000..75e53c576 --- /dev/null +++ b/examples/example_django/example_django/settings.py @@ -0,0 +1,138 @@ +""" +Django settings for example_django project. + +Generated by 'django-admin startproject' using Django 3.2.4. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.2/ref/settings/ +""" +import os.path +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "django-insecure-1r3v1fc$q%sqy)0#bybc4pd##g+!tpm%+4^5opqyu93o0hqk$w" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ["127.0.0.1", "vm.demo.okeso.fr", "test.vm.demo.okeso.fr"] + + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "blog", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "example_django.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "example_django.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/3.2/ref/settings/#databases + +if os.path.isdir("/var/lib/sqlite"): + # Inside Aleph VM + DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": "/var/lib/sqlite/db.sqlite3", + } + } +else: + # On developer setup + DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3", + } + } + + +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.2/howto/static-files/ + +STATIC_URL = "https://ipfs.io/ipfs/QmUhm7UWrGrjoJY5cVZ9ur9PtT7nHzdmXJuNpD8s7VLcJR/" + +STATIC_ROOT = os.path.join(BASE_DIR, "static") + +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" diff --git a/examples/example_django/example_django/urls.py b/examples/example_django/example_django/urls.py new file mode 100644 index 000000000..477c80598 --- /dev/null +++ b/examples/example_django/example_django/urls.py @@ -0,0 +1,22 @@ +"""example_django URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path, include + +urlpatterns = [ + path("", include("blog.urls")), + path("admin/", admin.site.urls), +] diff --git a/examples/example_django/example_django/wsgi.py b/examples/example_django/example_django/wsgi.py new file mode 100644 index 000000000..35cccb4bf --- /dev/null +++ b/examples/example_django/example_django/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for example_django project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example_django.settings") + +application = get_wsgi_application() diff --git a/examples/example_django/manage.py b/examples/example_django/manage.py new file mode 100755 index 000000000..77dced937 --- /dev/null +++ b/examples/example_django/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example_django.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index ec242c2b6..25eba8b49 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -21,7 +21,7 @@ apt-get install -y --no-install-recommends --no-install-suggests \ iproute2 unzip \ nodejs -pip3 install fastapi +pip3 install fastapi django echo "Pip installing aleph-client" pip3 install 'aleph-client>=0.2.7' 'coincurve==15.0.0' diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index d46e7c34d..b3f5ae49a 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -88,6 +88,7 @@ class Settings(BaseSettings): FAKE_DATA: bool = False FAKE_DATA_EXAMPLE: str = "example_fastapi_2" + # FAKE_DATA_EXAMPLE: str = "example_django" def update(self, **kwargs): for key, value in kwargs.items(): From 263e976e50a0636117007c3f018a3139d68efb05 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 7 Jul 2021 09:44:00 +0200 Subject: [PATCH 122/990] Fix: Use UUID as model ids in Django example --- .../migrations/0002_auto_20210702_1331.py | 29 +++++++++++++++++++ examples/example_django/blog/models.py | 8 +++-- 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 examples/example_django/blog/migrations/0002_auto_20210702_1331.py diff --git a/examples/example_django/blog/migrations/0002_auto_20210702_1331.py b/examples/example_django/blog/migrations/0002_auto_20210702_1331.py new file mode 100644 index 000000000..7d72238e4 --- /dev/null +++ b/examples/example_django/blog/migrations/0002_auto_20210702_1331.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.4 on 2021-07-02 13:31 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('blog', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='article', + name='id', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='comment', + name='date', + field=models.DateTimeField(auto_now_add=True), + ), + migrations.AlterField( + model_name='comment', + name='id', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), + ), + ] diff --git a/examples/example_django/blog/models.py b/examples/example_django/blog/models.py index 8a73e3842..b60094096 100644 --- a/examples/example_django/blog/models.py +++ b/examples/example_django/blog/models.py @@ -1,8 +1,10 @@ +import uuid + from django.db import models class Article(models.Model): - id = models.UUIDField(primary_key=True) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) title = models.CharField(max_length=256, help_text="Title of the blog article") body = models.TextField(help_text="Body of the blog article") date = models.DateTimeField(auto_created=True) @@ -12,10 +14,10 @@ def __str__(self): class Comment(models.Model): - id = models.UUIDField(primary_key=True) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) text = models.CharField(max_length=1024) article = models.ForeignKey(to=Article, on_delete=models.CASCADE) - date = models.DateTimeField(auto_created=True, auto_now_add=True) + date = models.DateTimeField(auto_now_add=True, editable=False) def __str__(self): return f"Comment on {self.article.title}" From 107a805761068b490e4478062da85fc82e3f6b31 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 7 Jul 2021 09:44:45 +0200 Subject: [PATCH 123/990] Feature: Initialize DB and load fixture in Django example Closes #54 --- .../blog/fixtures/default_articles.json | 11 +++++++++++ examples/example_django/example_django/asgi.py | 5 +++++ 2 files changed, 16 insertions(+) create mode 100644 examples/example_django/blog/fixtures/default_articles.json diff --git a/examples/example_django/blog/fixtures/default_articles.json b/examples/example_django/blog/fixtures/default_articles.json new file mode 100644 index 000000000..b9a135fb2 --- /dev/null +++ b/examples/example_django/blog/fixtures/default_articles.json @@ -0,0 +1,11 @@ +[ + { + "model": "blog.article", + "pk": "f115d067-f6c9-4532-a140-40c51f37a1bc", + "fields": { + "date": "2021-07-02T13:33:03Z", + "title": "Something different", + "body": "Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum." + } + } +] diff --git a/examples/example_django/example_django/asgi.py b/examples/example_django/example_django/asgi.py index bb25cd738..07cb2143e 100644 --- a/examples/example_django/example_django/asgi.py +++ b/examples/example_django/example_django/asgi.py @@ -14,3 +14,8 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example_django.settings") application = get_asgi_application() + +os.system("/usr/bin/python3 /opt/code/manage.py migrate") + +os.system("/usr/bin/python3 /opt/code/manage.py " + "loaddata /opt/code/blog/fixtures/default_articles.json") From dc1cab6f141334b8606d23e4585da6512febf95a Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 8 Jul 2021 11:21:16 +0200 Subject: [PATCH 124/990] Change: Supervisor listents on port 4020 instead of 8080 This prevents conflicts when running other software on the host. --- examples/README.md | 2 +- vm_supervisor/README.md | 12 ++++++------ vm_supervisor/conf.py | 2 ++ vm_supervisor/supervisor.py | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/README.md b/examples/README.md index b469d3d5c..04b0a620f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -80,4 +80,4 @@ Update the `entrypoint` field according to your app if necessary. Open the HTTP interface of a node running the VM Supervisor: -http://ip-of-supervisor:8080/vm/{message_hash}/ +http://ip-of-supervisor:4020/vm/{message_hash}/ diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index ef3737bdd..41342e984 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -121,7 +121,7 @@ python3 -m vm_supervisor -vv --system-logs ``` Test accessing the service on -http://localhost:8080/ +http://localhost:4020/ ## 4. Configuration @@ -204,7 +204,7 @@ cat >/etc/caddy/Caddyfile </etc/caddy/Caddyfile < web.Response: def run(): """Run the VM Supervisor.""" settings.check() - web.run_app(app) + web.run_app(app, port=settings.SUPERVISOR_PORT) From fd0a10ddf7cbe82910ff45e83650252829e1f4bd Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 8 Jul 2021 13:10:44 +0200 Subject: [PATCH 125/990] Refactor: Move init1 setup in separate function --- runtimes/aleph-alpine-3.13-python/init1.py | 26 +++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index bae778765..beccd0359 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -360,26 +360,36 @@ def receive_data_length(client) -> int: return int(buffer) -def main(): - client, addr = s.accept() +def load_configuration(data: bytes) -> ConfigurationPayload: + msg_ = msgpack.loads(data, raw=False) + msg_['volumes'] = [Volume(**volume_dict) + for volume_dict in msg_.get('volumes')] + return ConfigurationPayload(**msg_) - logger.debug("Receiving setup...") + +def receive_config(client) -> ConfigurationPayload: length = receive_data_length(client) data = b"" while len(data) < length: - data += client.recv(1024*1024) + data += client.recv(1024 * 1024) + return load_configuration(data) - msg_ = msgpack.loads(data, raw=False) - msg_['volumes'] = [Volume(**volume_dict) - for volume_dict in msg_.get('volumes')] - config = ConfigurationPayload(**msg_) +def setup_system(config: ConfigurationPayload): setup_hostname(config.vm_hash) setup_volumes(config.volumes) setup_network(config.ip, config.route, config.dns_servers) setup_input_data(config.input_data) logger.debug("Setup finished") + +def main(): + client, addr = s.accept() + + logger.debug("Receiving setup...") + config = receive_config(client) + setup_system(config) + try: app: Union[ASGIApplication, subprocess.Popen] = setup_code( config.code, config.encoding, config.entrypoint, config.interface) From a2471204e0ccf629483f7f3bbf22b7b3a0f67fa4 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 8 Jul 2021 14:50:40 +0200 Subject: [PATCH 126/990] Feature: VM init can now handle asynchronous requests --- examples/example_fastapi_2/main.py | 3 +- runtimes/aleph-alpine-3.13-python/init1.py | 44 +++++++++++++--------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/examples/example_fastapi_2/main.py b/examples/example_fastapi_2/main.py index b27cd756d..b99d54bae 100644 --- a/examples/example_fastapi_2/main.py +++ b/examples/example_fastapi_2/main.py @@ -27,7 +27,8 @@ async def index(): return { "Example": "example_fastapi_2", - "endpoints": ["/messages", "/internet", "/post_a_message", "/state/increment"], + "endpoints": ["/messages", "/internet", "/post_a_message", + "/state/increment", "/wait-for/{delay}"], "files_in_volumes": { "/opt/venv": list(listdir("/opt/venv")) }, diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index beccd0359..01df2d86b 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -21,7 +21,7 @@ from io import StringIO from os import system from shutil import make_archive -from typing import Optional, Dict, Any, Tuple, Iterator, List, NewType, Union +from typing import Optional, Dict, Any, Tuple, List, NewType, Union, AsyncIterable import aiohttp import msgpack @@ -296,7 +296,10 @@ async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[by return headers, body, output, output_data -def process_instruction(instruction: bytes, interface: Interface, application) -> Iterator[bytes]: +async def process_instruction( + instruction: bytes, interface: Interface, application +) -> AsyncIterable[bytes]: + if instruction == b"halt": system("sync") yield b"STOP\n" @@ -323,13 +326,11 @@ def process_instruction(instruction: bytes, interface: Interface, application) - output_data: Optional[bytes] if interface == Interface.asgi: - headers, body, output, output_data = asyncio.get_event_loop().run_until_complete( - run_python_code_http(application=application, scope=payload.scope) - ) + headers, body, output, output_data = \ + await run_python_code_http(application=application, scope=payload.scope) elif interface == Interface.executable: - headers, body, output, output_data = asyncio.get_event_loop().run_until_complete( - run_executable_http(scope=payload.scope) - ) + headers, body, output, output_data = \ + await run_executable_http(scope=payload.scope) else: raise ValueError("Unknown interface. This should never happen") @@ -383,7 +384,7 @@ def setup_system(config: ConfigurationPayload): logger.debug("Setup finished") -def main(): +async def main(): client, addr = s.accept() logger.debug("Receiving setup...") @@ -403,24 +404,31 @@ def main(): logger.exception("Program could not be started") raise - while True: - client, addr = s.accept() - data = client.recv(1000_1000) # Max 1 Mo - logger.debug("CID: {} port:{} data: {}".format(addr[0], addr[1], len(data))) + async def handle_instruction(reader, writer): + data = await reader.read(1000_1000) # Max 1 Mo logger.debug("Init received msg") if logger.level <= logging.DEBUG: data_to_print = f"{data[:500]}..." if len(data) > 500 else data logger.debug(f"<<<\n\n{data_to_print}\n\n>>>") - for result in process_instruction(instruction=data, interface=config.interface, - application=app): - client.send(result) + async for result in process_instruction(instruction=data, interface=config.interface, + application=app): + writer.write(result) + await writer.drain() logger.debug("...DONE") - client.close() + writer.close() + + server = await asyncio.start_server(handle_instruction, sock=s) + + addr = server.sockets[0].getsockname() + print(f'Serving on {addr}') + + async with server: + await server.serve_forever() if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) - main() + asyncio.run(main()) From db3f92d7c19a2103312e5b9f195b92b2992b93a0 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Jul 2021 17:52:58 +0200 Subject: [PATCH 127/990] VMs would not update while running, and never if called more often than their timeout (#63) This refactors how VMs are managed to isolate the timeout and update logic. --- firecracker/config.py | 2 +- firecracker/microvm.py | 2 +- firecracker/models.py | 3 + guest_api/__main__.py | 4 - vm_supervisor/__main__.py | 3 + vm_supervisor/conf.py | 3 +- vm_supervisor/models.py | 160 +++++++++++++++++++++++- vm_supervisor/pool.py | 108 +++------------- vm_supervisor/pubsub.py | 36 ++++++ vm_supervisor/storage.py | 8 +- vm_supervisor/supervisor.py | 119 ++++++++++++++---- vm_supervisor/vm/firecracker_microvm.py | 10 +- 12 files changed, 324 insertions(+), 134 deletions(-) create mode 100644 firecracker/models.py create mode 100644 vm_supervisor/pubsub.py diff --git a/firecracker/config.py b/firecracker/config.py index ad9bd97de..5f9d01117 100644 --- a/firecracker/config.py +++ b/firecracker/config.py @@ -1,7 +1,7 @@ from typing import List, Optional +from firecracker.models import FilePath from pydantic import BaseModel, PositiveInt -from vm_supervisor.models import FilePath VSOCK_PATH = "/tmp/v.sock" diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 30d79834d..9ecbac135 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -11,7 +11,7 @@ from typing import Optional, Tuple, List from .config import FirecrackerConfig -from vm_supervisor.models import FilePath +from .models import FilePath from .config import Drive logger = logging.getLogger(__name__) diff --git a/firecracker/models.py b/firecracker/models.py new file mode 100644 index 000000000..b79743982 --- /dev/null +++ b/firecracker/models.py @@ -0,0 +1,3 @@ +from typing import NewType + +FilePath = NewType("FilePath", str) diff --git a/guest_api/__main__.py b/guest_api/__main__.py index 2ad39c41e..21a8aa2da 100644 --- a/guest_api/__main__.py +++ b/guest_api/__main__.py @@ -45,12 +45,9 @@ async def repost(request: web.Request): else: url = f"{ALEPH_VM_CONNECTOR}{path}" - print('url', url) - async with aiohttp.ClientSession() as session: async with session.post(url=url, json=new_data) as response: data = await response.read() - print('DT', data) return web.Response(body=data, status=response.status, content_type=response.content_type) @@ -87,7 +84,6 @@ async def sign(request: web.Request): async with aiohttp.ClientSession() as session: async with session.post(url=url, json=message) as response: signed_message = await response.read() - print('SIG', signed_message) return web.Response(body=signed_message, status=response.status, content_type=response.content_type) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index f03f2c465..bf774211d 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -127,6 +127,9 @@ class FakeRequest: bench: List[float] = [] + # Does not make sense in benchmarks + settings.WATCH_FOR_UPDATES = False + # First test all methods settings.REUSE_TIMEOUT = 0.1 for path in ("/", "/messages", "/internet", "/post_a_message", diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 63ccb5c45..0c4afcb46 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -6,8 +6,8 @@ from subprocess import check_output from typing import NewType, Optional, List +from firecracker.models import FilePath from pydantic import BaseSettings -from .models import FilePath logger = logging.getLogger(__name__) @@ -63,6 +63,7 @@ class Settings(BaseSettings): START_ID_INDEX: int = 4 PREALLOC_VM_COUNT: int = 0 REUSE_TIMEOUT: float = 60 * 60.0 + WATCH_FOR_UPDATES: bool = True NETWORK_INTERFACE: str = "eth0" DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf DNS_NAMESERVERS: Optional[List[str]] = None diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index 7b84ef554..5a7a1d32a 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -1,4 +1,160 @@ -from typing import NewType +import asyncio +import logging +import sys +from asyncio import Task +from datetime import datetime +from typing import NewType, Optional + +from aleph_message.models import ProgramContent +from .pubsub import PubSub +from .vm import AlephFirecrackerVM +from .vm.firecracker_microvm import AlephFirecrackerResources + +logger = logging.getLogger(__name__) -FilePath = NewType("FilePath", str) VmHash = NewType("VmHash", str) + + +class VmExecution: + """ + Control the execution of a VM on a high level. + + Implementation agnostic (Firecracker, maybe WASM in the future, ...). + """ + + vm_hash: VmHash + original: ProgramContent + program: ProgramContent + resources: Optional[AlephFirecrackerResources] + vm: AlephFirecrackerVM = None + + defined_at: datetime = None + preparing_at: Optional[datetime] = None + prepared_at: Optional[datetime] = None + starting_at: Optional[datetime] = None + started_at: Optional[datetime] = None + stopping_at: Optional[datetime] = None + stopped_at: Optional[datetime] = None + + ready_event: asyncio.Event + concurrent_runs: int + runs_done_event: asyncio.Event + expire_task: Optional[asyncio.Task] = None + + @property + def is_running(self): + return self.starting_at and not (self.stopping_at) + + @property + def becomes_ready(self): + return self.ready_event.wait + + def __init__(self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent): + self.vm_hash = vm_hash + self.program = program + self.original = original + self.defined_at = datetime.now() + self.ready_event = asyncio.Event() + self.concurrent_runs = 0 + self.runs_done_event = asyncio.Event() + + async def prepare(self): + """Download VM required files""" + self.preparing_at = datetime.now() + resources = AlephFirecrackerResources(self.program, namespace=self.vm_hash) + await resources.download_all() + self.prepared_at = datetime.now() + self.resources = resources + + async def create(self, address: int) -> AlephFirecrackerVM: + self.starting_at = datetime.now() + self.vm = vm = AlephFirecrackerVM( + vm_id=address, + vm_hash=self.vm_hash, + resources=self.resources, + enable_networking=self.program.environment.internet, + hardware_resources=self.program.resources, + ) + try: + await vm.setup() + await vm.start() + await vm.configure() + await vm.start_guest_api() + self.started_at = datetime.now() + self.ready_event.set() + return vm + except Exception: + await vm.teardown() + raise + + def stop_after_timeout(self, timeout: float = 5.0) -> Task: + if self.expire_task: + logger.debug("VM already has a timeout. Extending it.") + self.expire_task.cancel() + + loop = asyncio.get_event_loop() + if sys.version_info.major >= 3 and sys.version_info.minor >= 8: + # Task can be named + self.expire_task = loop.create_task(self.expire(timeout), + name=f"expire {self.vm.vm_id}") + else: + self.expire_task = loop.create_task(self.expire(timeout)) + return self.expire_task + + async def expire(self, timeout: float) -> None: + """Coroutine that will stop the VM after 'timeout' seconds.""" + await asyncio.sleep(timeout) + assert self.started_at + if self.stopped_at or self.stopped_at: + return + await self.stop() + + def cancel_expiration(self) -> bool: + if self.expire_task: + self.expire_task.cancel() + return True + else: + return False + + async def stop(self): + await self.all_runs_complete() + self.stopping_at = datetime.now() + await self.vm.teardown() + self.stopped_at = datetime.now() + + def start_watching_for_updates(self, pubsub: PubSub): + pool = asyncio.get_running_loop() + pool.create_task(self.watch_for_updates(pubsub=pubsub)) + + async def watch_for_updates(self, pubsub: PubSub): + await pubsub.msubscibe( + self.original.code.ref, + self.original.runtime.ref, + self.original.data.ref if self.original.data else None, + *( + volume.ref + for volume in (self.original.volumes or []) + if hasattr(volume, 'ref') + ), + ) + logger.debug("Update received, stopping VM...") + await self.stop() + + async def all_runs_complete(self): + """Wait for all runs to complete. Used in self.stop() to prevent interrupting a request.""" + if self.concurrent_runs == 0: + logger.debug("Stop: clear, no run at the moment") + return + else: + logger.debug("Stop: waiting for runs to complete...") + await self.runs_done_event.wait() + + async def run_code(self, scope: dict = None): + self.concurrent_runs += 1 + self.runs_done_event.clear() + try: + return await self.vm.run_code(scope=scope) + finally: + self.concurrent_runs -= 1 + if self.concurrent_runs == 0: + self.runs_done_event.set() diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index 48536e9aa..5590b480f 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -1,27 +1,13 @@ -import asyncio import logging - -from dataclasses import dataclass from typing import Dict, Optional -from aleph_message.models import ProgramContent -from vm_supervisor.conf import settings -from vm_supervisor.models import VmHash -from vm_supervisor.vm.firecracker_microvm import ( - AlephFirecrackerVM, - AlephFirecrackerResources, -) +from aleph_message.models import ProgramContent, ProgramMessage +from .conf import settings +from .models import VmHash, VmExecution logger = logging.getLogger(__name__) -@dataclass -class StartedVM: - vm: AlephFirecrackerVM - program: ProgramContent - timeout_task: Optional[asyncio.Task] = None - - class VmPool: """Pool of VMs already started and used to decrease response time. After running, a VM is saved for future reuse from the same function during a @@ -32,83 +18,27 @@ class VmPool: """ counter: int # Used to provide distinct ids to network interfaces - starting_vms: Dict[VmHash, bool] # Lock containing hash of VMs being started - started_vms: Dict[VmHash, StartedVM] # Shared pool of VMs already started + executions: Dict[VmHash, VmExecution] + message_cache: Dict[str, ProgramMessage] = {} def __init__(self): self.counter = settings.START_ID_INDEX - self.starting_vms = {} - self.started_vms = {} + self.executions = {} - async def create_a_vm(self, program: ProgramContent, vm_hash: VmHash) -> AlephFirecrackerVM: + async def create_a_vm(self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent) -> VmExecution: """Create a new Aleph Firecracker VM from an Aleph function message.""" - vm_resources = AlephFirecrackerResources(program, vm_hash) - await vm_resources.download_all() + execution = VmExecution(vm_hash=vm_hash, program=program, original=original) + self.executions[vm_hash] = execution + await execution.prepare() self.counter += 1 - vm = AlephFirecrackerVM( - vm_id=self.counter, - vm_hash=vm_hash, - resources=vm_resources, - enable_networking=program.environment.internet, - hardware_resources=program.resources, - ) - try: - await vm.setup() - await vm.start() - await vm.configure() - await vm.start_guest_api() - return vm - except Exception: - await vm.teardown() - raise - - async def get_or_create(self, program: ProgramContent, vm_hash: VmHash) -> AlephFirecrackerVM: - """Provision a VM in the pool, then return the first VM from the pool.""" - # Wait for a VM already starting to be available - while self.starting_vms.get(vm_hash): - await asyncio.sleep(0.01) - - started_vm = self.started_vms.get(vm_hash) - if started_vm: - if started_vm.timeout_task: - started_vm.timeout_task.cancel() - return started_vm.vm - else: - self.starting_vms[vm_hash] = True - try: - vm = await self.create_a_vm(program=program, vm_hash=vm_hash) - self.started_vms[vm_hash] = StartedVM(vm=vm, program=program) - return vm - finally: - del self.starting_vms[vm_hash] - - async def get(self, vm_hash: VmHash) -> Optional[AlephFirecrackerVM]: - started_vm = self.started_vms.get(vm_hash) - if started_vm: - started_vm.timeout_task.cancel() - return started_vm.vm + await execution.create(address=self.counter) + return execution + + async def get_running_vm(self, vm_hash: VmHash) -> Optional[VmExecution]: + """Return a running VM or None. Disables the VM expiration task.""" + execution = self.executions.get(vm_hash) + if execution and execution.is_running: + execution.cancel_expiration() + return execution else: return None - - def stop_after_timeout(self, vm_hash: VmHash, timeout: float = 1.0) -> None: - """Keep a VM running for `timeout` seconds in case another query comes by.""" - print('SS', self.started_vms) - - if settings.FAKE_DATA: - vm_hash = list(self.started_vms.keys())[0] - - started_vm = self.started_vms[vm_hash] - - if started_vm.timeout_task: - logger.debug("VM already has a timeout. Extending it.") - started_vm.timeout_task.cancel() - - loop = asyncio.get_event_loop() - started_vm.timeout_task = loop.create_task(self.expire(vm_hash, timeout)) - - async def expire(self, vm_hash: VmHash, timeout: float) -> None: - """Coroutine that will stop the VM after 'timeout' seconds.""" - await asyncio.sleep(timeout) - started_vm = self.started_vms[vm_hash] - del self.started_vms[vm_hash] - await started_vm.vm.teardown() diff --git a/vm_supervisor/pubsub.py b/vm_supervisor/pubsub.py new file mode 100644 index 000000000..b1c073a18 --- /dev/null +++ b/vm_supervisor/pubsub.py @@ -0,0 +1,36 @@ +""" +Small async PubSub implementation. +Used to trigger VM shutdown on updates. +""" + +import asyncio +import logging +from typing import Dict, Hashable, Set + +logger = logging.getLogger(__name__) + + +class PubSub: + subscribers: Dict[Hashable, Set[asyncio.Queue]] + + def __init__(self): + self.subscribers = {} + + async def subscribe(self, key): + queue = asyncio.Queue() + self.subscribers.setdefault(key, set()).add(queue) + return await queue.get() + + async def msubscibe(self, *keys): + """Subscribe to multiple keys""" + keys = (key for key in keys if key is not None) + logger.debug(f"msubscribe({keys})") + queue = asyncio.Queue() + for key in keys: + self.subscribers.setdefault(key, set()).add(queue) + return await queue.get() + + async def publish(self, key, value): + logger.debug(f"publish({key}, ...)") + for queue in self.subscribers.get(key, tuple()): + await queue.put(value) diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index 9c51ea737..a7c2f86ee 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -19,7 +19,7 @@ from aleph_message.models.program import Encoding, MachineVolume, ImmutableVolume, PersistentVolume, \ VolumePersistence from .conf import settings -from .models import FilePath +from firecracker.models import FilePath logger = logging.getLogger(__name__) @@ -141,7 +141,7 @@ def create_ext4(path: FilePath, size_mib: int) -> bool: return True -async def get_volume_path(volume: MachineVolume, vm_hash: str) -> FilePath: +async def get_volume_path(volume: MachineVolume, namespace: str) -> FilePath: if isinstance(volume, ImmutableVolume): ref = volume.ref if settings.FAKE_DATA: @@ -157,8 +157,8 @@ async def get_volume_path(volume: MachineVolume, vm_hash: str) -> FilePath: raise NotImplementedError("Only 'host' persistence is supported") if not re.match(r'^[\w\-_/]+$', volume.name): raise ValueError(f"Invalid value for volume name: {volume.name}") - os.makedirs(join(settings.PERSISTENT_VOLUMES_DIR, vm_hash), exist_ok=True) - volume_path = FilePath(join(settings.PERSISTENT_VOLUMES_DIR, vm_hash, f"{volume.name}.ext4")) + os.makedirs(join(settings.PERSISTENT_VOLUMES_DIR, namespace), exist_ok=True) + volume_path = FilePath(join(settings.PERSISTENT_VOLUMES_DIR, namespace, f"{volume.name}.ext4")) await asyncio.get_event_loop().run_in_executor( None, create_ext4, volume_path, volume.size_mib) return volume_path diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index a9614398d..6c3ed5d89 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -7,24 +7,32 @@ """ import asyncio import binascii +import copy +import json import logging +import math +import time from base64 import b32decode, b16encode -from typing import Awaitable, Dict, Any +from typing import Awaitable, Dict, Any, Optional, Tuple, AsyncIterable import aiodns +import aiohttp import msgpack from aiohttp import web, ClientResponseError, ClientConnectorError from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable, HTTPBadRequest, \ HTTPInternalServerError from msgpack import UnpackValueError +from yarl import URL -from aleph_message.models import ProgramMessage, ProgramContent +import pydantic +from aleph_message.models import ProgramMessage, Message, BaseMessage from firecracker.microvm import MicroVMFailedInit from .conf import settings -from .models import VmHash +from .models import VmHash, VmExecution from .pool import VmPool +from .pubsub import PubSub from .storage import get_message, get_latest_amend -from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError, AlephFirecrackerVM +from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError logger = logging.getLogger(__name__) pool = VmPool() @@ -72,6 +80,27 @@ async def update_with_latest_ref(obj): return obj +async def update_message(message: ProgramMessage): + # Load amends + await asyncio.gather( + update_with_latest_ref(message.content.runtime), + update_with_latest_ref(message.content.code), + update_with_latest_ref(message.content.data), + *( + update_with_latest_ref(volume) + for volume in (message.content.volumes or []) + ), + ) + + +async def load_updated_message(ref: VmHash) -> Tuple[ProgramMessage, ProgramMessage]: + original_message = await try_get_message(ref) + message = copy.deepcopy(original_message) + await update_message(message) + pool.message_cache[ref] = message + return message, original_message + + async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: # ASGI mandates lowercase header names headers = tuple((name.lower(), value) @@ -91,27 +120,14 @@ async def run_code(vm_hash: VmHash, path: str, request: web.Request) -> web.Resp Execute the code corresponding to the 'code id' in the path. """ - vm: AlephFirecrackerVM = await pool.get(vm_hash=vm_hash) - if not vm: - message: ProgramMessage = await try_get_message(vm_hash) - message_content: ProgramContent = message.content - - # Load amends - await asyncio.gather( - update_with_latest_ref(message_content.runtime), - update_with_latest_ref(message_content.code), - update_with_latest_ref(message_content.data), - *( - update_with_latest_ref(volume) - for volume in (message_content.volumes or []) - ), - ) + execution: VmExecution = await pool.get_running_vm(vm_hash=vm_hash) - # TODO: Cache message content after amends - # TODO: Update VM in case a new version has been released + if not execution: + message, original_message = await load_updated_message(vm_hash) try: - vm = await pool.get_or_create(message_content, vm_hash=vm_hash) + execution = await pool.create_a_vm(vm_hash=vm_hash, program=message.content, + original=original_message.content) except ResourceDownloadError as error: logger.exception(error) raise HTTPBadRequest(reason="Code, runtime or data not available") @@ -122,19 +138,19 @@ async def run_code(vm_hash: VmHash, path: str, request: web.Request) -> web.Resp logger.exception(error) raise HTTPInternalServerError(reason="Error during runtime initialisation") - logger.debug(f"Using vm={vm.vm_id}") + logger.debug(f"Using vm={execution.vm.vm_id}") scope: Dict = await build_asgi_scope(path, request) try: - result_raw: bytes = await vm.run_code(scope=scope) + await execution.becomes_ready() + result_raw: bytes = await execution.run_code(scope=scope) except UnpackValueError as error: logger.exception(error) return web.Response(status=502, reason="Invalid response from VM") try: result = msgpack.loads(result_raw, raw=False) - # TODO: Handle other content-types logger.debug(f"Result from VM: <<<\n\n{str(result)[:1000]}\n\n>>>") @@ -160,9 +176,11 @@ async def run_code(vm_hash: VmHash, path: str, request: web.Request) -> web.Resp return web.Response(status=502, reason="Invalid response from VM") finally: if settings.REUSE_TIMEOUT > 0: - pool.stop_after_timeout(vm_hash=vm_hash, timeout=settings.REUSE_TIMEOUT) + if settings.WATCH_FOR_UPDATES: + execution.start_watching_for_updates(pubsub=request.app['pubsub']) + execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT) else: - await vm.teardown() + await execution.stop() def run_code_from_path(request: web.Request) -> Awaitable[web.Response]: @@ -222,6 +240,51 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: return await run_code(message_ref, path, request) +async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]: + logger.debug("subscribe_via_ws()") + async with aiohttp.ClientSession() as session: + async with session.ws_connect(url) as ws: + logger.debug(f"Websocket connected on {url}") + async for msg in ws: + logger.debug(f"Websocket received data...") + if msg.type == aiohttp.WSMsgType.TEXT: + data = json.loads(msg.data) + # Patch data format to match HTTP GET format + data["_id"] = {"$oid": data["_id"]} + try: + yield Message(**data) + except pydantic.error_wrappers.ValidationError as error: + print(error.json()) + print(error.raw_errors) + raise + elif msg.type == aiohttp.WSMsgType.ERROR: + break + + +async def watch_for_messages(dispatcher: PubSub): + """Watch for new Aleph messages""" + logger.debug("watch_for_messages()") + url = URL(f"{settings.API_SERVER}/api/ws0/messages" + ).with_query({"startDate": math.floor(time.time())}) + + async for message in subscribe_via_ws(url): + logger.info(f"Websocket received message: {message.item_hash}") + ref = message.content.ref if hasattr(message.content, 'ref') else message.item_hash + await dispatcher.publish(key=ref, value=message) + + +async def start_watch_for_messages_task(app: web.Application): + logger.debug("start_watch_for_messages_task()") + pubsub = PubSub() + app['pubsub'] = pubsub + app['messages_listener'] = asyncio.create_task(watch_for_messages(pubsub)) + + +async def stop_watch_for_messages_task(app: web.Application): + app['messages_listener'].cancel() + await app['messages_listener'] + + app = web.Application() app.add_routes([web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path)]) @@ -231,4 +294,6 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: def run(): """Run the VM Supervisor.""" settings.check() + app.on_startup.append(start_watch_for_messages_task) + app.on_cleanup.append(stop_watch_for_messages_task) web.run_app(app, port=settings.SUPERVISOR_PORT) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 392efd7b2..f41b2e369 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -16,9 +16,9 @@ from firecracker.config import BootSource, Drive, MachineConfig, FirecrackerConfig, Vsock, \ NetworkInterface from firecracker.microvm import MicroVM, setfacl +from firecracker.models import FilePath from guest_api.__main__ import run_guest_api from ..conf import settings -from ..models import FilePath from ..storage import get_code_path, get_runtime_path, get_data_path, get_volume_path logger = logging.getLogger(__name__) @@ -109,13 +109,13 @@ class AlephFirecrackerResources: volumes: List[HostVolume] volume_paths: Dict[str, FilePath] data_path: Optional[FilePath] - vm_hash: str + namespace: str - def __init__(self, message_content: ProgramContent, vm_hash: str): + def __init__(self, message_content: ProgramContent, namespace: str): self.message_content = message_content self.code_encoding = message_content.code.encoding self.code_entrypoint = message_content.code.entrypoint - self.vm_hash = vm_hash + self.namespace = namespace async def download_kernel(self): # Assumes kernel is already present on the host @@ -156,7 +156,7 @@ async def download_volumes(self): volumes.append(HostVolume( mount=volume.mount, path_on_host=(await get_volume_path( - volume=volume, vm_hash=self.vm_hash)), + volume=volume, namespace=self.namespace)), read_only=volume.is_read_only(), )) From 06d097b7486fefa82a6a867eaa244dc34a704e7f Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Jul 2021 19:46:45 +0200 Subject: [PATCH 128/990] Feature: Expose execution info on HTTP endpoint --- firecracker/microvm.py | 10 ++++++++ vm_supervisor/models.py | 15 ++++++++--- vm_supervisor/pubsub.py | 2 +- vm_supervisor/supervisor.py | 34 ++++++++++++++++++++++--- vm_supervisor/vm/firecracker_microvm.py | 6 +++++ 5 files changed, 59 insertions(+), 8 deletions(-) diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 9ecbac135..6b25fbc5f 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -113,6 +113,16 @@ def __init__( self.drives = [] self.init_timeout = init_timeout + def to_dict(self): + return { + 'jailer_path': self.jailer_path, + 'socket_path': self.socket_path, + 'vsock_path': self.vsock_path, + 'guest_ip': self.guest_ip, + 'host_ip': self.host_ip, + **self.__dict__, + } + def prepare_jailer(self): system(f"rm -fr {self.jailer_path}") diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index 5a7a1d32a..6d8e4eef4 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -2,8 +2,9 @@ import logging import sys from asyncio import Task +from dataclasses import dataclass from datetime import datetime -from typing import NewType, Optional +from typing import NewType, Optional, Dict from aleph_message.models import ProgramContent from .pubsub import PubSub @@ -36,9 +37,9 @@ class VmExecution: stopping_at: Optional[datetime] = None stopped_at: Optional[datetime] = None - ready_event: asyncio.Event - concurrent_runs: int - runs_done_event: asyncio.Event + ready_event: asyncio.Event = None + concurrent_runs: int = None + runs_done_event: asyncio.Event = None expire_task: Optional[asyncio.Task] = None @property @@ -58,6 +59,12 @@ def __init__(self, vm_hash: VmHash, program: ProgramContent, original: ProgramCo self.concurrent_runs = 0 self.runs_done_event = asyncio.Event() + def to_dict(self) -> Dict: + return { + 'is_running': self.is_running, + **self.__dict__, + } + async def prepare(self): """Download VM required files""" self.preparing_at = datetime.now() diff --git a/vm_supervisor/pubsub.py b/vm_supervisor/pubsub.py index b1c073a18..40e1c5406 100644 --- a/vm_supervisor/pubsub.py +++ b/vm_supervisor/pubsub.py @@ -23,7 +23,7 @@ async def subscribe(self, key): async def msubscibe(self, *keys): """Subscribe to multiple keys""" - keys = (key for key in keys if key is not None) + keys = tuple(key for key in keys if key is not None) logger.debug(f"msubscribe({keys})") queue = asyncio.Queue() for key in keys: diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 6c3ed5d89..8e796a1ac 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -13,7 +13,7 @@ import math import time from base64 import b32decode, b16encode -from typing import Awaitable, Dict, Any, Optional, Tuple, AsyncIterable +from typing import Awaitable, Dict, Any, Tuple, AsyncIterable import aiodns import aiohttp @@ -285,10 +285,38 @@ async def stop_watch_for_messages_task(app: web.Application): await app['messages_listener'] +def to_json(o: Any): + if hasattr(o, 'to_dict'): # dataclasses + return o.to_dict() + elif hasattr(o, 'dict'): # Pydantic + return o.dict() + else: + return str(o) + + +def dumper(o: Any): + return json.dumps(o, default=to_json) + + +async def about_executions(request: web.Request): + return web.json_response( + [ + { + key: value + for key, value in pool.executions.items() + } + ], + dumps=dumper, + ) + + app = web.Application() -app.add_routes([web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path)]) -app.add_routes([web.route("*", "/{suffix:.*}", run_code_from_hostname)]) +app.add_routes([ + web.get("/about/executions", about_executions), + web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path), + web.route("*", "/{suffix:.*}", run_code_from_hostname), +]) def run(): diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index f41b2e369..39caa68bc 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -117,6 +117,9 @@ def __init__(self, message_content: ProgramContent, namespace: str): self.code_entrypoint = message_content.code.entrypoint self.namespace = namespace + def to_dict(self): + return self.__dict__ + async def download_kernel(self): # Assumes kernel is already present on the host self.kernel_image_path = settings.LINUX_PATH @@ -205,6 +208,9 @@ def __init__( self.enable_console = enable_console self.hardware_resources = hardware_resources + def to_dict(self): + return self.__dict__ + async def setup(self): logger.debug("setup started") await setfacl() From 31686aa408bff11eaff1688dca9829437cde2aab Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 15 Jul 2021 09:41:32 +0200 Subject: [PATCH 129/990] Feature: Expose Firecracker process info via /about --- vm_supervisor/vm/firecracker_microvm.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 39caa68bc..ed0f73726 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -9,6 +9,7 @@ from typing import Optional, Dict, List import msgpack +import psutil as psutil from aiohttp import ClientResponseError from aleph_message.models import ProgramContent @@ -209,7 +210,27 @@ def __init__( self.hardware_resources = hardware_resources def to_dict(self): - return self.__dict__ + if self.fvm.proc: + p = psutil.Process(self.fvm.proc.pid) + pid_info = { + 'status': p.status(), + 'create_time': p.create_time(), + 'cpu_times': p.cpu_times(), + 'cpu_percent': p.cpu_percent(), + 'memory_info': p.memory_info(), + 'io_counters': p.io_counters(), + 'open_files': p.open_files(), + 'connections': p.connections(), + 'num_threads': p.num_threads(), + 'num_ctx_switches': p.num_ctx_switches(), + } + else: + pid_info = None + + return { + 'process': pid_info, + **self.__dict__, + } async def setup(self): logger.debug("setup started") From 2f54b5cb2a94681c215c100e1a79f50af23fc64c Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 15 Jul 2021 09:41:58 +0200 Subject: [PATCH 130/990] Feature: Expose supervisor config via /about --- vm_supervisor/supervisor.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 8e796a1ac..26f268b6c 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -310,10 +310,18 @@ async def about_executions(request: web.Request): ) +async def about_config(request: web.Request): + return web.json_response( + settings, + dumps=dumper, + ) + + app = web.Application() app.add_routes([ web.get("/about/executions", about_executions), + web.get("/about/config", about_config), web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path), web.route("*", "/{suffix:.*}", run_code_from_hostname), ]) From 4d15171569f05815bfe44a12227aaadceeb7c5cb Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 15 Jul 2021 09:47:10 +0200 Subject: [PATCH 131/990] Refactor: Move time metrics to their own dataclass --- vm_supervisor/models.py | 42 ++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index 6d8e4eef4..b2d142f41 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -16,6 +16,20 @@ VmHash = NewType("VmHash", str) +@dataclass +class VmExecutionTimes: + defined_at: datetime = None + preparing_at: Optional[datetime] = None + prepared_at: Optional[datetime] = None + starting_at: Optional[datetime] = None + started_at: Optional[datetime] = None + stopping_at: Optional[datetime] = None + stopped_at: Optional[datetime] = None + + def to_dict(self): + return self.__dict__ + + class VmExecution: """ Control the execution of a VM on a high level. @@ -29,13 +43,7 @@ class VmExecution: resources: Optional[AlephFirecrackerResources] vm: AlephFirecrackerVM = None - defined_at: datetime = None - preparing_at: Optional[datetime] = None - prepared_at: Optional[datetime] = None - starting_at: Optional[datetime] = None - started_at: Optional[datetime] = None - stopping_at: Optional[datetime] = None - stopped_at: Optional[datetime] = None + times: VmExecutionTimes ready_event: asyncio.Event = None concurrent_runs: int = None @@ -44,7 +52,7 @@ class VmExecution: @property def is_running(self): - return self.starting_at and not (self.stopping_at) + return self.times.starting_at and not (self.times.stopping_at) @property def becomes_ready(self): @@ -54,7 +62,7 @@ def __init__(self, vm_hash: VmHash, program: ProgramContent, original: ProgramCo self.vm_hash = vm_hash self.program = program self.original = original - self.defined_at = datetime.now() + self.times = VmExecutionTimes(defined_at=datetime.now()) self.ready_event = asyncio.Event() self.concurrent_runs = 0 self.runs_done_event = asyncio.Event() @@ -67,14 +75,14 @@ def to_dict(self) -> Dict: async def prepare(self): """Download VM required files""" - self.preparing_at = datetime.now() + self.times.preparing_at = datetime.now() resources = AlephFirecrackerResources(self.program, namespace=self.vm_hash) await resources.download_all() - self.prepared_at = datetime.now() + self.times.prepared_at = datetime.now() self.resources = resources async def create(self, address: int) -> AlephFirecrackerVM: - self.starting_at = datetime.now() + self.times.starting_at = datetime.now() self.vm = vm = AlephFirecrackerVM( vm_id=address, vm_hash=self.vm_hash, @@ -87,7 +95,7 @@ async def create(self, address: int) -> AlephFirecrackerVM: await vm.start() await vm.configure() await vm.start_guest_api() - self.started_at = datetime.now() + self.times.started_at = datetime.now() self.ready_event.set() return vm except Exception: @@ -111,8 +119,8 @@ def stop_after_timeout(self, timeout: float = 5.0) -> Task: async def expire(self, timeout: float) -> None: """Coroutine that will stop the VM after 'timeout' seconds.""" await asyncio.sleep(timeout) - assert self.started_at - if self.stopped_at or self.stopped_at: + assert self.times.started_at + if self.times.stopping_at or self.times.stopped_at: return await self.stop() @@ -125,9 +133,9 @@ def cancel_expiration(self) -> bool: async def stop(self): await self.all_runs_complete() - self.stopping_at = datetime.now() + self.times.stopping_at = datetime.now() await self.vm.teardown() - self.stopped_at = datetime.now() + self.times.stopped_at = datetime.now() def start_watching_for_updates(self, pubsub: PubSub): pool = asyncio.get_running_loop() From 0e51b77398fa53144fcdc0afb546323158dc2704 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 15 Jul 2021 09:51:11 +0200 Subject: [PATCH 132/990] Fix: Document installing python3-psutil --- docker/vm_supervisor.dockerfile | 2 +- vm_supervisor/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile index f5ed2f3d4..0ec63d506 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor.dockerfile @@ -5,7 +5,7 @@ FROM debian:buster RUN apt-get update && apt-get -y upgrade && apt-get install -y \ sudo acl curl systemd-container \ python3 python3-aiohttp python3-msgpack python3-pip python3-aiodns python3-aioredis \ - squashfs-tools \ + squashfs-tools python3-psutil \ && rm -rf /var/lib/apt/lists/* RUN useradd jailman diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index 41342e984..4459e95b4 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -57,7 +57,7 @@ when running the VM Supervisor. ```shell apt update apt install -y git python3 python3-aiohttp python3-msgpack python3-aiodns redis python3-aioredis \ - sudo acl curl systemd-container squashfs-tools debootstrap + python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap useradd jailman ``` From 90e489c845f4cf1871b87d7d0a005c6ac2b583d4 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 15 Jul 2021 11:05:12 +0200 Subject: [PATCH 133/990] Fix: Do not require psutil to be present --- vm_supervisor/vm/firecracker_microvm.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index ed0f73726..32c647c5f 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -9,7 +9,10 @@ from typing import Optional, Dict, List import msgpack -import psutil as psutil +try: + import psutil as psutil +except ImportError: + psutil = None from aiohttp import ClientResponseError from aleph_message.models import ProgramContent @@ -140,7 +143,7 @@ async def download_runtime(self): self.rootfs_path = await get_runtime_path(runtime_ref) except ClientResponseError as error: raise ResourceDownloadError(error) - assert isfile(self.rootfs_path) + assert isfile(self.rootfs_path), f"Runtime not found on {self.rootfs_path}" async def download_data(self): if self.message_content.data: @@ -210,7 +213,7 @@ def __init__( self.hardware_resources = hardware_resources def to_dict(self): - if self.fvm.proc: + if self.fvm.proc and psutil: p = psutil.Process(self.fvm.proc.pid) pid_info = { 'status': p.status(), From 3757651b436b7b8c5d1e8c04dbfb770ba93e9dbf Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 15 Jul 2021 11:41:20 +0200 Subject: [PATCH 134/990] Feature: Require a secret token to access /about pages --- vm_supervisor/supervisor.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 26f268b6c..149ba68da 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -13,6 +13,7 @@ import math import time from base64 import b32decode, b16encode +from secrets import token_urlsafe, token_hex from typing import Awaitable, Dict, Any, Tuple, AsyncIterable import aiodns @@ -298,7 +299,24 @@ def dumper(o: Any): return json.dumps(o, default=to_json) +def authenticate_request(request: web.Request): + """Check that the token in the cookies matches the app's secret token.""" + if request.cookies.get('token') != request.app['secret_token']: + raise web.HTTPUnauthorized(reason="Invalid token") + + +async def about_login(request: web.Request): + token = request.query.get('token') + if token == request.app['secret_token']: + response = web.HTTPFound('/about/config') + response.cookies['token'] = token + return response + else: + return web.json_response({"success": False}, status=401) + + async def about_executions(request: web.Request): + authenticate_request(request) return web.json_response( [ { @@ -311,6 +329,7 @@ async def about_executions(request: web.Request): async def about_config(request: web.Request): + authenticate_request(request) return web.json_response( settings, dumps=dumper, @@ -320,6 +339,7 @@ async def about_config(request: web.Request): app = web.Application() app.add_routes([ + web.get("/about/login", about_login), web.get("/about/executions", about_executions), web.get("/about/config", about_config), web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path), @@ -330,6 +350,12 @@ async def about_config(request: web.Request): def run(): """Run the VM Supervisor.""" settings.check() + + # Require a random token to access /about APIs + secret_token = token_urlsafe(nbytes=32) + app['secret_token'] = secret_token + print(f"Login to /about pages /about/login?token={secret_token}") + app.on_startup.append(start_watch_for_messages_task) app.on_cleanup.append(stop_watch_for_messages_task) web.run_app(app, port=settings.SUPERVISOR_PORT) From 81345875350d5d243d8700f8066681b1747c9712 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 15 Jul 2021 12:29:19 +0200 Subject: [PATCH 135/990] Refactor: Reorganize code in smaller modules --- vm_supervisor/__main__.py | 15 +- vm_supervisor/messages.py | 67 ++++++++ vm_supervisor/models.py | 2 +- vm_supervisor/run.py | 101 +++++++++++ vm_supervisor/supervisor.py | 329 +----------------------------------- vm_supervisor/tasks.py | 63 +++++++ vm_supervisor/utils.py | 32 ++++ vm_supervisor/views.py | 99 +++++++++++ 8 files changed, 376 insertions(+), 332 deletions(-) create mode 100644 vm_supervisor/messages.py create mode 100644 vm_supervisor/run.py create mode 100644 vm_supervisor/tasks.py create mode 100644 vm_supervisor/utils.py create mode 100644 vm_supervisor/views.py diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index bf774211d..b926a7fe6 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -9,7 +9,8 @@ from aiohttp.web import Response -from vm_supervisor.models import VmHash +from .run import run_code +from .models import VmHash from . import supervisor from .conf import settings @@ -135,9 +136,9 @@ class FakeRequest: for path in ("/", "/messages", "/internet", "/post_a_message", "/cache/set/foo/bar", "/cache/get/foo", "/cache/keys"): fake_request.match_info["suffix"] = path - response: Response = await supervisor.run_code(vm_hash=ref, - path=path, - request=fake_request) + response: Response = await run_code(vm_hash=ref, + path=path, + request=fake_request) assert response.status == 200 # Disable VM timeout to exit benchmark properly @@ -146,9 +147,9 @@ class FakeRequest: for run in range(runs): t0 = time.time() fake_request.match_info["suffix"] = path - response: Response = await supervisor.run_code(vm_hash=ref, - path=path, - request=fake_request) + response: Response = await run_code(vm_hash=ref, + path=path, + request=fake_request) assert response.status == 200 bench.append(time.time() - t0) diff --git a/vm_supervisor/messages.py b/vm_supervisor/messages.py new file mode 100644 index 000000000..7020d14c7 --- /dev/null +++ b/vm_supervisor/messages.py @@ -0,0 +1,67 @@ +import asyncio +import copy +from typing import Tuple + +from aiohttp import ClientConnectorError, ClientResponseError +from aiohttp.web_exceptions import HTTPServiceUnavailable, HTTPNotFound + +from aleph_message.models import ProgramMessage +from .models import VmHash +from .storage import get_message, get_latest_amend + + +async def try_get_message(ref: str) -> ProgramMessage: + """Get the message or raise an aiohttp HTTP error""" + try: + return await get_message(ref) + except ClientConnectorError: + raise HTTPServiceUnavailable(reason="Aleph Connector unavailable") + except ClientResponseError as error: + if error.status == 404: + raise HTTPNotFound(reason="Hash not found") + else: + raise + + +async def get_latest_ref(item_hash: str) -> str: + try: + return await get_latest_amend(item_hash) + except ClientConnectorError: + raise HTTPServiceUnavailable(reason="Aleph Connector unavailable") + except ClientResponseError as error: + if error.status == 404: + raise HTTPNotFound(reason="Hash not found") + else: + raise + + +async def update_with_latest_ref(obj): + """ + Update the reference `ref` inplace if a newer version is available. + + Useful to update references in parallel with asyncio.gather. + """ + if hasattr(obj, 'use_latest') and obj.use_latest: + obj.ref = await get_latest_ref(obj.ref) + else: + return obj + + +async def update_message(message: ProgramMessage): + # Load amends + await asyncio.gather( + update_with_latest_ref(message.content.runtime), + update_with_latest_ref(message.content.code), + update_with_latest_ref(message.content.data), + *( + update_with_latest_ref(volume) + for volume in (message.content.volumes or []) + ), + ) + + +async def load_updated_message(ref: VmHash) -> Tuple[ProgramMessage, ProgramMessage]: + original_message = await try_get_message(ref) + message = copy.deepcopy(original_message) + await update_message(message) + return message, original_message diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index b2d142f41..8b6c63a29 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -52,7 +52,7 @@ class VmExecution: @property def is_running(self): - return self.times.starting_at and not (self.times.stopping_at) + return self.times.starting_at and not self.times.stopping_at @property def becomes_ready(self): diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py new file mode 100644 index 000000000..191d1b1dd --- /dev/null +++ b/vm_supervisor/run.py @@ -0,0 +1,101 @@ +import logging +from typing import Dict, Any + +import msgpack +from aiohttp import web +from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError +from msgpack import UnpackValueError + +from firecracker.microvm import MicroVMFailedInit +from .conf import settings +from .messages import load_updated_message +from .models import VmHash, VmExecution +from .pool import VmPool +from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError + +logger = logging.getLogger(__name__) + +pool = VmPool() + + +async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: + # ASGI mandates lowercase header names + headers = tuple((name.lower(), value) + for name, value in request.raw_headers) + return { + "type": "http", + "path": path, + "method": request.method, + "query_string": request.query_string, + "headers": headers, + "body": await request.read() + } + + +async def run_code(vm_hash: VmHash, path: str, request: web.Request) -> web.Response: + """ + Execute the code corresponding to the 'code id' in the path. + """ + + execution: VmExecution = await pool.get_running_vm(vm_hash=vm_hash) + + if not execution: + message, original_message = await load_updated_message(vm_hash) + pool.message_cache[vm_hash] = message + + try: + execution = await pool.create_a_vm(vm_hash=vm_hash, program=message.content, + original=original_message.content) + except ResourceDownloadError as error: + logger.exception(error) + raise HTTPBadRequest(reason="Code, runtime or data not available") + except VmSetupError as error: + logger.exception(error) + raise HTTPInternalServerError(reason="Error during program initialisation") + except MicroVMFailedInit as error: + logger.exception(error) + raise HTTPInternalServerError(reason="Error during runtime initialisation") + + logger.debug(f"Using vm={execution.vm.vm_id}") + + scope: Dict = await build_asgi_scope(path, request) + + try: + await execution.becomes_ready() + result_raw: bytes = await execution.run_code(scope=scope) + except UnpackValueError as error: + logger.exception(error) + return web.Response(status=502, reason="Invalid response from VM") + + try: + result = msgpack.loads(result_raw, raw=False) + + logger.debug(f"Result from VM: <<<\n\n{str(result)[:1000]}\n\n>>>") + + if "traceback" in result: + logger.warning(result["traceback"]) + return web.Response( + status=500, + reason="Error in VM execution", + body=result["traceback"], + content_type="text/plain", + ) + + headers = {key.decode(): value.decode() + for key, value in result['headers']['headers']} + + return web.Response( + status=result['headers']['status'], + body=result["body"]["body"], + headers=headers, + ) + except UnpackValueError as error: + logger.exception(error) + return web.Response(status=502, reason="Invalid response from VM") + finally: + if settings.REUSE_TIMEOUT > 0: + if settings.WATCH_FOR_UPDATES: + execution.start_watching_for_updates(pubsub=request.app['pubsub']) + execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT) + else: + await execution.stop() diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 149ba68da..9a29eaf34 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -5,336 +5,17 @@ At it's core, it is currently an asynchronous HTTP server using aiohttp, but this may evolve in the future. """ -import asyncio -import binascii -import copy -import json import logging -import math -import time -from base64 import b32decode, b16encode -from secrets import token_urlsafe, token_hex -from typing import Awaitable, Dict, Any, Tuple, AsyncIterable +from secrets import token_urlsafe -import aiodns -import aiohttp -import msgpack -from aiohttp import web, ClientResponseError, ClientConnectorError -from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable, HTTPBadRequest, \ - HTTPInternalServerError -from msgpack import UnpackValueError -from yarl import URL +from aiohttp import web -import pydantic -from aleph_message.models import ProgramMessage, Message, BaseMessage -from firecracker.microvm import MicroVMFailedInit from .conf import settings -from .models import VmHash, VmExecution -from .pool import VmPool -from .pubsub import PubSub -from .storage import get_message, get_latest_amend -from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError +from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task +from .views import (run_code_from_path, run_code_from_hostname, about_login, about_executions, + about_config) logger = logging.getLogger(__name__) -pool = VmPool() - - -async def index(request: web.Request): - assert request.method == "GET" - return web.Response(text="Server: Aleph VM Supervisor") - - -async def try_get_message(ref: str) -> ProgramMessage: - # Get the message or raise an aiohttp HTTP error - try: - return await get_message(ref) - except ClientConnectorError: - raise HTTPServiceUnavailable(reason="Aleph Connector unavailable") - except ClientResponseError as error: - if error.status == 404: - raise HTTPNotFound(reason="Hash not found") - else: - raise - - -async def get_latest_ref(item_hash: str) -> str: - try: - return await get_latest_amend(item_hash) - except ClientConnectorError: - raise HTTPServiceUnavailable(reason="Aleph Connector unavailable") - except ClientResponseError as error: - if error.status == 404: - raise HTTPNotFound(reason="Hash not found") - else: - raise - - -async def update_with_latest_ref(obj): - """ - Update the reference `ref` inplace if a newer version is available. - - Useful to update references in parallel with asyncio.gather. - """ - if hasattr(obj, 'use_latest') and obj.use_latest: - obj.ref = await get_latest_ref(obj.ref) - else: - return obj - - -async def update_message(message: ProgramMessage): - # Load amends - await asyncio.gather( - update_with_latest_ref(message.content.runtime), - update_with_latest_ref(message.content.code), - update_with_latest_ref(message.content.data), - *( - update_with_latest_ref(volume) - for volume in (message.content.volumes or []) - ), - ) - - -async def load_updated_message(ref: VmHash) -> Tuple[ProgramMessage, ProgramMessage]: - original_message = await try_get_message(ref) - message = copy.deepcopy(original_message) - await update_message(message) - pool.message_cache[ref] = message - return message, original_message - - -async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: - # ASGI mandates lowercase header names - headers = tuple((name.lower(), value) - for name, value in request.raw_headers) - return { - "type": "http", - "path": path, - "method": request.method, - "query_string": request.query_string, - "headers": headers, - "body": await request.read() - } - - -async def run_code(vm_hash: VmHash, path: str, request: web.Request) -> web.Response: - """ - Execute the code corresponding to the 'code id' in the path. - """ - - execution: VmExecution = await pool.get_running_vm(vm_hash=vm_hash) - - if not execution: - message, original_message = await load_updated_message(vm_hash) - - try: - execution = await pool.create_a_vm(vm_hash=vm_hash, program=message.content, - original=original_message.content) - except ResourceDownloadError as error: - logger.exception(error) - raise HTTPBadRequest(reason="Code, runtime or data not available") - except VmSetupError as error: - logger.exception(error) - raise HTTPInternalServerError(reason="Error during program initialisation") - except MicroVMFailedInit as error: - logger.exception(error) - raise HTTPInternalServerError(reason="Error during runtime initialisation") - - logger.debug(f"Using vm={execution.vm.vm_id}") - - scope: Dict = await build_asgi_scope(path, request) - - try: - await execution.becomes_ready() - result_raw: bytes = await execution.run_code(scope=scope) - except UnpackValueError as error: - logger.exception(error) - return web.Response(status=502, reason="Invalid response from VM") - - try: - result = msgpack.loads(result_raw, raw=False) - - logger.debug(f"Result from VM: <<<\n\n{str(result)[:1000]}\n\n>>>") - - if "traceback" in result: - logger.warning(result["traceback"]) - return web.Response( - status=500, - reason="Error in VM execution", - body=result["traceback"], - content_type="text/plain", - ) - - headers = {key.decode(): value.decode() - for key, value in result['headers']['headers']} - - return web.Response( - status=result['headers']['status'], - body=result["body"]["body"], - headers=headers, - ) - except UnpackValueError as error: - logger.exception(error) - return web.Response(status=502, reason="Invalid response from VM") - finally: - if settings.REUSE_TIMEOUT > 0: - if settings.WATCH_FOR_UPDATES: - execution.start_watching_for_updates(pubsub=request.app['pubsub']) - execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT) - else: - await execution.stop() - - -def run_code_from_path(request: web.Request) -> Awaitable[web.Response]: - """Allow running an Aleph VM function from a URL path - - The path is expected to follow the scheme defined in `app.add_routes` below, - where the identifier of the message is named `ref`. - """ - path = request.match_info["suffix"] - path = path if path.startswith("/") else f"/{path}" - - message_ref: VmHash = request.match_info["ref"] - return run_code(message_ref, path, request) - - -def b32_to_b16(hash: str) -> bytes: - """Convert base32 encoded bytes to base16 encoded bytes.""" - # Add padding - hash_b32: str = hash.upper() + "=" * (56 - len(hash)) - hash_bytes: bytes = b32decode(hash_b32.encode()) - return b16encode(hash_bytes).lower() - - -async def get_ref_from_dns(domain): - resolver = aiodns.DNSResolver() - record = await resolver.query(domain, 'TXT') - return record[0].text - - -async def run_code_from_hostname(request: web.Request) -> web.Response: - """Allow running an Aleph VM function from a hostname - - The first component of the hostname is used as identifier of the message defining the - Aleph VM function. - - Since hostname labels are limited to 63 characters and hex(sha256(...)) has a length of 64, - we expect the hash to be encoded in base32 instead of hexadecimal. Padding is added - automatically. - """ - path = request.match_info["suffix"] - path = path if path.startswith("/") else f"/{path}" - - message_ref_base32 = request.host.split(".")[0] - if settings.FAKE_DATA: - message_ref = "TEST_HASH" - else: - try: - message_ref = b32_to_b16(message_ref_base32).decode() - logger.debug(f"Using base32 message id from hostname to obtain '{message_ref}") - except binascii.Error: - try: - message_ref = await get_ref_from_dns(domain=f"_aleph-id.{request.host}") - logger.debug(f"Using DNS TXT record to obtain '{message_ref}'") - except aiodns.error.DNSError: - raise HTTPNotFound(reason="Invalid message reference") - - return await run_code(message_ref, path, request) - - -async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]: - logger.debug("subscribe_via_ws()") - async with aiohttp.ClientSession() as session: - async with session.ws_connect(url) as ws: - logger.debug(f"Websocket connected on {url}") - async for msg in ws: - logger.debug(f"Websocket received data...") - if msg.type == aiohttp.WSMsgType.TEXT: - data = json.loads(msg.data) - # Patch data format to match HTTP GET format - data["_id"] = {"$oid": data["_id"]} - try: - yield Message(**data) - except pydantic.error_wrappers.ValidationError as error: - print(error.json()) - print(error.raw_errors) - raise - elif msg.type == aiohttp.WSMsgType.ERROR: - break - - -async def watch_for_messages(dispatcher: PubSub): - """Watch for new Aleph messages""" - logger.debug("watch_for_messages()") - url = URL(f"{settings.API_SERVER}/api/ws0/messages" - ).with_query({"startDate": math.floor(time.time())}) - - async for message in subscribe_via_ws(url): - logger.info(f"Websocket received message: {message.item_hash}") - ref = message.content.ref if hasattr(message.content, 'ref') else message.item_hash - await dispatcher.publish(key=ref, value=message) - - -async def start_watch_for_messages_task(app: web.Application): - logger.debug("start_watch_for_messages_task()") - pubsub = PubSub() - app['pubsub'] = pubsub - app['messages_listener'] = asyncio.create_task(watch_for_messages(pubsub)) - - -async def stop_watch_for_messages_task(app: web.Application): - app['messages_listener'].cancel() - await app['messages_listener'] - - -def to_json(o: Any): - if hasattr(o, 'to_dict'): # dataclasses - return o.to_dict() - elif hasattr(o, 'dict'): # Pydantic - return o.dict() - else: - return str(o) - - -def dumper(o: Any): - return json.dumps(o, default=to_json) - - -def authenticate_request(request: web.Request): - """Check that the token in the cookies matches the app's secret token.""" - if request.cookies.get('token') != request.app['secret_token']: - raise web.HTTPUnauthorized(reason="Invalid token") - - -async def about_login(request: web.Request): - token = request.query.get('token') - if token == request.app['secret_token']: - response = web.HTTPFound('/about/config') - response.cookies['token'] = token - return response - else: - return web.json_response({"success": False}, status=401) - - -async def about_executions(request: web.Request): - authenticate_request(request) - return web.json_response( - [ - { - key: value - for key, value in pool.executions.items() - } - ], - dumps=dumper, - ) - - -async def about_config(request: web.Request): - authenticate_request(request) - return web.json_response( - settings, - dumps=dumper, - ) - app = web.Application() diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py new file mode 100644 index 000000000..08b96921c --- /dev/null +++ b/vm_supervisor/tasks.py @@ -0,0 +1,63 @@ +import asyncio +import json +import logging +import math +import time +from typing import AsyncIterable + +import aiohttp +import pydantic +from aiohttp import web +from yarl import URL + +from aleph_message import Message +from aleph_message.models import BaseMessage +from .conf import settings +from .pubsub import PubSub + +logger = logging.getLogger(__name__) + + +async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]: + logger.debug("subscribe_via_ws()") + async with aiohttp.ClientSession() as session: + async with session.ws_connect(url) as ws: + logger.debug(f"Websocket connected on {url}") + async for msg in ws: + logger.debug(f"Websocket received data...") + if msg.type == aiohttp.WSMsgType.TEXT: + data = json.loads(msg.data) + # Patch data format to match HTTP GET format + data["_id"] = {"$oid": data["_id"]} + try: + yield Message(**data) + except pydantic.error_wrappers.ValidationError as error: + print(error.json()) + print(error.raw_errors) + raise + elif msg.type == aiohttp.WSMsgType.ERROR: + break + + +async def watch_for_messages(dispatcher: PubSub): + """Watch for new Aleph messages""" + logger.debug("watch_for_messages()") + url = URL(f"{settings.API_SERVER}/api/ws0/messages" + ).with_query({"startDate": math.floor(time.time())}) + + async for message in subscribe_via_ws(url): + logger.info(f"Websocket received message: {message.item_hash}") + ref = message.content.ref if hasattr(message.content, 'ref') else message.item_hash + await dispatcher.publish(key=ref, value=message) + + +async def start_watch_for_messages_task(app: web.Application): + logger.debug("start_watch_for_messages_task()") + pubsub = PubSub() + app['pubsub'] = pubsub + app['messages_listener'] = asyncio.create_task(watch_for_messages(pubsub)) + + +async def stop_watch_for_messages_task(app: web.Application): + app['messages_listener'].cancel() + await app['messages_listener'] diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py new file mode 100644 index 000000000..8b751b494 --- /dev/null +++ b/vm_supervisor/utils.py @@ -0,0 +1,32 @@ +import json +from base64 import b32decode, b16encode +from typing import Any + +import aiodns + + +def b32_to_b16(hash: str) -> bytes: + """Convert base32 encoded bytes to base16 encoded bytes.""" + # Add padding + hash_b32: str = hash.upper() + "=" * (56 - len(hash)) + hash_bytes: bytes = b32decode(hash_b32.encode()) + return b16encode(hash_bytes).lower() + + +async def get_ref_from_dns(domain): + resolver = aiodns.DNSResolver() + record = await resolver.query(domain, 'TXT') + return record[0].text + + +def to_json(o: Any): + if hasattr(o, 'to_dict'): # dataclasses + return o.to_dict() + elif hasattr(o, 'dict'): # Pydantic + return o.dict() + else: + return str(o) + + +def dumps_for_json(o: Any): + return json.dumps(o, default=to_json) diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py new file mode 100644 index 000000000..14aa4aed7 --- /dev/null +++ b/vm_supervisor/views.py @@ -0,0 +1,99 @@ +import binascii +import logging +from typing import Awaitable + +import aiodns +from aiohttp import web +from aiohttp.web_exceptions import HTTPNotFound + +from .conf import settings +from .models import VmHash +from .run import run_code, pool +from .utils import b32_to_b16, get_ref_from_dns, dumps_for_json + +logger = logging.getLogger(__name__) + + +def run_code_from_path(request: web.Request) -> Awaitable[web.Response]: + """Allow running an Aleph VM function from a URL path + + The path is expected to follow the scheme defined in `app.add_routes` below, + where the identifier of the message is named `ref`. + """ + path = request.match_info["suffix"] + path = path if path.startswith("/") else f"/{path}" + + message_ref: VmHash = request.match_info["ref"] + return run_code(message_ref, path, request) + + +async def run_code_from_hostname(request: web.Request) -> web.Response: + """Allow running an Aleph VM function from a hostname + + The first component of the hostname is used as identifier of the message defining the + Aleph VM function. + + Since hostname labels are limited to 63 characters and hex(sha256(...)) has a length of 64, + we expect the hash to be encoded in base32 instead of hexadecimal. Padding is added + automatically. + """ + path = request.match_info["suffix"] + path = path if path.startswith("/") else f"/{path}" + + message_ref_base32 = request.host.split(".")[0] + if settings.FAKE_DATA: + message_ref = "TEST_HASH" + else: + try: + message_ref = b32_to_b16(message_ref_base32).decode() + logger.debug(f"Using base32 message id from hostname to obtain '{message_ref}") + except binascii.Error: + try: + message_ref = await get_ref_from_dns(domain=f"_aleph-id.{request.host}") + logger.debug(f"Using DNS TXT record to obtain '{message_ref}'") + except aiodns.error.DNSError: + raise HTTPNotFound(reason="Invalid message reference") + + return await run_code(message_ref, path, request) + + +def authenticate_request(request: web.Request): + """Check that the token in the cookies matches the app's secret token.""" + if request.cookies.get('token') != request.app['secret_token']: + raise web.HTTPUnauthorized(reason="Invalid token") + + +async def about_login(request: web.Request): + token = request.query.get('token') + if token == request.app['secret_token']: + response = web.HTTPFound('/about/config') + response.cookies['token'] = token + return response + else: + return web.json_response({"success": False}, status=401) + + +async def about_executions(request: web.Request): + authenticate_request(request) + return web.json_response( + [ + { + key: value + for key, value in pool.executions.items() + } + ], + dumps=dumps_for_json, + ) + + +async def about_config(request: web.Request): + authenticate_request(request) + return web.json_response( + settings, + dumps=dumps_for_json, + ) + + +async def index(request: web.Request): + assert request.method == "GET" + return web.Response(text="Server: Aleph VM Supervisor") From cf8485d8b4bf76bb102ee6c5b397534b9ed1823a Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 15 Jul 2021 12:32:34 +0200 Subject: [PATCH 136/990] Reformat: Use `black` on Python code --- firecracker/config.py | 8 +- firecracker/microvm.py | 56 ++++++++----- vm_connector/main.py | 27 ++++--- vm_supervisor/__main__.py | 44 +++++----- vm_supervisor/conf.py | 13 +-- vm_supervisor/messages.py | 7 +- vm_supervisor/models.py | 13 +-- vm_supervisor/pool.py | 4 +- vm_supervisor/run.py | 21 ++--- vm_supervisor/storage.py | 36 ++++++--- vm_supervisor/supervisor.py | 27 ++++--- vm_supervisor/tasks.py | 19 +++-- vm_supervisor/utils.py | 6 +- vm_supervisor/views.py | 21 +++-- vm_supervisor/vm/firecracker_microvm.py | 103 +++++++++++++++--------- 15 files changed, 243 insertions(+), 162 deletions(-) diff --git a/firecracker/config.py b/firecracker/config.py index 5f9d01117..439f93c7a 100644 --- a/firecracker/config.py +++ b/firecracker/config.py @@ -8,8 +8,10 @@ class BootSource(BaseModel): kernel_image_path: FilePath = "vmlinux.bin" - boot_args: str = "console=ttyS0 reboot=k panic=1 pci=off " \ - "ro noapic nomodules random.trust_cpu=on" + boot_args: str = ( + "console=ttyS0 reboot=k panic=1 pci=off " + "ro noapic nomodules random.trust_cpu=on" + ) @staticmethod def args(enable_console: bool = True): @@ -54,4 +56,4 @@ class FirecrackerConfig(BaseModel): class Config: allow_population_by_field_name = True - alias_generator = lambda x: x.replace('_', '-') + alias_generator = lambda x: x.replace("_", "-") diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 6b25fbc5f..4cbfef826 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -104,7 +104,7 @@ def __init__( firecracker_bin_path: str, use_jailer: bool = True, jailer_bin_path: Optional[str] = None, - init_timeout: float = 5., + init_timeout: float = 5.0, ): self.vm_id = vm_id self.use_jailer = use_jailer @@ -115,11 +115,11 @@ def __init__( def to_dict(self): return { - 'jailer_path': self.jailer_path, - 'socket_path': self.socket_path, - 'vsock_path': self.vsock_path, - 'guest_ip': self.guest_ip, - 'host_ip': self.host_ip, + "jailer_path": self.jailer_path, + "socket_path": self.socket_path, + "vsock_path": self.vsock_path, + "guest_ip": self.guest_ip, + "host_ip": self.host_ip, **self.__dict__, } @@ -147,7 +147,9 @@ async def start(self, config: FirecrackerConfig) -> asyncio.subprocess.Process: else: return await self.start_firecracker(config) - async def start_firecracker(self, config: FirecrackerConfig) -> asyncio.subprocess.Process: + async def start_firecracker( + self, config: FirecrackerConfig + ) -> asyncio.subprocess.Process: if os.path.exists(VSOCK_PATH): os.remove(VSOCK_PATH) @@ -155,14 +157,23 @@ async def start_firecracker(self, config: FirecrackerConfig) -> asyncio.subproce os.remove(self.socket_path) config_file = NamedTemporaryFile() - config_file.write(config.json(by_alias=True, exclude_none=True, indent=4).encode()) + config_file.write( + config.json(by_alias=True, exclude_none=True, indent=4).encode() + ) config_file.flush() self.config_file = config_file print(self.config_file) logger.debug( - " ".join((self.firecracker_bin_path, "--api-sock", self.socket_path, - "--config-file", config_file.name)) + " ".join( + ( + self.firecracker_bin_path, + "--api-sock", + self.socket_path, + "--config-file", + config_file.name, + ) + ) ) self.proc = await asyncio.create_subprocess_exec( @@ -177,15 +188,19 @@ async def start_firecracker(self, config: FirecrackerConfig) -> asyncio.subproce ) return self.proc - async def start_jailed_firecracker(self, config: FirecrackerConfig) -> asyncio.subprocess.Process: + async def start_jailed_firecracker( + self, config: FirecrackerConfig + ) -> asyncio.subprocess.Process: if not self.jailer_bin_path: raise ValueError("Jailer binary path is missing") uid = str(getpwnam("jailman").pw_uid) gid = str(getpwnam("jailman").pw_gid) # config_file = NamedTemporaryFile(dir=f"{self.jailer_path}/tmp/", suffix='.json') - config_file = open(f"{self.jailer_path}/tmp/config.json", 'wb') - config_file.write(config.json(by_alias=True, exclude_none=True, indent=4).encode()) + config_file = open(f"{self.jailer_path}/tmp/config.json", "wb") + config_file.write( + config.json(by_alias=True, exclude_none=True, indent=4).encode() + ) config_file.flush() os.chmod(config_file.name, 0o644) self.config_file = config_file @@ -291,9 +306,7 @@ async def create_network_interface(self, interface: str = "eth0") -> str: self.network_tap = host_dev_name system(f"ip tuntap add {host_dev_name} mode tap") - system( - f"ip addr add {self.host_ip}/24 dev {host_dev_name}" - ) + system(f"ip addr add {self.host_ip}/24 dev {host_dev_name}") system(f"ip link set {host_dev_name} up") system('sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"') # TODO: Don't fill iptables with duplicate rules; purge rules on delete @@ -369,11 +382,15 @@ async def teardown(self): self.stderr_task.cancel() if self.network_tap: - await asyncio.sleep(0.01) # Used to prevent `ioctl(TUNSETIFF): Device or resource busy` + await asyncio.sleep( + 0.01 + ) # Used to prevent `ioctl(TUNSETIFF): Device or resource busy` logger.debug(f"Removing interface {self.network_tap}") system(f"ip tuntap del {self.network_tap} mode tap") logger.debug("Removing iptables rules") - system(f"iptables -t nat -D POSTROUTING -o {self.network_interface} -j MASQUERADE") + system( + f"iptables -t nat -D POSTROUTING -o {self.network_interface} -j MASQUERADE" + ) system( "iptables -D FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT" ) @@ -384,13 +401,12 @@ async def teardown(self): logger.debug("Removing files") system(f"rm -fr {self.jailer_path}") - def __del__(self): try: loop = asyncio.get_running_loop() loop.create_task(self.teardown()) except RuntimeError as error: - if error.args == ('no running event loop',): + if error.args == ("no running event loop",): return else: raise diff --git a/vm_connector/main.py b/vm_connector/main.py index e7bb468e0..d919c6190 100644 --- a/vm_connector/main.py +++ b/vm_connector/main.py @@ -27,11 +27,12 @@ def read_root(): return {"Server": "Aleph.im VM Connector"} - async def get_latest_message_amend(ref: str, sender: str) -> Optional[Dict]: async with aiohttp.ClientSession() as session: - url = f"{settings.ALEPH_SERVER}/api/v0/messages.json?msgType=STORE&sort_order=-1" \ - f"&refs={ref}&addresses={sender}" + url = ( + f"{settings.ALEPH_SERVER}/api/v0/messages.json?msgType=STORE&sort_order=-1" + f"&refs={ref}&addresses={sender}" + ) resp = await session.get(url) resp.raise_for_status() resp_data = await resp.json() @@ -70,12 +71,12 @@ async def download_message( ref: str, use_latest: Optional[bool] = True ) -> Union[Dict, Response]: """ - Fetch on Aleph and return a VM function message, after checking its validity. - Used by the VM Supervisor run the code. -K - :param ref: item_hash of the code file - :param use_latest: should the last amend to the code be used - :return: a file containing the code file + Fetch on Aleph and return a VM function message, after checking its validity. + Used by the VM Supervisor run the code. + + :param ref: item_hash of the code file + :param use_latest: should the last amend to the code be used + :return: a file containing the code file """ if settings.OFFLINE_TEST_MODE: @@ -176,14 +177,14 @@ async def download_runtime( @app.get("/compute/latest_amend/{item_hash}") async def compute_latest_amend(item_hash: str) -> str: msg = await get_message(hash_=item_hash) - sender = msg['sender'] + sender = msg["sender"] latest_amend = await get_latest_message_amend(ref=item_hash, sender=sender) if latest_amend: # Validation - assert latest_amend['sender'] == sender - assert latest_amend['content']['ref'] == item_hash + assert latest_amend["sender"] == sender + assert latest_amend["content"]["ref"] == item_hash - return latest_amend['item_hash'] + return latest_amend["item_hash"] else: # Original message is the latest return item_hash diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index b926a7fe6..1e293da6a 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -112,17 +112,13 @@ class FakeRequest: fake_request.method = "GET" fake_request.query_string = "" - fake_request.headers = { - 'host': '127.0.0.1', - 'content-type': 'application/json' - } + fake_request.headers = {"host": "127.0.0.1", "content-type": "application/json"} fake_request.raw_headers = [ - (name.encode(), value.encode()) - for name, value in fake_request.headers.items() + (name.encode(), value.encode()) for name, value in fake_request.headers.items() ] # noinspection PyDeprecation - fake_request.read = coroutine(lambda: b'') + fake_request.read = coroutine(lambda: b"") logger.info("--- Start benchmark ---") @@ -133,12 +129,19 @@ class FakeRequest: # First test all methods settings.REUSE_TIMEOUT = 0.1 - for path in ("/", "/messages", "/internet", "/post_a_message", - "/cache/set/foo/bar", "/cache/get/foo", "/cache/keys"): + for path in ( + "/", + "/messages", + "/internet", + "/post_a_message", + "/cache/set/foo/bar", + "/cache/get/foo", + "/cache/keys", + ): fake_request.match_info["suffix"] = path - response: Response = await run_code(vm_hash=ref, - path=path, - request=fake_request) + response: Response = await run_code( + vm_hash=ref, path=path, request=fake_request + ) assert response.status == 200 # Disable VM timeout to exit benchmark properly @@ -147,22 +150,27 @@ class FakeRequest: for run in range(runs): t0 = time.time() fake_request.match_info["suffix"] = path - response: Response = await run_code(vm_hash=ref, - path=path, - request=fake_request) + response: Response = await run_code( + vm_hash=ref, path=path, request=fake_request + ) assert response.status == 200 bench.append(time.time() - t0) - logger.info(f"BENCHMARK: n={len(bench)} avg={mean(bench):03f} " - f"min={min(bench):03f} max={max(bench):03f}") + logger.info( + f"BENCHMARK: n={len(bench)} avg={mean(bench):03f} " + f"min={min(bench):03f} max={max(bench):03f}" + ) logger.info(bench) def main(): args = parse_args(sys.argv[1:]) - log_format = "%(relativeCreated)4f | %(levelname)s | %(message)s" if args.profile \ + log_format = ( + "%(relativeCreated)4f | %(levelname)s | %(message)s" + if args.profile else "%(asctime)s | %(levelname)s | %(message)s" + ) logging.basicConfig( level=args.loglevel, format=log_format, diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 0c4afcb46..30026230b 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -87,7 +87,9 @@ class Settings(BaseSettings): RUNTIME_CACHE: FilePath = FilePath(join(CACHE_ROOT, "runtime")) DATA_CACHE: FilePath = FilePath(join(CACHE_ROOT, "data")) - PERSISTENT_VOLUMES_DIR: FilePath = FilePath(join("/var/tmp/aleph", "volumes", "persistent")) + PERSISTENT_VOLUMES_DIR: FilePath = FilePath( + join("/var/tmp/aleph", "volumes", "persistent") + ) FAKE_DATA: bool = False FAKE_DATA_EXAMPLE: str = "example_fastapi_2" @@ -122,14 +124,13 @@ def setup(self): if self.DNS_RESOLUTION == DnsResolver.resolv_conf: self.DNS_NAMESERVERS = list(etc_resolv_conf_dns_servers()) - elif self.DNS_RESOLUTION == DnsResolver.resolvectl: - self.DNS_NAMESERVERS = list(systemd_resolved_dns_servers( - interface=self.NETWORK_INTERFACE)) + self.DNS_NAMESERVERS = list( + systemd_resolved_dns_servers(interface=self.NETWORK_INTERFACE) + ) else: assert "This should never happen" - def display(self) -> str: return "\n".join( f"{annotation:<17} = {getattr(self, annotation)}" @@ -139,7 +140,7 @@ def display(self) -> str: class Config: env_prefix = "ALEPH_VM_" case_sensitive = False - env_file = '.env' + env_file = ".env" # Settings singleton diff --git a/vm_supervisor/messages.py b/vm_supervisor/messages.py index 7020d14c7..e0610de65 100644 --- a/vm_supervisor/messages.py +++ b/vm_supervisor/messages.py @@ -41,7 +41,7 @@ async def update_with_latest_ref(obj): Useful to update references in parallel with asyncio.gather. """ - if hasattr(obj, 'use_latest') and obj.use_latest: + if hasattr(obj, "use_latest") and obj.use_latest: obj.ref = await get_latest_ref(obj.ref) else: return obj @@ -53,10 +53,7 @@ async def update_message(message: ProgramMessage): update_with_latest_ref(message.content.runtime), update_with_latest_ref(message.content.code), update_with_latest_ref(message.content.data), - *( - update_with_latest_ref(volume) - for volume in (message.content.volumes or []) - ), + *(update_with_latest_ref(volume) for volume in (message.content.volumes or [])), ) diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index 8b6c63a29..40222b89f 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -58,7 +58,9 @@ def is_running(self): def becomes_ready(self): return self.ready_event.wait - def __init__(self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent): + def __init__( + self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent + ): self.vm_hash = vm_hash self.program = program self.original = original @@ -69,7 +71,7 @@ def __init__(self, vm_hash: VmHash, program: ProgramContent, original: ProgramCo def to_dict(self) -> Dict: return { - 'is_running': self.is_running, + "is_running": self.is_running, **self.__dict__, } @@ -110,8 +112,9 @@ def stop_after_timeout(self, timeout: float = 5.0) -> Task: loop = asyncio.get_event_loop() if sys.version_info.major >= 3 and sys.version_info.minor >= 8: # Task can be named - self.expire_task = loop.create_task(self.expire(timeout), - name=f"expire {self.vm.vm_id}") + self.expire_task = loop.create_task( + self.expire(timeout), name=f"expire {self.vm.vm_id}" + ) else: self.expire_task = loop.create_task(self.expire(timeout)) return self.expire_task @@ -149,7 +152,7 @@ async def watch_for_updates(self, pubsub: PubSub): *( volume.ref for volume in (self.original.volumes or []) - if hasattr(volume, 'ref') + if hasattr(volume, "ref") ), ) logger.debug("Update received, stopping VM...") diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index 5590b480f..31f87ed45 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -25,7 +25,9 @@ def __init__(self): self.counter = settings.START_ID_INDEX self.executions = {} - async def create_a_vm(self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent) -> VmExecution: + async def create_a_vm( + self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent + ) -> VmExecution: """Create a new Aleph Firecracker VM from an Aleph function message.""" execution = VmExecution(vm_hash=vm_hash, program=program, original=original) self.executions[vm_hash] = execution diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py index 191d1b1dd..b1ffa4ace 100644 --- a/vm_supervisor/run.py +++ b/vm_supervisor/run.py @@ -20,15 +20,14 @@ async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: # ASGI mandates lowercase header names - headers = tuple((name.lower(), value) - for name, value in request.raw_headers) + headers = tuple((name.lower(), value) for name, value in request.raw_headers) return { "type": "http", "path": path, "method": request.method, "query_string": request.query_string, "headers": headers, - "body": await request.read() + "body": await request.read(), } @@ -44,8 +43,11 @@ async def run_code(vm_hash: VmHash, path: str, request: web.Request) -> web.Resp pool.message_cache[vm_hash] = message try: - execution = await pool.create_a_vm(vm_hash=vm_hash, program=message.content, - original=original_message.content) + execution = await pool.create_a_vm( + vm_hash=vm_hash, + program=message.content, + original=original_message.content, + ) except ResourceDownloadError as error: logger.exception(error) raise HTTPBadRequest(reason="Code, runtime or data not available") @@ -81,11 +83,12 @@ async def run_code(vm_hash: VmHash, path: str, request: web.Request) -> web.Resp content_type="text/plain", ) - headers = {key.decode(): value.decode() - for key, value in result['headers']['headers']} + headers = { + key.decode(): value.decode() for key, value in result["headers"]["headers"] + } return web.Response( - status=result['headers']['status'], + status=result["headers"]["status"], body=result["body"]["body"], headers=headers, ) @@ -95,7 +98,7 @@ async def run_code(vm_hash: VmHash, path: str, request: web.Request) -> web.Resp finally: if settings.REUSE_TIMEOUT > 0: if settings.WATCH_FOR_UPDATES: - execution.start_watching_for_updates(pubsub=request.app['pubsub']) + execution.start_watching_for_updates(pubsub=request.app["pubsub"]) execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT) else: await execution.stop() diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index a7c2f86ee..984689ffb 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -16,8 +16,13 @@ import aiohttp from aleph_message.models import ProgramMessage -from aleph_message.models.program import Encoding, MachineVolume, ImmutableVolume, PersistentVolume, \ - VolumePersistence +from aleph_message.models.program import ( + Encoding, + MachineVolume, + ImmutableVolume, + PersistentVolume, + VolumePersistence, +) from .conf import settings from firecracker.models import FilePath @@ -73,8 +78,10 @@ async def get_message(ref: str) -> ProgramMessage: with open(cache_path, "r") as cache_file: msg = json.load(cache_file) if settings.FAKE_DATA: - msg['item_content'] = json.dumps(msg['content']) - msg['item_hash'] = hashlib.sha256(msg['item_content'].encode('utf-8')).hexdigest() + msg["item_content"] = json.dumps(msg["content"]) + msg["item_hash"] = hashlib.sha256( + msg["item_content"].encode("utf-8") + ).hexdigest() return ProgramMessage(**msg) @@ -83,20 +90,20 @@ async def get_code_path(ref: str) -> FilePath: root_dir = abspath(join(__file__, "../../examples/")) archive_path = join(root_dir, settings.FAKE_DATA_EXAMPLE) - encoding: Encoding = (await get_message(ref="fake-message")).content.code.encoding + encoding: Encoding = ( + await get_message(ref="fake-message") + ).content.code.encoding if encoding == Encoding.squashfs: if os.path.exists(f"{archive_path}.squashfs"): os.remove(f"{archive_path}.squashfs") os.system(f"mksquashfs {archive_path} {archive_path}.squashfs") return FilePath(f"{archive_path}.squashfs") elif encoding == Encoding.zip: - make_archive( - archive_path, "zip", root_dir=archive_path) + make_archive(archive_path, "zip", root_dir=archive_path) return FilePath(f"{archive_path}.zip") else: raise ValueError(f"Unsupported encoding: {encoding}") - cache_path = FilePath(join(settings.CODE_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/code/{ref}" await download_file(url, cache_path) @@ -145,7 +152,9 @@ async def get_volume_path(volume: MachineVolume, namespace: str) -> FilePath: if isinstance(volume, ImmutableVolume): ref = volume.ref if settings.FAKE_DATA: - data_dir = abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs")) + data_dir = abspath( + join(__file__, "../../examples/volumes/volume-venv.squashfs") + ) return FilePath(data_dir) cache_path = FilePath(join(settings.DATA_CACHE, ref)) @@ -155,12 +164,15 @@ async def get_volume_path(volume: MachineVolume, namespace: str) -> FilePath: elif isinstance(volume, PersistentVolume): if volume.persistence != VolumePersistence.host: raise NotImplementedError("Only 'host' persistence is supported") - if not re.match(r'^[\w\-_/]+$', volume.name): + if not re.match(r"^[\w\-_/]+$", volume.name): raise ValueError(f"Invalid value for volume name: {volume.name}") os.makedirs(join(settings.PERSISTENT_VOLUMES_DIR, namespace), exist_ok=True) - volume_path = FilePath(join(settings.PERSISTENT_VOLUMES_DIR, namespace, f"{volume.name}.ext4")) + volume_path = FilePath( + join(settings.PERSISTENT_VOLUMES_DIR, namespace, f"{volume.name}.ext4") + ) await asyncio.get_event_loop().run_in_executor( - None, create_ext4, volume_path, volume.size_mib) + None, create_ext4, volume_path, volume.size_mib + ) return volume_path else: raise NotImplementedError("Only immutable volumes are supported") diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 9a29eaf34..8ea569570 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -12,20 +12,27 @@ from .conf import settings from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task -from .views import (run_code_from_path, run_code_from_hostname, about_login, about_executions, - about_config) +from .views import ( + run_code_from_path, + run_code_from_hostname, + about_login, + about_executions, + about_config, +) logger = logging.getLogger(__name__) app = web.Application() -app.add_routes([ - web.get("/about/login", about_login), - web.get("/about/executions", about_executions), - web.get("/about/config", about_config), - web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path), - web.route("*", "/{suffix:.*}", run_code_from_hostname), -]) +app.add_routes( + [ + web.get("/about/login", about_login), + web.get("/about/executions", about_executions), + web.get("/about/config", about_config), + web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path), + web.route("*", "/{suffix:.*}", run_code_from_hostname), + ] +) def run(): @@ -34,7 +41,7 @@ def run(): # Require a random token to access /about APIs secret_token = token_urlsafe(nbytes=32) - app['secret_token'] = secret_token + app["secret_token"] = secret_token print(f"Login to /about pages /about/login?token={secret_token}") app.on_startup.append(start_watch_for_messages_task) diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py index 08b96921c..aeff9c85d 100644 --- a/vm_supervisor/tasks.py +++ b/vm_supervisor/tasks.py @@ -42,22 +42,27 @@ async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]: async def watch_for_messages(dispatcher: PubSub): """Watch for new Aleph messages""" logger.debug("watch_for_messages()") - url = URL(f"{settings.API_SERVER}/api/ws0/messages" - ).with_query({"startDate": math.floor(time.time())}) + url = URL(f"{settings.API_SERVER}/api/ws0/messages").with_query( + {"startDate": math.floor(time.time())} + ) async for message in subscribe_via_ws(url): logger.info(f"Websocket received message: {message.item_hash}") - ref = message.content.ref if hasattr(message.content, 'ref') else message.item_hash + ref = ( + message.content.ref + if hasattr(message.content, "ref") + else message.item_hash + ) await dispatcher.publish(key=ref, value=message) async def start_watch_for_messages_task(app: web.Application): logger.debug("start_watch_for_messages_task()") pubsub = PubSub() - app['pubsub'] = pubsub - app['messages_listener'] = asyncio.create_task(watch_for_messages(pubsub)) + app["pubsub"] = pubsub + app["messages_listener"] = asyncio.create_task(watch_for_messages(pubsub)) async def stop_watch_for_messages_task(app: web.Application): - app['messages_listener'].cancel() - await app['messages_listener'] + app["messages_listener"].cancel() + await app["messages_listener"] diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py index 8b751b494..e925dbeb8 100644 --- a/vm_supervisor/utils.py +++ b/vm_supervisor/utils.py @@ -15,14 +15,14 @@ def b32_to_b16(hash: str) -> bytes: async def get_ref_from_dns(domain): resolver = aiodns.DNSResolver() - record = await resolver.query(domain, 'TXT') + record = await resolver.query(domain, "TXT") return record[0].text def to_json(o: Any): - if hasattr(o, 'to_dict'): # dataclasses + if hasattr(o, "to_dict"): # dataclasses return o.to_dict() - elif hasattr(o, 'dict'): # Pydantic + elif hasattr(o, "dict"): # Pydantic return o.dict() else: return str(o) diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py index 14aa4aed7..ef19bc501 100644 --- a/vm_supervisor/views.py +++ b/vm_supervisor/views.py @@ -46,7 +46,9 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: else: try: message_ref = b32_to_b16(message_ref_base32).decode() - logger.debug(f"Using base32 message id from hostname to obtain '{message_ref}") + logger.debug( + f"Using base32 message id from hostname to obtain '{message_ref}" + ) except binascii.Error: try: message_ref = await get_ref_from_dns(domain=f"_aleph-id.{request.host}") @@ -59,15 +61,15 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: def authenticate_request(request: web.Request): """Check that the token in the cookies matches the app's secret token.""" - if request.cookies.get('token') != request.app['secret_token']: + if request.cookies.get("token") != request.app["secret_token"]: raise web.HTTPUnauthorized(reason="Invalid token") async def about_login(request: web.Request): - token = request.query.get('token') - if token == request.app['secret_token']: - response = web.HTTPFound('/about/config') - response.cookies['token'] = token + token = request.query.get("token") + if token == request.app["secret_token"]: + response = web.HTTPFound("/about/config") + response.cookies["token"] = token return response else: return web.json_response({"success": False}, status=401) @@ -76,12 +78,7 @@ async def about_login(request: web.Request): async def about_executions(request: web.Request): authenticate_request(request) return web.json_response( - [ - { - key: value - for key, value in pool.executions.items() - } - ], + [{key: value for key, value in pool.executions.items()}], dumps=dumps_for_json, ) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 32c647c5f..7a976a96c 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -9,6 +9,7 @@ from typing import Optional, Dict, List import msgpack + try: import psutil as psutil except ImportError: @@ -17,8 +18,14 @@ from aleph_message.models import ProgramContent from aleph_message.models.program import MachineResources, Encoding -from firecracker.config import BootSource, Drive, MachineConfig, FirecrackerConfig, Vsock, \ - NetworkInterface +from firecracker.config import ( + BootSource, + Drive, + MachineConfig, + FirecrackerConfig, + Vsock, + NetworkInterface, +) from firecracker.microvm import MicroVM, setfacl from firecracker.models import FilePath from guest_api.__main__ import run_guest_api @@ -160,16 +167,17 @@ async def download_volumes(self): volumes = [] # TODO: Download in parallel for volume in self.message_content.volumes: - volumes.append(HostVolume( - mount=volume.mount, - path_on_host=(await get_volume_path( - volume=volume, namespace=self.namespace)), - - read_only=volume.is_read_only(), - )) + volumes.append( + HostVolume( + mount=volume.mount, + path_on_host=( + await get_volume_path(volume=volume, namespace=self.namespace) + ), + read_only=volume.is_read_only(), + ) + ) self.volumes = volumes - async def download_all(self): await asyncio.gather( self.download_kernel(), @@ -201,7 +209,7 @@ def __init__( resources: AlephFirecrackerResources, enable_networking: bool = False, enable_console: Optional[bool] = None, - hardware_resources: MachineResources = MachineResources() + hardware_resources: MachineResources = MachineResources(), ): self.vm_id = vm_id self.vm_hash = vm_hash @@ -216,22 +224,22 @@ def to_dict(self): if self.fvm.proc and psutil: p = psutil.Process(self.fvm.proc.pid) pid_info = { - 'status': p.status(), - 'create_time': p.create_time(), - 'cpu_times': p.cpu_times(), - 'cpu_percent': p.cpu_percent(), - 'memory_info': p.memory_info(), - 'io_counters': p.io_counters(), - 'open_files': p.open_files(), - 'connections': p.connections(), - 'num_threads': p.num_threads(), - 'num_ctx_switches': p.num_ctx_switches(), + "status": p.status(), + "create_time": p.create_time(), + "cpu_times": p.cpu_times(), + "cpu_percent": p.cpu_percent(), + "memory_info": p.memory_info(), + "io_counters": p.io_counters(), + "open_files": p.open_files(), + "connections": p.connections(), + "num_threads": p.num_threads(), + "num_ctx_switches": p.num_ctx_switches(), } else: pid_info = None return { - 'process': pid_info, + "process": pid_info, **self.__dict__, } @@ -249,20 +257,27 @@ async def setup(self): config = FirecrackerConfig( boot_source=BootSource( - kernel_image_path=FilePath(fvm.enable_kernel(self.resources.kernel_image_path)), + kernel_image_path=FilePath( + fvm.enable_kernel(self.resources.kernel_image_path) + ), boot_args=BootSource.args(enable_console=self.enable_console), ), drives=[ Drive( drive_id="rootfs", - path_on_host=FilePath(fvm.enable_rootfs(self.resources.rootfs_path)), + path_on_host=FilePath( + fvm.enable_rootfs(self.resources.rootfs_path) + ), is_root_device=True, is_read_only=True, ), - ] + ( + ] + + ( [fvm.enable_drive(self.resources.code_path)] - if self.resources.code_encoding == Encoding.squashfs else [] - ) + [ + if self.resources.code_encoding == Encoding.squashfs + else [] + ) + + [ fvm.enable_drive(volume.path_on_host, read_only=volume.read_only) for volume in self.resources.volumes ], @@ -271,12 +286,14 @@ async def setup(self): mem_size_mib=self.hardware_resources.memory, ), vsock=Vsock(), - network_interfaces = [ + network_interfaces=[ NetworkInterface( iface_id="eth0", host_dev_name=await fvm.create_network_interface(interface="eth0"), ) - ] if self.enable_networking else [], + ] + if self.enable_networking + else [], ) logger.debug(config.json(by_alias=True, exclude_none=True, indent=4)) @@ -307,22 +324,31 @@ async def configure(self): input_data: bytes = load_file_content(self.resources.data_path) - interface = Interface.asgi if ":" in self.resources.code_entrypoint \ + interface = ( + Interface.asgi + if ":" in self.resources.code_entrypoint else Interface.executable + ) volumes: List[Volume] if self.resources.code_encoding == Encoding.squashfs: - code = b'' + code = b"" volumes = [Volume(mount="/opt/code", device="vdb", read_only=True)] + [ - Volume(mount=volume.mount, device=self.fvm.drives[index+1].drive_id, - read_only=volume.read_only) + Volume( + mount=volume.mount, + device=self.fvm.drives[index + 1].drive_id, + read_only=volume.read_only, + ) for index, volume in enumerate(self.resources.volumes) ] else: code: bytes = load_file_content(self.resources.code_path) volumes = [ - Volume(mount=volume.mount, device=self.fvm.drives[index].drive_id, - read_only=volume.read_only) + Volume( + mount=volume.mount, + device=self.fvm.drives[index].drive_id, + read_only=volume.read_only, + ) for index, volume in enumerate(self.resources.volumes) ] @@ -346,8 +372,7 @@ async def configure(self): await reader.readline() # Ignore the acknowledgement from the socket response_raw = await reader.read(1000_000) - response = ConfigurationResponse( - **msgpack.loads(response_raw, raw=False)) + response = ConfigurationResponse(**msgpack.loads(response_raw, raw=False)) if response.success is False: logger.exception(response.traceback) raise VmSetupError(response.error) @@ -356,7 +381,9 @@ async def start_guest_api(self): logger.debug(f"starting guest API for {self.vm_id}") vsock_path = f"{self.fvm.vsock_path}_53" vm_hash = self.vm_hash - self.guest_api_process = Process(target=run_guest_api, args=(vsock_path, vm_hash)) + self.guest_api_process = Process( + target=run_guest_api, args=(vsock_path, vm_hash) + ) self.guest_api_process.start() while not exists(vsock_path): await asyncio.sleep(0.01) From 83130674aa7eefad9ecb4182b07c32db3d0a2bf3 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 15 Jul 2021 16:03:15 +0200 Subject: [PATCH 137/990] Feature: Install the supervisor from a Debian package --- packaging/Makefile | 42 +++++++++++++++++++ packaging/aleph-vm/DEBIAN/control | 6 +++ packaging/aleph-vm/DEBIAN/postinst | 15 +++++++ .../aleph-vm/etc/aleph-vm/supervisor.env | 2 + .../system/aleph-vm-supervisor.service | 16 +++++++ vm_supervisor/conf.py | 8 ++-- 6 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 packaging/Makefile create mode 100644 packaging/aleph-vm/DEBIAN/control create mode 100755 packaging/aleph-vm/DEBIAN/postinst create mode 100644 packaging/aleph-vm/etc/aleph-vm/supervisor.env create mode 100644 packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service diff --git a/packaging/Makefile b/packaging/Makefile new file mode 100644 index 000000000..9247d0951 --- /dev/null +++ b/packaging/Makefile @@ -0,0 +1,42 @@ +all: debian-package + +debian-package: debian-package-resources debian-package-code + sudo dpkg-deb --build aleph-vm target/aleph-vm.deb + +debian-package-code: + rm -fr ./aleph-vm/opt/aleph-vm + mkdir -p ./aleph-vm/opt/aleph-vm + cp -r ../vm_supervisor ./aleph-vm/opt/aleph-vm/ + cp -r ../guest_api ./aleph-vm/opt/aleph-vm/ + cp -r ../firecracker ./aleph-vm/opt/aleph-vm/ + +debian-package-resources: firecracker-bins vmlinux + rm -fr ./aleph-vm/opt/firecracker + mkdir -p ./aleph-vm/opt/firecracker + cp -pr ./target/vmlinux.bin ./aleph-vm/opt/firecracker/ + cp -pr ./target/firecracker ./aleph-vm/opt/firecracker/ + cp -pr ./target/jailer ./aleph-vm/opt/firecracker/ + +firecracker-bins: target-dir build-dir + mkdir -p ./build/firecracker-release + # Download latest release + curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v0.24.2/firecracker-v0.24.2-x86_64.tgz | tar -xz --directory ./build/firecracker-release + # Copy binaries: + cp ./build/firecracker-release/firecracker-v* ./target/firecracker + cp ./build/firecracker-release/jailer-v* ./target/jailer + chmod +x ./target/firecracker + chmod +x ./target/jailer + +vmlinux: + #curl -fsSL -o ./target/vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin + cp ../kernels/vmlinux.bin ./target/vmlinux.bin + +build-dir: + mkdir -p target + +target-dir: + mkdir -p target + +clean: + rm -fr ./target/* + rm -fr ./build/* diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control new file mode 100644 index 000000000..ceb118ee3 --- /dev/null +++ b/packaging/aleph-vm/DEBIAN/control @@ -0,0 +1,6 @@ +Package: aleph-vm +Version: 0.1.0-8 +Architecture: all +Maintainer: Aleph.im +Description: Aleph.im VM execution engine +Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap diff --git a/packaging/aleph-vm/DEBIAN/postinst b/packaging/aleph-vm/DEBIAN/postinst new file mode 100755 index 000000000..1bac33236 --- /dev/null +++ b/packaging/aleph-vm/DEBIAN/postinst @@ -0,0 +1,15 @@ +#!/bin/bash +set -euf -o pipefail + +if ! id -u jailman > /dev/null 2>&1; then + useradd jailman +fi + +# No suggestions since only pure Python dependencies will be required: +pip3 install 'aleph-message==0.1.12' + +mkdir -p /srv/jailer + +systemctl daemon-reload +systemctl enable aleph-vm-supervisor.service +systemctl restart aleph-vm-supervisor.service diff --git a/packaging/aleph-vm/etc/aleph-vm/supervisor.env b/packaging/aleph-vm/etc/aleph-vm/supervisor.env new file mode 100644 index 000000000..d23371e08 --- /dev/null +++ b/packaging/aleph-vm/etc/aleph-vm/supervisor.env @@ -0,0 +1,2 @@ +ALEPH_VM_PRINT_SYSTEM_LOGS=True +ALEPH_VM_USE_JAILER=True diff --git a/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service b/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service new file mode 100644 index 000000000..6f5395176 --- /dev/null +++ b/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service @@ -0,0 +1,16 @@ +[Unit] +Description=Aleph.im VM execution engine +After=network.target + +[Service] +User=0 +Group=0 +WorkingDirectory=/opt/aleph-vm +Environment=PYTHONPATH=/opt/aleph-vm/:$PYTHONPATH +EnvironmentFile=/etc/aleph-vm/supervisor.env +ExecStart=python3 -m vm_supervisor --print-settings --very-verbose +Restart=always +RestartSec=10s + +[Install] +WantedBy=multi-user.target diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 30026230b..b533ada92 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -76,7 +76,7 @@ class Settings(BaseSettings): ALLOW_VM_NETWORKING: bool = True FIRECRACKER_PATH: str = "/opt/firecracker/firecracker" JAILER_PATH: str = "/opt/firecracker/jailer" - LINUX_PATH: str = os.path.abspath("./kernels/vmlinux.bin") + LINUX_PATH: str = "/opt/firecracker/vmlinux.bin" INIT_TIMEOUT: float = 20 CONNECTOR_URL: Url = Url("http://localhost:8000") @@ -105,9 +105,9 @@ def update(self, **kwargs): raise ValueError(f"Unknown setting '{key}'") def check(self): - assert isfile(self.FIRECRACKER_PATH) - assert isfile(self.JAILER_PATH) - assert isfile(self.LINUX_PATH) + assert isfile(self.FIRECRACKER_PATH), f"File not found {self.FIRECRACKER_PATH}" + assert isfile(self.JAILER_PATH), f"File not found {self.JAILER_PATH}" + assert isfile(self.LINUX_PATH), f"File not found {self.LINUX_PATH}" assert self.CONNECTOR_URL.startswith( "http://" ) or self.CONNECTOR_URL.startswith("https://") From e5fca8da05ef56a74a31b1044d5db7a69801ae3c Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 16 Jul 2021 11:51:56 +0200 Subject: [PATCH 138/990] Refactor: Update the VM connector, install it from Docker Hub --- docker/publish_vm_connector.sh | 18 ++++++++++++++++++ docker/vm_connector.dockerfile | 11 ++--------- vm_connector/README.md | 21 ++++++--------------- vm_connector/conf.py | 31 +++++++++++++++++++++---------- vm_connector/main.py | 32 ++++++-------------------------- vm_supervisor/conf.py | 2 +- 6 files changed, 54 insertions(+), 61 deletions(-) create mode 100644 docker/publish_vm_connector.sh diff --git a/docker/publish_vm_connector.sh b/docker/publish_vm_connector.sh new file mode 100644 index 000000000..cf5de87c4 --- /dev/null +++ b/docker/publish_vm_connector.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -euf -o pipefail + +if hash docker 2> /dev/null +then + DOCKER_COMMAND=docker +else + DOCKER_COMMAND=podman +fi + +#VERSION=$(git describe --tags)-alpha +VERSION=alpha + +$DOCKER_COMMAND build -t alephim/vm-connector -f docker/vm_connector.dockerfile . + +$DOCKER_COMMAND tag alephim/vm-connector alephim/vm-connector:$VERSION +$DOCKER_COMMAND push alephim/vm-connector:$VERSION docker.io/alephim/vm-connector:$VERSION +echo docker.io/alephim/pyaleph-node:$VERSION diff --git a/docker/vm_connector.dockerfile b/docker/vm_connector.dockerfile index 0d02c5560..3aebf7478 100644 --- a/docker/vm_connector.dockerfile +++ b/docker/vm_connector.dockerfile @@ -7,16 +7,9 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \ RUN pip install fastapi aiofiles uvicorn aleph-client eth-account -COPY ./examples /opt/examples -COPY ./runtimes /opt/runtimes -COPY ./kernels /opt/kernels - -WORKDIR /opt/examples -RUN make - WORKDIR /opt ENV PYTHONPATH=/opt -EXPOSE 8000 +EXPOSE 4021 COPY ./vm_connector /opt/vm_connector -CMD ["uvicorn", "vm_connector.main:app", "--host", "0.0.0.0", "--reload"] +CMD ["uvicorn", "vm_connector.main:app", "--host", "0.0.0.0", "--port", "4021", "--reload"] diff --git a/vm_connector/README.md b/vm_connector/README.md index e67b4c0d7..acfaf6237 100644 --- a/vm_connector/README.md +++ b/vm_connector/README.md @@ -20,33 +20,24 @@ apt update apt install -y docker.io ``` -### 2.b. Build the Docker image +### 2.b. Pull the Docker image -Clone this reposotiry on the host machine and enter it: ```shell -git clone https://github.com/aleph-im/aleph-vm.git -cd aleph-vm/ -```` - -Build the image: -```shell -docker build -t aleph-connector -f docker/vm_connector.dockerfile . +docker pull alephim/vm-connector:alpha ``` ## 3. Running -### Run the Docker image +Run the Docker image ```shell -docker run -ti --rm -p 8000:8000/tcp aleph-connector +docker run -d -p 4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha ``` -http://localhost:8000/ - ## 4. Configuration -The VM Supervisor can be configured using environment variables: +The VM Supervisor can be configured using environment variables: -`ALEPH_SERVER` should point to your Aleph Node. +`API_SERVER` should point to your Aleph Node. Defaults to https://api2.aleph.im `IPFS_SERVER` should point to your IPFS Gateway, defaults to https://ipfs.aleph.im/ipfs diff --git a/vm_connector/conf.py b/vm_connector/conf.py index cc3c6ffba..893b90a04 100644 --- a/vm_connector/conf.py +++ b/vm_connector/conf.py @@ -1,27 +1,38 @@ -from os import getenv +import logging from typing import NewType +from pydantic import BaseSettings + +logger = logging.getLogger(__name__) + Url = NewType("Url", str) -class Settings: - ALEPH_SERVER: Url = Url(getenv("ALEPH_API_SERVER", "https://api2.aleph.im")) - IPFS_SERVER: Url = Url(getenv("ALEPH_IPFS_SERVER", "https://ipfs.aleph.im/ipfs")) - OFFLINE_TEST_MODE: bool = getenv("ALEPH_OFFLINE_TEST_MODE", "false") == "true" +class ConnectorSettings(BaseSettings): + API_SERVER: Url = "https://api2.aleph.im" + IPFS_SERVER: Url = "https://ipfs.aleph.im/ipfs" + OFFLINE_TEST_MODE: bool = False def update(self, **kwargs): for key, value in kwargs.items(): + if key != key.upper(): + logger.warning(f"Setting {key} is not uppercase") if hasattr(self, key): setattr(self, key, value) else: raise ValueError(f"Unknown setting '{key}'") def display(self) -> str: - result = "" - for annotation, value in self.__annotations__.items(): - result += f"{annotation} ({value.__name__}) = {getattr(self, annotation)}" - return result + return "\n".join( + f"{annotation:<17} = {getattr(self, annotation)}" + for annotation, value in self.__annotations__.items() + ) + + class Config: + env_prefix = "ALEPH_" + case_sensitive = False + env_file = ".env" # Settings singleton -settings = Settings() +settings = ConnectorSettings() diff --git a/vm_connector/main.py b/vm_connector/main.py index d919c6190..c5c303c58 100644 --- a/vm_connector/main.py +++ b/vm_connector/main.py @@ -1,17 +1,14 @@ import json import logging -import os.path from typing import Optional, Dict, Union -from aleph_client.asynchronous import get_posts, create_post +import aiohttp +from aleph_client.asynchronous import create_post from aleph_client.chains.common import get_fallback_private_key from aleph_client.chains.ethereum import ETHAccount - -import aiohttp from fastapi import FastAPI -from fastapi.responses import StreamingResponse, Response, FileResponse - from fastapi import Request +from fastapi.responses import StreamingResponse, Response from pydantic import BaseModel from .conf import settings @@ -30,7 +27,7 @@ def read_root(): async def get_latest_message_amend(ref: str, sender: str) -> Optional[Dict]: async with aiohttp.ClientSession() as session: url = ( - f"{settings.ALEPH_SERVER}/api/v0/messages.json?msgType=STORE&sort_order=-1" + f"{settings.API_SERVER}/api/v0/messages.json?msgType=STORE&sort_order=-1" f"&refs={ref}&addresses={sender}" ) resp = await session.get(url) @@ -44,7 +41,7 @@ async def get_latest_message_amend(ref: str, sender: str) -> Optional[Dict]: async def get_message(hash_: str) -> Optional[Dict]: async with aiohttp.ClientSession() as session: - url = f"{settings.ALEPH_SERVER}/api/v0/messages.json?hashes={hash_}" + url = f"{settings.API_SERVER}/api/v0/messages.json?hashes={hash_}" resp = await session.get(url) resp.raise_for_status() resp_data = await resp.json() @@ -79,11 +76,6 @@ async def download_message( :return: a file containing the code file """ - if settings.OFFLINE_TEST_MODE: - filepath = os.path.abspath("./tests/test_message.json") - with open(filepath) as fd: - return json.load(fd) - msg = await get_message(hash_=ref) # TODO: Validate the validity of the message (signature, hashes) @@ -104,10 +96,6 @@ async def download_code( :return: a file containing the code file """ - if settings.OFFLINE_TEST_MODE: - filepath = os.path.abspath("./examples/example_fastapi_2.zip") - return FileResponse(filepath, filename=f"{ref}") - msg = await get_message(hash_=ref) if not msg: return Response(status_code=404, content="Hash not found") @@ -116,7 +104,7 @@ async def download_code( if msg["content"]["item_type"] == "ipfs": url = f"{settings.IPFS_SERVER}/{data_hash}" else: - url = f"{settings.ALEPH_SERVER}/api/v0/storage/raw/{data_hash}" + url = f"{settings.API_SERVER}/api/v0/storage/raw/{data_hash}" return StreamingResponse(stream_url_chunks(url), media_type="application/zip") @@ -133,10 +121,6 @@ async def download_data( :return: a file containing the data """ - if settings.OFFLINE_TEST_MODE: - filepath = os.path.abspath("./examples/data.tgz") - return FileResponse(filepath, filename=f"{ref}.tgz") - # Download message msg = await get_message(hash_=ref) if not msg: @@ -160,10 +144,6 @@ async def download_runtime( :return: a file containing the runtime """ - if settings.OFFLINE_TEST_MODE: - filepath = os.path.abspath("./runtimes/aleph-alpine-3.13-python/rootfs.ext4") - return FileResponse(filepath, filename=f"{ref}.ext4") - # Download message msg = await get_message(hash_=ref) if not msg: diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index b533ada92..e3c7d0fe3 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -79,7 +79,7 @@ class Settings(BaseSettings): LINUX_PATH: str = "/opt/firecracker/vmlinux.bin" INIT_TIMEOUT: float = 20 - CONNECTOR_URL: Url = Url("http://localhost:8000") + CONNECTOR_URL: Url = Url("http://localhost:4021") CACHE_ROOT: FilePath = FilePath("/tmp/aleph/vm_supervisor") MESSAGE_CACHE: FilePath = FilePath(join(CACHE_ROOT, "message")) From f01156223e6813438003551f9c5cd357a0ae2ab8 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 16 Jul 2021 12:43:50 +0200 Subject: [PATCH 139/990] Documentation: Improve and update, split Caddy setup --- CONFIGURE_CADDY.md | 120 ++++++++++++++++++++++++++++++++++++++++ README.md | 72 +++++++++++++++++++++--- vm_supervisor/README.md | 116 -------------------------------------- 3 files changed, 185 insertions(+), 123 deletions(-) create mode 100644 CONFIGURE_CADDY.md diff --git a/CONFIGURE_CADDY.md b/CONFIGURE_CADDY.md new file mode 100644 index 000000000..383d2d8ac --- /dev/null +++ b/CONFIGURE_CADDY.md @@ -0,0 +1,120 @@ +# Caddy Reverse-proxy for Aleph-VM + +A reverse-proxy is required for production use. It allows: + + - A different domain name for each VM function + - Secure connections using HTTPS + - Load balancing between multiple servers + +Using a different domain name for each VM function is important when running web applications, +both for security and usability purposes. + +The VM Supervisor supports using domains in the form `https://identifer.vm.yourdomain.org`, where +_identifier_ is the identifier/hash of the message describing the VM function and `yourdomain.org` +represents your domain name. + +## 1. Wildcard certificates + +A wildcard certificate is recommended to allow any subdomain of your domain to work. + +You can create one using [Let's Encrypt](https://letsencrypt.org/) and +[Certbot](https://certbot.eff.org/) with the following instructions. + +```shell +sudo apt install -y certbot + +certbot certonly --manual --email email@yourdomain.org --preferred-challenges dns \ + --server https://acme-v02.api.letsencrypt.org/directory --agree-tos \ + -d 'vm.yourdomain.org,*.vm.youdomain.org' +``` + +## 2. Caddy Server + +In this documentation, we will install the modern [Caddy](https://caddyserver.com/) reverse-proxy. + +Replace `vm.yourdomain.org` with your domain of choice. + +To install on Debian/Ubuntu, according to the +[official instructions](https://caddyserver.com/docs/install#debian-ubuntu-raspbian): +```shell +sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https +curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo apt-key add - +curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list +sudo apt update +sudo apt install caddy +``` + +Then give Caddy access to the certificates generated by Certbot: +```shell +chmod 750 /etc/letsencrypt/live/ +chmod 750 /etc/letsencrypt/archive/ +chmod 640 /etc/letsencrypt/archive/vm.yourdomain.org/privkey1.pem +chgrp -R caddy /etc/letsencrypt/archive/ +chgrp -R caddy /etc/letsencrypt/live/ +``` + +Configure Caddy: +```shell +cat >/etc/caddy/Caddyfile </etc/caddy/Caddyfile < Note: This project is still early prototyping. -The Aleph VM project allows you to run programs on [Aleph.im](https://aleph.im/). +The Aleph-VM project allows you to run programs on [Aleph.im](https://aleph.im/). -These programs can currently be written in Python using ASGI compatible frameworks ( +Programs can currently be written in Python using ASGI compatible frameworks ( [FastAPI](https://github.com/tiangolo/fastapi), -[Django](https://docs.djangoproject.com/en/3.0/topics/async/), -[Sanic](https://sanicframework.org/), -...) and respond to HTTP requests. +[Django](https://docs.djangoproject.com/en/3.0/topics/async/), +...) and respond to HTTP requests. -## Architecture +Alternatively, programs written in any language can listen to HTTP requests on port 8080. + +### 1. Writing Aleph-VM programs + +Have a look at [examples/example_fastapi_2](examples/example_fastapi_2) for an example of VM. + +## 1. Quick install + +To quickly install Aleph-VM on a [supported Linux system](./vm_supervisor/README.md#1-supported-platforms) +for production purposes: + +```shell +sudo apt update +sudo apt install -y docker.io +sudo docker run -d -p 4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha +wget +sudo apt install ./aleph-vm.deb +``` + +### Reverse Proxy + +We document how to use Caddy as a reverse proxy since it does automatic HTTPS certificates. + +First, create a domain name that points to the server on IPv4 and IPv6. + +This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CONFIGURE_CADDY.md). +```shell +sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https +curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo apt-key add - +curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list +sudo apt update +sudo apt install caddy + +cat >/etc/caddy/Caddyfile </etc/caddy/Caddyfile </etc/caddy/Caddyfile < Date: Fri, 16 Jul 2021 14:50:48 +0200 Subject: [PATCH 140/990] Documentation: Add a tutorial to create programs --- tutorials/README.md | 142 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 tutorials/README.md diff --git a/tutorials/README.md b/tutorials/README.md new file mode 100644 index 000000000..ff05d0a3c --- /dev/null +++ b/tutorials/README.md @@ -0,0 +1,142 @@ +# Tutorial: Creating and hosting a program on Aleph-VM + +This is the tutorial for Creating and hosting a program on Aleph-VM, which has been developed and maintained by Aleph.im. + +## Welcome + +Hi, welcome to _Creating and hosting a program on Aleph-VM_. In this tutorial we will take you +through the fundamentals of running programs on the [Aleph.im](https://aleph.im/) Virtual Machines. +After the tutorial you should have a rough mental picture of how the virtual machines work and +some good pointers for getting further with running programs of your own. + +We expect you to know a little Python and have some experience with +the [FastAPI framework](https://fastapi.tiangolo.com/). +The first chapters of the [FastAPI Tutorial](https://fastapi.tiangolo.com/tutorial/) should cover +enough to get started. + +You will need a recent version of Python and [pip](https://pip.pypa.io/en/stable/), +preferably running on Debian or Ubuntu Linux since we have not tested other platforms yet, +but feel free to use the platform of your choice if you have the skills to adapt our instructions to it. + +## What we will cover + +First we will see how to run the first example from FastAPI's tutorial on Aleph.im, how to +access it and how to update it. + +Then we will extend the program to add some Aleph specific functionalities. + +## Understanding the VMs + +Aleph is a cross-blockchain layer-2 network specifically focused on decentralized applications and +their related infrastructure (storage, computing servers, security). + +Aleph-VM is the computing part of the network: It allows the execution of programs stored on the +Aleph network. These programs can interact with the network itself and with the rest of the internet. + +In the current stage, these programs can only be triggered from outside HTTP calls. Future ways to +trigger the launch of the programs are planned, such as reacting to specific messages on the +network. + +## 1. Writing + +To create the first program, open your favourite code editor and create a directory named +`my-program`, containing a file named `app.py`. + +Then write the following code in file: +```python +from fastapi import FastAPI + +app = FastAPI() + + +@app.get("/") +async def root(): + return {"message": "Hello World"} +``` + +That's it for your first program. + +## 2. Testing locally + +Before uploading your program on Aleph, it is best to first test it locally. + +Do test your progam using uvicorn: + +```shell +pip install uvicorn[standard] +uvicorn main:app --reload +``` + +Then open http://127.0.0.1:8000 . + +## 3. Uploading + +Install [aleph-client](https://github.com/aleph-im/aleph-client). + +```shell +sudo apt-get install -y squashfs-tools libsecp256k1-dev +pip install aleph-client +``` + +```shell +aleph --help +``` + +Upload your program: + +```shell +aleph program ./my-program main:app +``` + +Press Enter at the following prompt to use the default runtime: +``` +Ref of runtime ? [bd79839bf96e595a06da5ac0b6ba51dea6f7e2591bb913deccded04d831d29f4] +``` + +Press Enter again to skip adding extra volumes to your program: +``` +Add volume ? [y/N] +``` + +You should then get a response similar to the following: +``` +Your program has been uploaded on Aleph . + +Available on: + https://aleph.sh/vm/1d3842fc4257c0fd4f9c7d5c55bba16264de8d44f47265a14f8f6eb4d542dda2 + https://du4ef7cck7ap2t44pvoflo5bmjsn5dke6rzglikpr5xljvkc3wra.aleph.sh +Visualise on: + https://explorer.aleph.im/address/ETH/0x101d8D16372dBf5f1614adaE95Ee5CCE61998Fc9/message/PROGRAM/1d3842fc4257c0fd4f9c7d5c55bba16264de8d44f47265a14f8f6eb4d542dda2 +``` + +You may get the warning `Message failed to publish on IPFS and/or P2P`. +This is common and usually not an issue. + +## 4. Running + +You can now run your program by opening one of the URLs above. Each URL is unique for one program. + + +https://aleph.sh/vm/1d3842fc4257c0fd4f9c7d5c55bba16264de8d44f47265a14f8f6eb4d542dda2 + +## 5. Uploading updates + +`"Hello World"` is a nice message, but wouldn't it be nice to have something more friendly, such +as `"Hello Friend"` ? Update the program with the message of your choice. + +You could upload the new version as a new program, but this would break the URL above and you +would have to give the updated URL to all your friends. While Aleph messages cannot be edited, +there is a solution to this issue: you can publish _amend_ messages that reference the original +message to add some changes to it. + +The `aleph update` command is similar to `aleph program`, except it requires the hash of the +program to update. + +```shell +aleph update $HASH ./my-program +``` + +Note that _amend_ messages must be sent from the same Aleph address as the original +program to work, else they will be ignored. + +| ℹ️ Backup your private key, else you may lose the ability to update a program From c58fabd1c72c9872726411a5d257783bbcb0b3cd Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 16 Jul 2021 14:55:47 +0200 Subject: [PATCH 141/990] Documentation: Start advanced usage tutorial --- README.md | 5 +- tutorials/ADVANCED.md | 146 ++++++++++++++++++++++++++++++++++++++++++ tutorials/README.md | 2 + 3 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 tutorials/ADVANCED.md diff --git a/README.md b/README.md index 42da19e4a..6b725a1f7 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,10 @@ Alternatively, programs written in any language can listen to HTTP requests on p ### 1. Writing Aleph-VM programs -Have a look at [examples/example_fastapi_2](examples/example_fastapi_2) for an example of VM. +Have a look at [tutorials/README.md](tutorials/README.md) for a tutorial on how to program VMs +as a user. + +The rest of this document focuses on how to run an Aleph-VM node that hosts and executes the programs. ## 1. Quick install diff --git a/tutorials/ADVANCED.md b/tutorials/ADVANCED.md new file mode 100644 index 000000000..291cef381 --- /dev/null +++ b/tutorials/ADVANCED.md @@ -0,0 +1,146 @@ +# Tutorial: Advanced usage of Aleph-VM + +## Aleph messages + +The [aleph-client](https://github.com/aleph-im/aleph-client) library is pre-installed and +pre-configured in the official Aleph-VM Python runtime. It is tweaked to work even +for programs with the access to internet disabled. + +### Get messages + +Use `aleph_client.asynchronous.get_messages` to get messages from the Aleph network. + +```python +from aleph_client.asynchronous import get_messages + +(...) +messages = await get_messages( + hashes=["f246f873c3e0f637a15c566e7a465d2ecbb83eaa024d54ccb8fb566b549a929e"] +) +``` + +## Post Aleph messages + +ℹ️ Messages posted by VMs may not be authorized by the Aleph network yet. + +Posting messages on the Aleph network requires signing them using a valid account. +Since programs on Aleph-VM are public, they should not contain secrets. Instead of signing messages +themselves, programs should therefore ask their execution host to sign messages on their behalf +using a `RemoteAccount`. The hash of the VM will be referenced in the message content `'address'` +field. + +```python +from aleph_client.chains.remote import RemoteAccount + +(...) + +account = await RemoteAccount.from_crypto_host( + host="http://localhost", unix_socket="/tmp/socat-socket") + +content = { + "date": datetime.utcnow().isoformat(), + "test": True, + "answer": 42, + "something": "interesting", +} +response = await create_post( + account=account, + post_content=content, + post_type="test", + ref=None, + channel="TEST", + inline=True, + storage_engine="storage", +) +``` + +## Shared cache + +The shared cache is a simple key-value store available to programs to store information that would +be useful to persist between executions but can be recovered from other sources. +The cache is specific to one program on one execution node. + +The persistence of the cache should not be relied on - its content can be deleted anytime when +the program is not running. Important data must be persisted on the Aleph network. + +To use the cache, you can use the following methods: +```python +from aleph_client.vm.cache import VmCache +cache = VmCache() + +async def f(): + await cache.set('key', 'value') + value = await cache.get('key') + await cache.delete('key') +``` + +## Volumes + +Volumes consist in extra storage that can be used by programs on Aleph-VM. If a `mount` point +is specified, they will be mounted on the virtual machine filesystem before your program is +started. + +### Immutable volumes + +Immutable volumes contain extra files that can be used by a program and are stored on the Aleph +network. They can be shared by multiple programs and updated independently of the code of the program. + +You can use them to store Python libraries that your program depends on, use them in multiple +programs and update them independently of other programs. + +#### 1. Create an immutable volume + +Create with a volume containing a Python library: + +```shell +mkdir extralib +cd extralib +mksquashfs extralib extra-lib.squashfs +``` + +Start an IPFS daemon: +```shell +ipfs daemon +``` + +Upload the volume to IPFS: +```shell +ipfs add extra-lib.squashfs +``` +and retrieve the printed IPFS hash. + +Pin the volume on Aleph using `aleph pin`: +```shell +aleph pin $IPFS_HASH --channel TEST +``` + +Mention the volume in the prompt of `aleph program (...)` + +#### 2. Update an immutable volume + +Follow the same procedure you used to create an immutable volume, but pin it with a +reference to the original using: + +```shell +aleph pin $IPFS_HASH --channel TEST --ref $ORIGINAL_HASH +``` + +### Host persistent volumes + +Host persistent volumes are empty volumes that your program can use to store information that +would be useful to persist between executions but can be recovered from other sources. +Like the cache, host persistent volumes are specific to one program on one execution node. + +Unlike the cache, you can use these volumes to store any kind of files, including databases. + +There is no guarantee that these volumes will not be deleted anytime when the +program is not running and important data must be persisted on the Aleph network. + +Host persistend volumes have a fixed size and must be named. The name will be used in the future +to allow changing the mount point of a volume. + + +## Message structure + +Full example +https://github.com/aleph-im/aleph-message/blob/main/aleph_message/tests/messages/machine.json diff --git a/tutorials/README.md b/tutorials/README.md index ff05d0a3c..bf99343d6 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -140,3 +140,5 @@ Note that _amend_ messages must be sent from the same Aleph address as the origi program to work, else they will be ignored. | ℹ️ Backup your private key, else you may lose the ability to update a program + +Check out the [Advanced usage](./ADVANCED.md) page for more options and capabilities. From b0514858052e4073f475e5c281b9734745920cad Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 16 Jul 2021 17:56:24 +0200 Subject: [PATCH 142/990] Documentation: Fix .deb path in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b725a1f7..9bc0c9688 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ for production purposes: sudo apt update sudo apt install -y docker.io sudo docker run -d -p 4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha -wget +wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/aleph-vm-0.1.0.deb sudo apt install ./aleph-vm.deb ``` From 5b4772c0628a5c9e06aa613e19b616296084df44 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 16 Jul 2021 17:56:59 +0200 Subject: [PATCH 143/990] Documentation: Fix filename inconsistency --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9bc0c9688..0fb1f3753 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ sudo apt update sudo apt install -y docker.io sudo docker run -d -p 4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/aleph-vm-0.1.0.deb -sudo apt install ./aleph-vm.deb +sudo apt install ./aleph-vm-0.1.0.deb ``` ### Reverse Proxy From 5c212db3fa8b58de3105c686862f910d572816ef Mon Sep 17 00:00:00 2001 From: aliel Date: Thu, 22 Jul 2021 23:53:48 +0200 Subject: [PATCH 144/990] fix msubscribe typo --- vm_supervisor/models.py | 2 +- vm_supervisor/pubsub.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index 40222b89f..420a4a11a 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -145,7 +145,7 @@ def start_watching_for_updates(self, pubsub: PubSub): pool.create_task(self.watch_for_updates(pubsub=pubsub)) async def watch_for_updates(self, pubsub: PubSub): - await pubsub.msubscibe( + await pubsub.msubscribe( self.original.code.ref, self.original.runtime.ref, self.original.data.ref if self.original.data else None, diff --git a/vm_supervisor/pubsub.py b/vm_supervisor/pubsub.py index 40e1c5406..e594c553c 100644 --- a/vm_supervisor/pubsub.py +++ b/vm_supervisor/pubsub.py @@ -21,7 +21,7 @@ async def subscribe(self, key): self.subscribers.setdefault(key, set()).add(queue) return await queue.get() - async def msubscibe(self, *keys): + async def msubscribe(self, *keys): """Subscribe to multiple keys""" keys = tuple(key for key in keys if key is not None) logger.debug(f"msubscribe({keys})") From 057ca42d56e98189bc5d0d9d3e14f1644fc043f2 Mon Sep 17 00:00:00 2001 From: aliel Date: Thu, 22 Jul 2021 12:23:06 +0200 Subject: [PATCH 145/990] [Doc] fix filename according to the commandline app.py -> main.py --- tutorials/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/README.md b/tutorials/README.md index bf99343d6..ae8e60d5f 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -40,7 +40,7 @@ network. ## 1. Writing To create the first program, open your favourite code editor and create a directory named -`my-program`, containing a file named `app.py`. +`my-program`, containing a file named `main.py`. Then write the following code in file: ```python From 866b5c99239c1aac8294746c9617884b8bf48112 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 27 Jul 2021 11:02:13 +0200 Subject: [PATCH 146/990] Update Architecture schema --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0fb1f3753..16e1224a8 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ https://vm.yourdomain.org/vm/17412050fa1c103c41f983fe305c1ce8c6a809040762cdc1614 ## 2. Architecture -![image](https://user-images.githubusercontent.com/404665/115885445-452f5180-a450-11eb-856e-f4071023a105.png) +![Aleph im VM - Details](https://user-images.githubusercontent.com/404665/127126908-3225a633-2c36-4129-8766-9810f2fcd7d6.png) ### VM Supervisor @@ -83,8 +83,7 @@ See [vm_supervisor/README.md](./vm_supervisor/README.md). ### VM Connector -Schedules the execution of programs on VM Supervisors and assists -them with operations related to the Aleph network. +Assist with operations related to the Aleph network. See [vm_connector/README.md](./vm_connector/README.md). From 5a2c057597341b346fb10522b4ec25d2ad8bd77a Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 28 Jul 2021 11:22:53 +0200 Subject: [PATCH 147/990] Fix: AWS vmlinux kernel is not compatible --- vm_supervisor/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index 13c854b97..3f0173786 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -99,13 +99,13 @@ mkdir /srv/jailer ### 2.g. Download a Linux kernel -This downloads the example kernel built by the Firecracker team. +This downloads an optimized kernel built by the Aleph team. -A more optimized kernel will be made available in the future. +A more optimized kernel may be made available in the future. See section _Compile your kernel_ below to build your own. ```shell -curl -fsSL -o ./kernels/vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin +curl -fsSL -o /opt/firecracker/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/vmlinux.bin ``` ## 3. Running From 84d2dd741f3ac0250392f4fd2bec8a958b76a72e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 28 Jul 2021 11:51:49 +0200 Subject: [PATCH 148/990] Security fix: Document binding connector on 127.0.0.1 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 16e1224a8..cfe3dafed 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ for production purposes: ```shell sudo apt update sudo apt install -y docker.io -sudo docker run -d -p 4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha +sudo docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/aleph-vm-0.1.0.deb sudo apt install ./aleph-vm-0.1.0.deb ``` From 80b3f3846293819c2513ff1c8a574b200fb2156e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 24 Aug 2021 12:23:20 +0200 Subject: [PATCH 149/990] Feature: Allow configuration of VM supervisor host --- vm_supervisor/conf.py | 1 + vm_supervisor/supervisor.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index e3c7d0fe3..5ddfb70f0 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -58,6 +58,7 @@ def systemd_resolved_dns_servers(interface): class Settings(BaseSettings): + SUPERVISOR_HOST: str = "127.0.0.1" SUPERVISOR_PORT: int = 4020 START_ID_INDEX: int = 4 diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 8ea569570..93c257b2b 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -46,4 +46,5 @@ def run(): app.on_startup.append(start_watch_for_messages_task) app.on_cleanup.append(stop_watch_for_messages_task) - web.run_app(app, port=settings.SUPERVISOR_PORT) + web.run_app(app, host=settings.SUPERVISOR_HOST, + port=settings.SUPERVISOR_PORT) From 5ad859eb8c882af7e56a19885216f4a6d5fe7711 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 24 Aug 2021 12:23:54 +0200 Subject: [PATCH 150/990] Fix: VM connector port update was missing from guest API --- guest_api/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guest_api/__main__.py b/guest_api/__main__.py index 21a8aa2da..a8b799d3f 100644 --- a/guest_api/__main__.py +++ b/guest_api/__main__.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) ALEPH_API_SERVER = "https://api2.aleph.im" -ALEPH_VM_CONNECTOR = "http://localhost:8000" +ALEPH_VM_CONNECTOR = "http://localhost:4021" CACHE_EXPIRES_AFTER = 7 * 24 * 3600 # Seconds From 930119e3dd3c1c37ea7e9ce0f4560a32c94510c4 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 24 Aug 2021 12:25:00 +0200 Subject: [PATCH 151/990] Fix: Example crashed when directory was missing --- examples/example_fastapi_2/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/example_fastapi_2/main.py b/examples/example_fastapi_2/main.py index b99d54bae..8b22ce98e 100644 --- a/examples/example_fastapi_2/main.py +++ b/examples/example_fastapi_2/main.py @@ -1,8 +1,8 @@ import json import logging +import os from datetime import datetime from os import listdir -from fastapi import Request from pydantic import BaseModel logger = logging.getLogger(__name__) @@ -25,12 +25,16 @@ @app.get("/") async def index(): + if os.path.exists("/opt/venv"): + opt_venv = list(listdir("/opt/venv")) + else: + opt_venv = [] return { "Example": "example_fastapi_2", "endpoints": ["/messages", "/internet", "/post_a_message", "/state/increment", "/wait-for/{delay}"], "files_in_volumes": { - "/opt/venv": list(listdir("/opt/venv")) + "/opt/venv": opt_venv, }, } From 7b143a2a36a7e614bfee075019a757f333ad41a4 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 1 Sep 2021 18:54:00 +0200 Subject: [PATCH 152/990] Doc: Improve the tutorial and add Rust example --- tutorials/README.md | 117 ++++++++++++++++++++++++++++++++++---------- tutorials/SERVER.md | 96 ++++++++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+), 25 deletions(-) create mode 100644 tutorials/SERVER.md diff --git a/tutorials/README.md b/tutorials/README.md index ae8e60d5f..2330697c3 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -1,23 +1,19 @@ # Tutorial: Creating and hosting a program on Aleph-VM -This is the tutorial for Creating and hosting a program on Aleph-VM, which has been developed and maintained by Aleph.im. +This is the tutorial for Creating and hosting a program on Aleph-VM, which has been developed and maintained by [Aleph.im](https://www.aleph.im). -## Welcome +## 0. Welcome Hi, welcome to _Creating and hosting a program on Aleph-VM_. In this tutorial we will take you through the fundamentals of running programs on the [Aleph.im](https://aleph.im/) Virtual Machines. After the tutorial you should have a rough mental picture of how the virtual machines work and some good pointers for getting further with running programs of your own. -We expect you to know a little Python and have some experience with -the [FastAPI framework](https://fastapi.tiangolo.com/). +We expect you to know a little Python and have some experience with Python web frameworks such as +[FastAPI](https://fastapi.tiangolo.com/) or Flask. The first chapters of the [FastAPI Tutorial](https://fastapi.tiangolo.com/tutorial/) should cover enough to get started. -You will need a recent version of Python and [pip](https://pip.pypa.io/en/stable/), -preferably running on Debian or Ubuntu Linux since we have not tested other platforms yet, -but feel free to use the platform of your choice if you have the skills to adapt our instructions to it. - ## What we will cover First we will see how to run the first example from FastAPI's tutorial on Aleph.im, how to @@ -25,7 +21,24 @@ access it and how to update it. Then we will extend the program to add some Aleph specific functionalities. -## Understanding the VMs +## Requirements + +You will need a recent version of Python and [pip](https://pip.pypa.io/en/stable/), +preferably running on Debian or Ubuntu Linux since we have not tested other platforms yet, +but feel free to use the platform of your choice if you have the skills to adapt our instructions to it. + +Some cryptographic functionalities of Aleph use curve secp256k1 and require installing [libsecp256k1](https://github.com/bitcoin-core/secp256k1). +Archiving programs and volumes requires +[Squashfs user space tools](https://github.com/plougher/squashfs-tools). + + sudo apt-get install -y python3-pip libsecp256k1-dev squashfs-tools + +You will also need [Uvicorn](https://www.uvicorn.org/) for local testing +and the [Python Aleph client](https://github.com/aleph-im/aleph-client) for it's command-line tools: + + pip3 install uvicorn[standard] aleph-client + +## 1. Understanding the VMs Aleph is a cross-blockchain layer-2 network specifically focused on decentralized applications and their related infrastructure (storage, computing servers, security). @@ -33,16 +46,65 @@ their related infrastructure (storage, computing servers, security). Aleph-VM is the computing part of the network: It allows the execution of programs stored on the Aleph network. These programs can interact with the network itself and with the rest of the internet. -In the current stage, these programs can only be triggered from outside HTTP calls. Future ways to +In the current stage, these programs can be triggered from outside HTTPS calls. Future ways to trigger the launch of the programs are planned, such as reacting to specific messages on the network. -## 1. Writing +### Virtual Machines + +Programs on Aleph run within virtual machines: emulated computer systems with dedicated +resources that run isolated from each other. + +Aleph Virtual Machines (VMs) are based on Linux and +use [Firecracker](https://firecracker-microvm.github.io/) under the hood. They boot very fast, +so they can be launched on demand and there is no need to keep them running while waiting for new +requests. + +Each program runs on its own dedicated Linux system, with the host providing additional +functionalities related to Aleph. + +### Runtime + +The base of each VM is a Linux +[root filesystem](https://en.wikipedia.org/wiki/Root_directory) named __runtime__ and configured +to run programs on the Aleph platform. + +Aleph provides a supported runtime to launch programs written in Python or binaries. +* Python programs must support the [ASGI interface](https://asgi.readthedocs.io/en/latest/), described in the example below. +* Binaries must listen for HTTP requests on port 8080 + +The runtime currently supported by Aleph is +[aleph-debian-11-python](../runtimes/aleph-debian-11-python). + +### Volumes + +VMs can be extended by specifying additional volumes that will be mounted in the system. + +**Read-only volumes** are useful to separate Python virtual environments, Javascript _node_modules_ +or static data from the program itself. These volumes can be updated independently from the +program and the runtime, and maintained by a third party. + +**Ephemeral volumes** provide temporary disk storage to a VM during its execution without requiring +more memory. + +**Host persistent volumes** are persisted on the VM execution node, but may be garbage collected +by the node without warning. + +**Store persistent volumes** (not available yet) are persisted on the Aleph network. New VMs will try to use the latest +version of this volume, with no guarantee against conflicts. + +## 2. Writing a Python program To create the first program, open your favourite code editor and create a directory named `my-program`, containing a file named `main.py`. -Then write the following code in file: +``` +. +└── my-program/ + └── main.py +``` + +Then write the following code in the file: ```python from fastapi import FastAPI @@ -56,27 +118,29 @@ async def root(): That's it for your first program. -## 2. Testing locally +This code comes from the [FastAPI tutorial](https://fastapi.tiangolo.com/tutorial/first-steps/). +Have a look at it for a better understanding of what it does and how it works. + +## 3. Testing locally Before uploading your program on Aleph, it is best to first test it locally. -Do test your progam using uvicorn: +Aleph uses the standard [ASGI interface](https://asgi.readthedocs.io/en/latest/introduction.html) to +interface with programs written in Python. ASGI interfaces with many Python frameworks, including +FastAPI but also [Django](https://www.djangoproject.com/) +or [Quart](https://github.com/pgjones/quart). + +Test your progam locally using uvicorn, an ASGI server: ```shell -pip install uvicorn[standard] uvicorn main:app --reload ``` Then open http://127.0.0.1:8000 . -## 3. Uploading +## 4. Uploading -Install [aleph-client](https://github.com/aleph-im/aleph-client). - -```shell -sudo apt-get install -y squashfs-tools libsecp256k1-dev -pip install aleph-client -``` +After installing [aleph-client](https://github.com/aleph-im/aleph-client), you should have access to the `aleph` command: ```shell aleph --help @@ -112,14 +176,13 @@ Visualise on: You may get the warning `Message failed to publish on IPFS and/or P2P`. This is common and usually not an issue. -## 4. Running +## 5. Running You can now run your program by opening one of the URLs above. Each URL is unique for one program. - https://aleph.sh/vm/1d3842fc4257c0fd4f9c7d5c55bba16264de8d44f47265a14f8f6eb4d542dda2 -## 5. Uploading updates +## 6. Uploading updates `"Hello World"` is a nice message, but wouldn't it be nice to have something more friendly, such as `"Hello Friend"` ? Update the program with the message of your choice. @@ -141,4 +204,8 @@ program to work, else they will be ignored. | ℹ️ Backup your private key, else you may lose the ability to update a program +## Next steps + +Check out the [Writing a non-Python program](./SERVER.md) page to run a program written in another language than Python. + Check out the [Advanced usage](./ADVANCED.md) page for more options and capabilities. diff --git a/tutorials/SERVER.md b/tutorials/SERVER.md new file mode 100644 index 000000000..c34548364 --- /dev/null +++ b/tutorials/SERVER.md @@ -0,0 +1,96 @@ +# Tutorial: Creating a non-Python program on Aleph-VM + +> This tutorial follows up the first tutorial [Creating and hosting a program on Aleph-VM](./README.md). + +## 0. Welcome + +In this second tutorial, we will guide you on how to run programs written in any programming language on Aleph Virtual Machines. + +In addition to running Python programs using ASGI as covered in the first tutorial, +Aleph VMs also support any program that listens for HTTP requests on port 8080. + +This can be used to run existing programs on Aleph VMs, or to use other programming languages to write programs and run them on Aleph-VM. + +### What we will cover + +Since Python is the only language currently supported, this tutorial we will cover two other languages: [Rust](https://www.rust-lang.org/) and Javascript ([NodeJS](https://nodejs.org/)). + +## 1. Rust + +In this first section, you will run a program written in Rust on an Aleph VM. + +### 1.a. Requirements + +You need a Rust compiler. You can install one using the [official Install Rust guide](https://www.rust-lang.org/tools/install) +or via your favourite package manager. + + $ sudo apt install rustc cargo + +### 1.b. Writing a Rust program + +Let's use a very simple HTTP server inspired by the [Building a Single-Threaded Web Server](https://doc.rust-lang.org/book/ch20-01-single-threaded.html) +section of The Rust Programming Language Book: + +```shell +$ cargo new example_http_rust + Created binary (application) `example_http_rust` project +$ cd example_http_rust +``` + +Filename: `src/main.rs` +```rust +use std::io::prelude::*; +use std::net::TcpListener; +use std::net::TcpStream; + +fn main() { + + let listener = TcpListener::bind("0.0.0.0:8080").unwrap(); + println!("Running on 0.0.0.0:8080"); + for stream in listener.incoming() { + let stream = stream.unwrap(); + handle_connection(stream); + } +} + +fn handle_connection(mut stream: TcpStream) { + println!("handling connection"); + + const MSG: &str = "helloworld"; + let msg = MSG.as_bytes(); + + let response = format!("{:x?}", msg); + + let mut buffer = [0; 1024]; + + stream.read(&mut buffer).unwrap(); + + let response = format!("HTTP/1.1 200 OK\n\nOKIDOK\n{}", response); + + stream.write(response.as_bytes()).unwrap(); + stream.flush().unwrap(); +} +``` + +```shell +cargo run +``` + +Open http://127.0.0.1:8080 in your browser to test your new server. + +### 1.c. Publishing a Rust program + +Compile your program: +```shell +cargo build --release +``` + +Publish it on Aleph using the same procedure as with the Python example, except the entrypoint refers to the name of the binary to execute. + +```shell +aleph program ./target/release/example_http_rust example_http_rust +``` + +If your program takes some arguments, pass them in the entrypoint by using quotes: `"example_http_rust --help`. + +ℹ️ If you get the error `Invalid zip archive`, you are probably missing the Squashfs user tool `mksquashfs`. In that case, first create the squashfs archive and then upload it using `aleph program ./target/release/example_http_rust.squashfs example_http_rust` From cea89a89cc93717a02b0adcde62179d5c56b2bdf Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 15 Sep 2021 10:02:42 +0200 Subject: [PATCH 153/990] Fix: Pool stored VM with failed setup (#80) The first request was returning an error message to clients, but subsequent requests were keeping the connection open with no response. --- vm_supervisor/pool.py | 3 +++ vm_supervisor/run.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index 31f87ed45..f380d3f85 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -44,3 +44,6 @@ async def get_running_vm(self, vm_hash: VmHash) -> Optional[VmExecution]: return execution else: return None + + def forget_vm(self, vm_hash: VmHash) -> None: + self.executions.pop(vm_hash) diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py index b1ffa4ace..c276b6ef8 100644 --- a/vm_supervisor/run.py +++ b/vm_supervisor/run.py @@ -50,12 +50,15 @@ async def run_code(vm_hash: VmHash, path: str, request: web.Request) -> web.Resp ) except ResourceDownloadError as error: logger.exception(error) + pool.forget_vm(vm_hash=vm_hash) raise HTTPBadRequest(reason="Code, runtime or data not available") except VmSetupError as error: logger.exception(error) + pool.forget_vm(vm_hash=vm_hash) raise HTTPInternalServerError(reason="Error during program initialisation") except MicroVMFailedInit as error: logger.exception(error) + pool.forget_vm(vm_hash=vm_hash) raise HTTPInternalServerError(reason="Error during runtime initialisation") logger.debug(f"Using vm={execution.vm.vm_id}") From fb62e9235b901e7673f72c30ab5c9da22875ecb6 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 16 Sep 2021 12:30:18 +0200 Subject: [PATCH 154/990] Fix: Python tutorial requires FastAPI --- tutorials/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutorials/README.md b/tutorials/README.md index 2330697c3..6bc9e3a65 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -24,7 +24,7 @@ Then we will extend the program to add some Aleph specific functionalities. ## Requirements You will need a recent version of Python and [pip](https://pip.pypa.io/en/stable/), -preferably running on Debian or Ubuntu Linux since we have not tested other platforms yet, +preferably running on Debian 11 or Ubuntu Linux since we have not tested other platforms yet, but feel free to use the platform of your choice if you have the skills to adapt our instructions to it. Some cryptographic functionalities of Aleph use curve secp256k1 and require installing [libsecp256k1](https://github.com/bitcoin-core/secp256k1). @@ -36,7 +36,7 @@ Archiving programs and volumes requires You will also need [Uvicorn](https://www.uvicorn.org/) for local testing and the [Python Aleph client](https://github.com/aleph-im/aleph-client) for it's command-line tools: - pip3 install uvicorn[standard] aleph-client + pip3 install uvicorn[standard] aleph-client fastapi ## 1. Understanding the VMs From 1c047228a42b6fd40c043a600422d4989430f5c1 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 16 Sep 2021 12:33:10 +0200 Subject: [PATCH 155/990] Fix: aleph-client CLI requires eth_account --- tutorials/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/README.md b/tutorials/README.md index 6bc9e3a65..981cd8f2d 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -36,7 +36,7 @@ Archiving programs and volumes requires You will also need [Uvicorn](https://www.uvicorn.org/) for local testing and the [Python Aleph client](https://github.com/aleph-im/aleph-client) for it's command-line tools: - pip3 install uvicorn[standard] aleph-client fastapi + pip3 install uvicorn[standard] aleph-client fastapi eth_account ## 1. Understanding the VMs From 676f640342d7109e5c95ad6b1a19210657ac770f Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 23 Sep 2021 11:51:28 +0200 Subject: [PATCH 156/990] Fix: Ethereum tools require eth_account, absent from runtime (#85) --- runtimes/aleph-debian-11-python/create_disk_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index 25eba8b49..a2f58c18b 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -24,7 +24,7 @@ apt-get install -y --no-install-recommends --no-install-suggests \ pip3 install fastapi django echo "Pip installing aleph-client" -pip3 install 'aleph-client>=0.2.7' 'coincurve==15.0.0' +pip3 install 'aleph-client>=0.2.7' 'coincurve==15.0.0' 'eth_account>=0.4.0' # Compile all Python bytecode python3 -m compileall -f /usr/local/lib/python3.9 From f44e9c25e3404bbee946408df93943335a912dff Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 23 Sep 2021 11:59:37 +0200 Subject: [PATCH 157/990] Doc: Document custom domains --- tutorials/ADVANCED.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tutorials/ADVANCED.md b/tutorials/ADVANCED.md index 291cef381..7b5ba271f 100644 --- a/tutorials/ADVANCED.md +++ b/tutorials/ADVANCED.md @@ -144,3 +144,14 @@ to allow changing the mount point of a volume. Full example https://github.com/aleph-im/aleph-message/blob/main/aleph_message/tests/messages/machine.json + +## Custom domains + +You can make your own domain point to a VM. Do achieve this, you need to create the following DNS +records: + +1. A `CNAME` record to the server, for example: +`hosted-on-aleph.net IN CNAME aleph.sh` +2. A `TXT` record to the VM hash with the prefix _aleph-id, for example: +`_aleph-id.hosted-on-aleph.org 60 IN TXT "b34f193470c349b1d9b60903a6d172e8c335710736d4999ff05971692febe8bc"` + From 1b4920bec211ef3bd379e9359f57f06b9308c1a1 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 23 Sep 2021 12:00:02 +0200 Subject: [PATCH 158/990] Doc: Improve tutorial doc --- tutorials/README.md | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tutorials/README.md b/tutorials/README.md index 981cd8f2d..7f911d9d5 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -23,8 +23,13 @@ Then we will extend the program to add some Aleph specific functionalities. ## Requirements -You will need a recent version of Python and [pip](https://pip.pypa.io/en/stable/), -preferably running on Debian 11 or Ubuntu Linux since we have not tested other platforms yet, +To complete this tutorial, you will use the `aleph` command from +[aleph-client](https://github.com/aleph-im/aleph-client), the `fastapi` framework to create a +simple API and the `uvicorn` server to test your program on your desktop before uploading it on +Aleph. + +First, you need a recent version of Python and [pip](https://pip.pypa.io/en/stable/), +preferably running on Debian 11 or Ubuntu Linux 20.04 since we have not tested other platforms yet, but feel free to use the platform of your choice if you have the skills to adapt our instructions to it. Some cryptographic functionalities of Aleph use curve secp256k1 and require installing [libsecp256k1](https://github.com/bitcoin-core/secp256k1). @@ -136,7 +141,16 @@ Test your progam locally using uvicorn, an ASGI server: uvicorn main:app --reload ``` -Then open http://127.0.0.1:8000 . +Then open http://127.0.0.1:8000 . The `--reload` option will automatically reload your app +when the code changes. + +> ℹ️ If you are running this on a different system than your desktop, specify the IP address of +> that system using `uvicorn main:app --reload --host 1.2.3.4`, where `1.2.3.4` is the IP address +> of the system. +> Then open your browser on http://1.2.3.4:8000 instead. + +> ℹ Installing uvicorn should add the `uvicorn` command to your shell. If it does not, use +> `python3 -m uvicorn` to run it. ## 4. Uploading @@ -176,6 +190,11 @@ Visualise on: You may get the warning `Message failed to publish on IPFS and/or P2P`. This is common and usually not an issue. +> ℹ The second URL uses a hostname dedicated to your VM. Aleph identifiers are too long to work +> for URL subdomains, so a base32 encoded version of the identifier is used instead. + +> ℹ You can make your own domain point to the VM. See the [ADVANCED](./ADVANCED.md) section. + ## 5. Running You can now run your program by opening one of the URLs above. Each URL is unique for one program. From bba73fa05b4ea24452c367dfaff6b15710e63f6a Mon Sep 17 00:00:00 2001 From: cpascariello Date: Thu, 23 Sep 2021 12:12:48 +0200 Subject: [PATCH 159/990] Doc: Fix typo in advanced tutorial --- tutorials/ADVANCED.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/ADVANCED.md b/tutorials/ADVANCED.md index 7b5ba271f..344e7e7a6 100644 --- a/tutorials/ADVANCED.md +++ b/tutorials/ADVANCED.md @@ -147,7 +147,7 @@ https://github.com/aleph-im/aleph-message/blob/main/aleph_message/tests/messages ## Custom domains -You can make your own domain point to a VM. Do achieve this, you need to create the following DNS +You can make your own domain point to a VM. To achieve this, you need to create the following DNS records: 1. A `CNAME` record to the server, for example: From fa905aa309a10489277ca6a85264afe49fa1ab93 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 6 Sep 2021 10:20:46 +0200 Subject: [PATCH 160/990] Refactor: run_code -> run_code_on_request --- vm_supervisor/__main__.py | 6 +++--- vm_supervisor/run.py | 2 +- vm_supervisor/views.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 1e293da6a..68e5ea340 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -9,7 +9,7 @@ from aiohttp.web import Response -from .run import run_code +from .run import run_code_on_request from .models import VmHash from . import supervisor from .conf import settings @@ -139,7 +139,7 @@ class FakeRequest: "/cache/keys", ): fake_request.match_info["suffix"] = path - response: Response = await run_code( + response: Response = await run_code_on_request( vm_hash=ref, path=path, request=fake_request ) assert response.status == 200 @@ -150,7 +150,7 @@ class FakeRequest: for run in range(runs): t0 = time.time() fake_request.match_info["suffix"] = path - response: Response = await run_code( + response: Response = await run_code_on_request( vm_hash=ref, path=path, request=fake_request ) assert response.status == 200 diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py index c276b6ef8..0ae3469d4 100644 --- a/vm_supervisor/run.py +++ b/vm_supervisor/run.py @@ -31,7 +31,7 @@ async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: } -async def run_code(vm_hash: VmHash, path: str, request: web.Request) -> web.Response: +async def run_code_on_request(vm_hash: VmHash, path: str, request: web.Request) -> web.Response: """ Execute the code corresponding to the 'code id' in the path. """ diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py index ef19bc501..7f57f3c75 100644 --- a/vm_supervisor/views.py +++ b/vm_supervisor/views.py @@ -8,7 +8,7 @@ from .conf import settings from .models import VmHash -from .run import run_code, pool +from .run import run_code_on_request, pool from .utils import b32_to_b16, get_ref_from_dns, dumps_for_json logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ def run_code_from_path(request: web.Request) -> Awaitable[web.Response]: path = path if path.startswith("/") else f"/{path}" message_ref: VmHash = request.match_info["ref"] - return run_code(message_ref, path, request) + return run_code_on_request(message_ref, path, request) async def run_code_from_hostname(request: web.Request) -> web.Response: @@ -56,7 +56,7 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: except aiodns.error.DNSError: raise HTTPNotFound(reason="Invalid message reference") - return await run_code(message_ref, path, request) + return await run_code_on_request(message_ref, path, request) def authenticate_request(request: web.Request): From f3e19c277afa5e12120c3141afa953ca29d0c87e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 24 Sep 2021 11:22:30 +0200 Subject: [PATCH 161/990] Fix: Submodules did not work in zip entrypoints --- runtimes/aleph-alpine-3.13-python/init1.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 01df2d86b..3337ceffe 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -164,6 +164,8 @@ def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApp module_name, app_name = entrypoint.split(":", 1) logger.debug("import module") module = __import__(module_name) + for level in module_name.split('.')[1:]: + module = getattr(module, level) app: ASGIApplication = getattr(module, app_name) elif encoding == Encoding.plain: # Execute the code and extract the entrypoint From 2af3ac5265ec4f4c6570263a5d518e65516cf35f Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 24 Sep 2021 11:24:16 +0200 Subject: [PATCH 162/990] Fix: Git ignore rootfs and squashfs archives --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index 72ede1e30..64b7aba61 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,9 @@ **/*.pyc **/__pycache__ +**/rootfs/ **/*.sqlite3 +**/*.squashfs **/*.bin **/*.ext4 **/*.zip From c81647a831afc5579118089108975a196bad6c39 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 24 Sep 2021 11:25:46 +0200 Subject: [PATCH 163/990] Fix: Exception was unclear when no message was found --- vm_connector/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vm_connector/main.py b/vm_connector/main.py index c5c303c58..a160d4076 100644 --- a/vm_connector/main.py +++ b/vm_connector/main.py @@ -6,8 +6,7 @@ from aleph_client.asynchronous import create_post from aleph_client.chains.common import get_fallback_private_key from aleph_client.chains.ethereum import ETHAccount -from fastapi import FastAPI -from fastapi import Request +from fastapi import FastAPI, Request, HTTPException from fastapi.responses import StreamingResponse, Response from pydantic import BaseModel @@ -157,6 +156,8 @@ async def download_runtime( @app.get("/compute/latest_amend/{item_hash}") async def compute_latest_amend(item_hash: str) -> str: msg = await get_message(hash_=item_hash) + if not msg: + raise HTTPException(status_code=404, detail="Hash not found") sender = msg["sender"] latest_amend = await get_latest_message_amend(ref=item_hash, sender=sender) if latest_amend: From 6fb071f1065d54493e32bb818807e08ad4bba548 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 27 Sep 2021 11:08:31 +0200 Subject: [PATCH 164/990] Feature: Run Aleph VMs in reaction to Aleph messages --- .gitignore | 5 ++ examples/example_fastapi_2/main.py | 21 ++++- examples/message_from_aleph.json | 15 +++- runtimes/aleph-alpine-3.13-python/init1.py | 20 ++++- .../create_disk_image.sh | 4 +- vm_connector/main.py | 1 + vm_supervisor/__main__.py | 10 ++- vm_supervisor/reactor.py | 73 +++++++++++++++++ vm_supervisor/run.py | 82 +++++++++++++++++++ vm_supervisor/tasks.py | 26 +++++- 10 files changed, 245 insertions(+), 12 deletions(-) create mode 100644 vm_supervisor/reactor.py diff --git a/.gitignore b/.gitignore index 0c1b40102..98069599f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,8 @@ __pycache__ /pydantic/ node_modules *.squashfs +/examples/example_http_rust/target/ +/examples/example_django/static/admin/ +/runtimes/aleph-debian-11-python/rootfs/ +/packaging/aleph-vm/opt/ +/packaging/target/ diff --git a/examples/example_fastapi_2/main.py b/examples/example_fastapi_2/main.py index 8b22ce98e..cde8a87f3 100644 --- a/examples/example_fastapi_2/main.py +++ b/examples/example_fastapi_2/main.py @@ -14,12 +14,14 @@ from aleph_client.asynchronous import get_messages, create_post from aleph_client.chains.remote import RemoteAccount from aleph_client.vm.cache import VmCache +from aleph_client.vm.app import AlephApp logger.debug("import fastapi") from fastapi import FastAPI logger.debug("imports done") -app = FastAPI() +http_app = FastAPI() +app = AlephApp(http_app=http_app) cache = VmCache() @@ -129,3 +131,20 @@ class Data(BaseModel): @app.post("/post") async def receive_post(data: Data): return str(data) + + +filters = [{ + # "sender": "0xB31B787AdA86c6067701d4C0A250c89C7f1f29A5", + "channel": "TEST" +}], + +@app.event(filters=filters) +async def aleph_event(event): + print("aleph_event", event) + async with aiohttp.ClientSession(connector=aiohttp.TCPConnector()) as session: + async with session.get("https://api2.aleph.im/api/v0/info/public.json") as resp: + print('RESP', resp) + resp.raise_for_status() + return { + "result": "Good" + } diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index 26a4f253d..bd69117fe 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -19,7 +19,18 @@ "use_latest": false }, "on": { - "http": true + "http": true, + "message": [ + { + "sender": "0xb5F010860b0964090d5414406273E6b3A8726E96", + "channel": "TEST" + }, + { + "content": { + "ref": "4d4db19afca380fdf06ba7f916153d0f740db9de9eee23ad26ba96a90d8a2920" + } + } + ] }, "environment": { "reproducible": true, @@ -64,7 +75,7 @@ "replaces": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", "time": 1619017773.8950517 }, - "item_content": "{\"type\": \"vm-function\", \"address\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"allow_amend\": false, \"code\": {\"encoding\": \"squashfs\", \"entrypoint\": \"__init__:app\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"on\": {\"http\": true}, \"environment\": {\"reproducible\": true, \"internet\": true, \"aleph_api\": true, \"shared_cache\": false}, \"resources\": {\"vcpus\": 1, \"memory\": 128, \"seconds\": 30}, \"runtime\": {\"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false, \"comment\": \"Aleph Alpine Linux with Python 3.8\"}, \"volumes\": [{\"mount\": \"/opt/venv\", \"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false}, {\"comment\": \"Working data persisted on the VM supervisor, not available on other nodes\", \"mount\": \"/var/lib/sqlite\", \"name\": \"database\", \"persistence\": \"host\"}], \"data\": {\"encoding\": \"zip\", \"mount\": \"/data\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"export\": {\"encoding\": \"zip\", \"mount\": \"/data\"}, \"replaces\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"time\": 1619017773.8950517}", + "item_content": "{\"type\": \"vm-function\", \"address\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"allow_amend\": false, \"code\": {\"encoding\": \"squashfs\", \"entrypoint\": \"main:app\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"on\": {\"http\": true, \"message\": [{\"sender\": \"0xB31B787AdA86c6067701d4C0A250c89C7f1f29A5\", \"channel\": \"TEST\"}, {\"content\": {\"ref\": \"4d4db19afca380fdf06ba7f916153d0f740db9de9eee23ad26ba96a90d8a2920\"}}]}, \"environment\": {\"reproducible\": true, \"internet\": true, \"aleph_api\": true, \"shared_cache\": false}, \"resources\": {\"vcpus\": 1, \"memory\": 128, \"seconds\": 30}, \"runtime\": {\"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false, \"comment\": \"Aleph Alpine Linux with Python 3.8\"}, \"volumes\": [{\"mount\": \"/opt/venv\", \"ref\": \"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\", \"use_latest\": false}, {\"comment\": \"Working data persisted on the VM supervisor, not available on other nodes\", \"mount\": \"/var/lib/sqlite\", \"name\": \"database\", \"persistence\": \"host\", \"size_mib\": 5}], \"data\": {\"encoding\": \"zip\", \"mount\": \"/data\", \"ref\": \"7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003\", \"use_latest\": false}, \"export\": {\"encoding\": \"zip\", \"mount\": \"/data\"}, \"replaces\": \"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\", \"time\": 1619017773.8950517}", "item_type": "inline", "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b", "size": 749, diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 3337ceffe..253c952d3 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -227,7 +227,8 @@ async def run_python_code_http(application: ASGIApplication, scope: dict body: bytes = scope.pop('body') async def receive(): - return {'type': 'http.request', + type_ = 'http.request' if scope['type'] in ('http', 'websocket') else 'aleph.message' + return {'type': type_, 'body': body, 'more_body': False} @@ -237,11 +238,26 @@ async def send(dico): await send_queue.put(dico) # TODO: Better error handling + logger.debug("Awaiting application...") await application(scope, receive, send) - headers: Dict = await send_queue.get() + + logger.debug("Waiting for headers") + headers: Dict + if scope['type'] == 'http': + headers = await send_queue.get() + else: + headers = {} + + logger.debug("Waiting for body") body: Dict = await send_queue.get() + + logger.debug("Waiting for buffer") output = buf.getvalue() + logger.debug(f"Headers {headers}") + logger.debug(f"Body {body}") + logger.debug(f"Output {output}") + logger.debug("Getting output data") output_data: bytes if os.path.isdir('/data') and os.listdir('/data'): diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index a2f58c18b..3008d4ece 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -19,12 +19,12 @@ apt-get install -y --no-install-recommends --no-install-suggests \ python3-setuptools \ python3-pip python3-cytoolz python3-pydantic \ iproute2 unzip \ - nodejs + nodejs npm pip3 install fastapi django echo "Pip installing aleph-client" -pip3 install 'aleph-client>=0.2.7' 'coincurve==15.0.0' 'eth_account>=0.4.0' +pip3 install 'aleph-client>=0.3.2' 'coincurve==15.0.0' # Compile all Python bytecode python3 -m compileall -f /usr/local/lib/python3.9 diff --git a/vm_connector/main.py b/vm_connector/main.py index a160d4076..675904925 100644 --- a/vm_connector/main.py +++ b/vm_connector/main.py @@ -219,6 +219,7 @@ async def properties(request: Request): @app.post("/sign") async def sign_message(request: Request): """Sign a message""" + # TODO: Check private_key = get_fallback_private_key() account: ETHAccount = ETHAccount(private_key=private_key) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 68e5ea340..d2a88675c 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -9,7 +9,8 @@ from aiohttp.web import Response -from .run import run_code_on_request +from vm_supervisor.pubsub import PubSub +from .run import run_code_on_request, run_code_on_event from .models import VmHash from . import supervisor from .conf import settings @@ -78,6 +79,7 @@ def parse_args(args): "-n", "--do-not-run", dest="do_not_run", + action="store_true", default=False, ) parser.add_argument( @@ -101,7 +103,7 @@ async def benchmark(runs: int): """Measure performance by immediately running the supervisor with fake requests. """ - ref = VmHash("TEST_HASH") + ref = VmHash("cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e") class FakeRequest: headers: Dict[str, str] @@ -162,6 +164,10 @@ class FakeRequest: ) logger.info(bench) + event = None + result = await run_code_on_event(vm_hash=ref, event=event, pubsub=PubSub()) + print("Event result", result) + def main(): args = parse_args(sys.argv[1:]) diff --git a/vm_supervisor/reactor.py b/vm_supervisor/reactor.py new file mode 100644 index 000000000..0038c7cc2 --- /dev/null +++ b/vm_supervisor/reactor.py @@ -0,0 +1,73 @@ +import asyncio +import logging +from typing import List, Dict, Coroutine + +from aleph_message.models import Message, ProgramMessage +from vm_supervisor.pubsub import PubSub +from vm_supervisor.run import run_code_on_event + +logger = logging.getLogger(__name__) + + +def is_equal_or_includes(value, compare_to) -> bool: + if isinstance(value, str): + return value == compare_to + elif isinstance(value, dict): + for subkey, subvalue in value.items(): + if not hasattr(compare_to, subkey): + return False + if not is_equal_or_includes(subvalue, getattr(compare_to, subkey)): + return False + return True + else: + raise ValueError("Unsupported value") + + +def subscription_matches(subscription: Dict, message: ProgramMessage) -> bool: + if not subscription: + # Require at least one value to match + return False + for key, value in subscription.items(): + if not is_equal_or_includes(value, getattr(message, key)): + return False + return True + + +class Reactor: + + pubsub: PubSub + listeners: List[ProgramMessage] + + def __init__(self, pubsub: PubSub): + self.pubsub = pubsub + self.listeners = [] + + async def trigger(self, message: Message): + coroutines: List[Coroutine] = [] + + for listener in self.listeners: + if not listener.content.on.message: + logger.warning("Program with no subscription was registered in reactor listeners: " + f"{listener.item_hash}") + continue + + for subscription in listener.content.on.message: + if subscription_matches(subscription, message): + vm_hash = listener.item_hash + event = message.json() + # Register the listener in the list of coroutines to run asynchronously: + coroutines.append(run_code_on_event(vm_hash, event, self.pubsub)) + break + + # Call all listeners asynchronously from the event loop: + loop = asyncio.get_event_loop() + for coroutine in coroutines: + loop.create_task(coroutine) + + def register(self, message: ProgramMessage): + if message.content.on.message: + self.listeners.append(message) + else: + logger.debug( + "Program with no subscription cannot be registered in reactor listeners: " + f"{message.item_hash}") diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py index 0ae3469d4..8278b31e3 100644 --- a/vm_supervisor/run.py +++ b/vm_supervisor/run.py @@ -11,6 +11,7 @@ from .messages import load_updated_message from .models import VmHash, VmExecution from .pool import VmPool +from .pubsub import PubSub from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError logger = logging.getLogger(__name__) @@ -31,6 +32,13 @@ async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]: } +async def build_event_scope(event) -> Dict[str, Any]: + return { + "type": "aleph.message", + "body": event, + } + + async def run_code_on_request(vm_hash: VmHash, path: str, request: web.Request) -> web.Response: """ Execute the code corresponding to the 'code id' in the path. @@ -105,3 +113,77 @@ async def run_code_on_request(vm_hash: VmHash, path: str, request: web.Request) execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT) else: await execution.stop() + + +async def run_code_on_event(vm_hash: VmHash, event, pubsub: PubSub): + """ + Execute code in response to an event. + """ + + try: + execution: VmExecution = await pool.get_running_vm(vm_hash=vm_hash) + except Exception as error: + logger.exception(error) + raise + + if not execution: + message, original_message = await load_updated_message(vm_hash) + pool.message_cache[vm_hash] = message + + try: + execution = await pool.create_a_vm( + vm_hash=vm_hash, + program=message.content, + original=original_message.content, + ) + except ResourceDownloadError as error: + logger.exception(error) + pool.forget_vm(vm_hash=vm_hash) + raise HTTPBadRequest(reason="Code, runtime or data not available") + except VmSetupError as error: + logger.exception(error) + pool.forget_vm(vm_hash=vm_hash) + raise HTTPInternalServerError(reason="Error during program initialisation") + except MicroVMFailedInit as error: + logger.exception(error) + pool.forget_vm(vm_hash=vm_hash) + raise HTTPInternalServerError(reason="Error during runtime initialisation") + + logger.debug(f"Using vm={execution.vm.vm_id}") + + scope: Dict = await build_event_scope(event) + + try: + await execution.becomes_ready() + result_raw: bytes = await execution.run_code(scope=scope) + except UnpackValueError as error: + logger.exception(error) + return web.Response(status=502, reason="Invalid response from VM") + + try: + result = msgpack.loads(result_raw, raw=False) + + logger.debug(f"Result from VM: <<<\n\n{str(result)[:1000]}\n\n>>>") + + if "traceback" in result: + logger.warning(result["traceback"]) + return web.Response( + status=500, + reason="Error in VM execution", + body=result["traceback"], + content_type="text/plain", + ) + + logger.info(f"Result: {result['body']}") + return result['body'] + + except UnpackValueError as error: + logger.exception(error) + return web.Response(status=502, reason="Invalid response from VM") + finally: + if settings.REUSE_TIMEOUT > 0: + if settings.WATCH_FOR_UPDATES: + execution.start_watching_for_updates(pubsub=pubsub) + execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT) + else: + await execution.stop() diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py index aeff9c85d..ca50d2e11 100644 --- a/vm_supervisor/tasks.py +++ b/vm_supervisor/tasks.py @@ -11,9 +11,11 @@ from yarl import URL from aleph_message import Message -from aleph_message.models import BaseMessage +from aleph_message.models import BaseMessage, ProgramMessage from .conf import settings +from .messages import load_updated_message from .pubsub import PubSub +from .reactor import Reactor logger = logging.getLogger(__name__) @@ -39,7 +41,7 @@ async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]: break -async def watch_for_messages(dispatcher: PubSub): +async def watch_for_messages(dispatcher: PubSub, reactor: Reactor): """Watch for new Aleph messages""" logger.debug("watch_for_messages()") url = URL(f"{settings.API_SERVER}/api/ws0/messages").with_query( @@ -48,6 +50,8 @@ async def watch_for_messages(dispatcher: PubSub): async for message in subscribe_via_ws(url): logger.info(f"Websocket received message: {message.item_hash}") + + # Dispatch update to running VMs ref = ( message.content.ref if hasattr(message.content, "ref") @@ -55,12 +59,28 @@ async def watch_for_messages(dispatcher: PubSub): ) await dispatcher.publish(key=ref, value=message) + # Register new VM to run on future messages: + if isinstance(message, ProgramMessage): + if message.content.on.message: + reactor.register(message) + await reactor.trigger(message=message) + async def start_watch_for_messages_task(app: web.Application): logger.debug("start_watch_for_messages_task()") pubsub = PubSub() + reactor = Reactor(pubsub) + + # Register an hardcoded initial program + # TODO: Register all programs with subscriptions + sample_message, _ = await load_updated_message( + ref="cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e") + assert sample_message.content.on.message, sample_message + reactor.register(sample_message) + app["pubsub"] = pubsub - app["messages_listener"] = asyncio.create_task(watch_for_messages(pubsub)) + app["reactor"] = reactor + app["messages_listener"] = asyncio.create_task(watch_for_messages(pubsub, reactor)) async def stop_watch_for_messages_task(app: web.Application): From ae2b6c3942aa563f632a8d46d9f9c054e5d06aa8 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 27 Sep 2021 18:51:14 +0200 Subject: [PATCH 165/990] Fix: Network interface was not read from config --- vm_supervisor/vm/firecracker_microvm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 7a976a96c..a821472a7 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -289,7 +289,8 @@ async def setup(self): network_interfaces=[ NetworkInterface( iface_id="eth0", - host_dev_name=await fvm.create_network_interface(interface="eth0"), + host_dev_name=await fvm.create_network_interface( + interface=settings.NETWORK_INTERFACE), ) ] if self.enable_networking From 55b4abe0600fd8a2903914e4b4a59fbbb0c4888d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 27 Sep 2021 18:51:14 +0200 Subject: [PATCH 166/990] Fix: SSH failed without a directory; Send SSH logs to stdout --- runtimes/aleph-alpine-3.13-python/init0.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/init0.sh b/runtimes/aleph-alpine-3.13-python/init0.sh index a114e8a3b..914debaa8 100644 --- a/runtimes/aleph-alpine-3.13-python/init0.sh +++ b/runtimes/aleph-alpine-3.13-python/init0.sh @@ -35,7 +35,8 @@ lsblk #cat /proc/sys/kernel/random/entropy_avail # TODO: Move in init1 -/usr/sbin/sshd -E /var/log/sshd & +mkdir -p /run/sshd +/usr/sbin/sshd & log "SSH UP" log "Setup socat" From 4385d00f4e1baa29ae93b2b19551eaebfa4a56f9 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 11 Oct 2021 17:21:53 +0200 Subject: [PATCH 167/990] Doc: Document how to upload Python packages in a VM --- .github/workflows/test-build-examples.yml | 44 ++++++++++++++ examples/Makefile | 5 ++ examples/example_pip/main.py | 12 ++++ examples/example_pip/requirements.txt | 1 + runtimes/aleph-alpine-3.13-python/init1.py | 3 + tutorials/README.md | 3 + tutorials/REQUIREMENTS.md | 67 ++++++++++++++++++++++ 7 files changed, 135 insertions(+) create mode 100644 .github/workflows/test-build-examples.yml create mode 100644 examples/example_pip/main.py create mode 100644 examples/example_pip/requirements.txt create mode 100644 tutorials/REQUIREMENTS.md diff --git a/.github/workflows/test-build-examples.yml b/.github/workflows/test-build-examples.yml new file mode 100644 index 000000000..974b56b2c --- /dev/null +++ b/.github/workflows/test-build-examples.yml @@ -0,0 +1,44 @@ + + +name: "Build Examples" +on: + push + +jobs: + build_pip: + name: "Build with Pip requirements" + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - run: | + sudo apt-get -y update + sudo apt-get -y upgrade + sudo apt-get -y install python3-pip python3-venv squashfs-tools build-essential + + sudo mkdir /opt/packages + sudo chown $(whoami) /opt/packages + + - run: | + pip3 install aleph-client + + - run: | + ls -la + + - run: | + ls + pwd + pip3 install -t /opt/packages -r ./examples/example_pip/requirements.txt + mksquashfs /opt/packages packages.squashfs + +# - run: | +# ipfs add packages.squashfs + +# TODO: There is currently no easy way pass the item_hash from a pin to a new program. +# - run: | +# aleph pin QmQr3dEd6LiFq6JmUJYPLrffy45RGFhPWsxWmzo9zZb7Sy +# +# - run: | +# aleph program ./examples/example_pip main:app diff --git a/examples/Makefile b/examples/Makefile index 791ebbb7a..d477eaed4 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -9,3 +9,8 @@ example_fastapi_2.zip: data.tgz: tar -cvzf data.tgz data + +example_pip.squashfs: + rm -fr /opt/python + pip3 install -t /opt/requirements -r example_pip/requirements.txt + mksquashfs /opt/requirements requirements.squashfs diff --git a/examples/example_pip/main.py b/examples/example_pip/main.py new file mode 100644 index 000000000..148941645 --- /dev/null +++ b/examples/example_pip/main.py @@ -0,0 +1,12 @@ +import pandas as pandas +from fastapi import FastAPI, Response + +app = FastAPI() + + +@app.get("/") +async def root(): + data = range(10) + df = pandas.DataFrame(data) + return Response(content=df.to_html(), + media_type='text/html') diff --git a/examples/example_pip/requirements.txt b/examples/example_pip/requirements.txt new file mode 100644 index 000000000..fb6c7ed7e --- /dev/null +++ b/examples/example_pip/requirements.txt @@ -0,0 +1 @@ +pandas diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 253c952d3..d9dc3e23f 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -145,6 +145,9 @@ def setup_volumes(volumes: List[Volume]): def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplication: + # Allow importing packages from /opt/packages + sys.path.append("/opt/packages") + logger.debug("Extracting code") if encoding == Encoding.squashfs: sys.path.append("/opt/code") diff --git a/tutorials/README.md b/tutorials/README.md index 7f911d9d5..5f9056c30 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -225,6 +225,9 @@ program to work, else they will be ignored. ## Next steps +Check out the [Requirements](./REQUIREMENTS.md) page to add additional Python packages to your +program from the Python Package Index ([PyPI](https://www.pypi.org)). + Check out the [Writing a non-Python program](./SERVER.md) page to run a program written in another language than Python. Check out the [Advanced usage](./ADVANCED.md) page for more options and capabilities. diff --git a/tutorials/REQUIREMENTS.md b/tutorials/REQUIREMENTS.md new file mode 100644 index 000000000..ea74ad9d4 --- /dev/null +++ b/tutorials/REQUIREMENTS.md @@ -0,0 +1,67 @@ +# Tutorial: Adding Python libraries to an Aleph VM + +## 0. Setup your environment +```shell +sudo apt install python3-pip python3-venv squashfs-tools +``` + +```shell +pip3 install aleph-client +``` + +### 1. Install the packages in a directory + +```shell +pip install -t /opt/packages -r requirements.txt +``` + +```shell +mksquashfs /opt/packages packages.squashfs +``` + +## 2. Upload the packages + +### 2.a. Without IPFS (small size) + +```shell +aleph upload packages.squashfs +``` + +### 2.b. With IPFS +```shell +/opt/go-ipfs/ipfs daemon +``` + +```shell +ipfs add venv.squashfs +``` +| added QmWWX6BaaRkRSr2iNdwH5e29ACPg2nCHHXTRTfuBmVm3Ga venv.squashfs + +```shell +aleph pin QmWWX6BaaRkRSr2iNdwH5e29ACPg2nCHHXTRTfuBmVm3Ga +``` + +## 3. Create your program + +```shell +aleph program ./my-program main:app +``` + +Press Enter at the following prompt to use the default runtime: +``` +Ref of runtime ? [bd79839bf96e595a06da5ac0b6ba51dea6f7e2591bb913deccded04d831d29f4] +``` + +Press `Y` to add extra volumes to your program: +``` +Add volume ? [y/N] Y +Description: Python Packages +Mount: /opt/packages +Ref: 61f43ab261060ff94838dc94313a70cdb939a5fc6c99924b96d55dcc2c108d03 +Use latest version ? [Y/n] +``` + +Finally, press Enter to skip adding more volumes. +```shell +Add volume ? [y/N] +``` From 40c5925014677192a373b748e39d7fd2d439c606 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 4 Nov 2021 12:39:32 +0100 Subject: [PATCH 168/990] Fix: Default value missing from optional fields --- runtimes/aleph-alpine-3.13-python/init1.py | 10 +++++----- vm_supervisor/vm/firecracker_microvm.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index d9dc3e23f..b73fd30c3 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -17,7 +17,7 @@ import sys import traceback from contextlib import redirect_stdout -from dataclasses import dataclass +from dataclasses import dataclass, field from io import StringIO from os import system from shutil import make_archive @@ -51,16 +51,16 @@ class Volume: @dataclass class ConfigurationPayload: - ip: Optional[str] - route: Optional[str] - dns_servers: List[str] code: bytes encoding: Encoding entrypoint: str input_data: bytes interface: Interface vm_hash: str - volumes: List[Volume] + ip: Optional[str] = None + route: Optional[str] = None + dns_servers: List[str] = field(default_factory=list) + volumes: List[Volume] = field(default_factory=list) @dataclass diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index a821472a7..8a13ee53f 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -78,16 +78,16 @@ class HostVolume: @dataclass class ConfigurationPayload: - ip: Optional[str] - route: Optional[str] - dns_servers: List[str] code: bytes encoding: Encoding entrypoint: str input_data: bytes interface: Interface vm_hash: str - volumes: List[Volume] + ip: Optional[str] = None + route: Optional[str] = None + dns_servers: List[str] = field(default_factory=list) + volumes: List[Volume] = field(default_factory=list) def as_msgpack(self) -> bytes: return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True) From dc029ef5c57244f1d9fd7f09e64c6c2fd7b8eabe Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 4 Nov 2021 12:45:36 +0100 Subject: [PATCH 169/990] Feature: Add support for environment variables in programs --- docker/vm_supervisor.dockerfile | 2 +- examples/example_fastapi_2/main.py | 8 ++++++++ examples/message_from_aleph.json | 3 +++ examples/volumes/Dockerfile | 2 +- runtimes/aleph-alpine-3.13-python/init1.py | 9 +++++++++ vm_supervisor/README.md | 2 +- vm_supervisor/__main__.py | 1 + vm_supervisor/vm/firecracker_microvm.py | 4 +++- 8 files changed, 27 insertions(+), 4 deletions(-) diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile index 0ec63d506..8ab3f016b 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor.dockerfile @@ -18,7 +18,7 @@ RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/downl RUN ln /opt/firecracker/firecracker-v* /opt/firecracker/firecracker RUN ln /opt/firecracker/jailer-v* /opt/firecracker/jailer -RUN pip3 install typing-extensions 'aleph-message>=0.1.12' pydantic +RUN pip3 install typing-extensions 'aleph-message>=0.1.18' RUN mkdir /srv/jailer diff --git a/examples/example_fastapi_2/main.py b/examples/example_fastapi_2/main.py index cde8a87f3..591bb5b2f 100644 --- a/examples/example_fastapi_2/main.py +++ b/examples/example_fastapi_2/main.py @@ -3,6 +3,8 @@ import os from datetime import datetime from os import listdir +from typing import Dict + from pydantic import BaseModel logger = logging.getLogger(__name__) @@ -41,6 +43,12 @@ async def index(): } +@app.get("/environ") +async def environ() -> Dict[str, str]: + """List environment variables""" + return dict(os.environ) + + @app.get("/messages") async def read_aleph_messages(): """Read data from Aleph using the Aleph Client library.""" diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index bd69117fe..36b12fa67 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -18,6 +18,9 @@ "ref": "7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003", "use_latest": false }, + "variables": { + "VM_CUSTOM_NUMBER": "32" + }, "on": { "http": true, "message": [ diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile index b23ab639a..af522dda5 100644 --- a/examples/volumes/Dockerfile +++ b/examples/volumes/Dockerfile @@ -6,6 +6,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \ && rm -rf /var/lib/apt/lists/* RUN python3 -m venv /opt/venv -RUN /opt/venv/bin/pip install 'aleph-message>=0.1.12' +RUN /opt/venv/bin/pip install 'aleph-message>=0.1.18' CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index b73fd30c3..918c1b8ab 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -61,6 +61,7 @@ class ConfigurationPayload: route: Optional[str] = None dns_servers: List[str] = field(default_factory=list) volumes: List[Volume] = field(default_factory=list) + variables: Optional[Dict[str, str]] = None @dataclass @@ -92,6 +93,13 @@ def setup_hostname(hostname: str): system(f"hostname {hostname}") +def setup_variables(variables: Optional[Dict[str, str]]): + if variables is None: + return + for key, value in variables.items(): + os.environ[key] = value + + def setup_network(ip: Optional[str], route: Optional[str], dns_servers: Optional[List[str]] = None): """Setup the system with info from the host.""" @@ -399,6 +407,7 @@ def receive_config(client) -> ConfigurationPayload: def setup_system(config: ConfigurationPayload): setup_hostname(config.vm_hash) + setup_variables(config.variables) setup_volumes(config.volumes) setup_network(config.ip, config.route, config.dns_servers) setup_input_data(config.input_data) diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index 3f0173786..1ced9bee8 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -88,7 +88,7 @@ is used to parse and validate Aleph messages. ```shell apt install -y --no-install-recommends --no-install-suggests python3-pip pip3 install pydantic[dotenv] -pip3 install aleph-message>=0.1.12 +pip3 install --update aleph-message ``` ### 2.f. Create the jailer working directory: diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index d2a88675c..e96ddb58d 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -133,6 +133,7 @@ class FakeRequest: settings.REUSE_TIMEOUT = 0.1 for path in ( "/", + "/environ", "/messages", "/internet", "/post_a_message", diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 8a13ee53f..50f21a313 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -1,7 +1,7 @@ import asyncio import dataclasses import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum from multiprocessing import Process, set_start_method from os import system @@ -88,6 +88,7 @@ class ConfigurationPayload: route: Optional[str] = None dns_servers: List[str] = field(default_factory=list) volumes: List[Volume] = field(default_factory=list) + variables: Optional[Dict[str, str]] = None def as_msgpack(self) -> bytes: return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True) @@ -365,6 +366,7 @@ async def configure(self): interface=interface, vm_hash=self.vm_hash, volumes=volumes, + variables=self.resources.message_content.variables, ) payload = config.as_msgpack() length = f"{len(payload)}\n".encode() From da90eb595bb3dda54c149a16c12d925fc91c18af Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 4 Nov 2021 12:48:10 +0100 Subject: [PATCH 170/990] Fix: Download kernel in supervisor Dockerfile --- docker/vm_supervisor.dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile index 8ab3f016b..a7dfe0dc0 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor.dockerfile @@ -13,6 +13,7 @@ RUN useradd jailman RUN mkdir /opt/firecracker RUN chown $(whoami) /opt/firecracker RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v0.24.2/firecracker-v0.24.2-x86_64.tgz | tar -xz --directory /opt/firecracker +RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/vmlinux.bin # Link binaries on version-agnostic paths: RUN ln /opt/firecracker/firecracker-v* /opt/firecracker/firecracker From f39bed03c1eeef46ca4f58bbf14cbb0175d8906e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 4 Nov 2021 12:48:37 +0100 Subject: [PATCH 171/990] Fix: Use enum instead of string to match type check --- examples/example_fastapi_2/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/example_fastapi_2/main.py b/examples/example_fastapi_2/main.py index 591bb5b2f..fd0c23a64 100644 --- a/examples/example_fastapi_2/main.py +++ b/examples/example_fastapi_2/main.py @@ -13,6 +13,7 @@ import aiohttp logger.debug("import aleph_client") +from aleph_client.types import StorageEnum from aleph_client.asynchronous import get_messages, create_post from aleph_client.chains.remote import RemoteAccount from aleph_client.vm.cache import VmCache @@ -87,7 +88,7 @@ async def post_a_message(): ref=None, channel="TEST", inline=True, - storage_engine="storage", + storage_engine=StorageEnum.storage, ) return { "response": response, From bfd3e59788bff40cdc06d4196b54dedf751d0e97 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 4 Nov 2021 12:48:54 +0100 Subject: [PATCH 172/990] Fix: Typo in example created a tuple --- examples/example_fastapi_2/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example_fastapi_2/main.py b/examples/example_fastapi_2/main.py index fd0c23a64..dfb4477e2 100644 --- a/examples/example_fastapi_2/main.py +++ b/examples/example_fastapi_2/main.py @@ -145,7 +145,7 @@ async def receive_post(data: Data): filters = [{ # "sender": "0xB31B787AdA86c6067701d4C0A250c89C7f1f29A5", "channel": "TEST" -}], +}] @app.event(filters=filters) async def aleph_event(event): From 9c2f0176e05c71a94f7eef081617087ce2d1cf13 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 4 Nov 2021 12:49:10 +0100 Subject: [PATCH 173/990] Fix: Errors within chroot did not raise an error --- runtimes/aleph-debian-11-python/create_disk_image.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index 3008d4ece..454ba746d 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -10,6 +10,9 @@ mkdir ./rootfs debootstrap --variant=minbase bullseye ./rootfs http://deb.debian.org/debian/ chroot ./rootfs /bin/sh < Date: Thu, 4 Nov 2021 12:49:48 +0100 Subject: [PATCH 174/990] Fix: Some runtime dependencies now require a C compiler --- runtimes/aleph-debian-11-python/create_disk_image.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index 454ba746d..b6b9a1fe2 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -22,12 +22,13 @@ apt-get install -y --no-install-recommends --no-install-suggests \ python3-setuptools \ python3-pip python3-cytoolz python3-pydantic \ iproute2 unzip \ - nodejs npm + nodejs npm \ + build-essential python3-dev pip3 install fastapi django echo "Pip installing aleph-client" -pip3 install 'aleph-client>=0.3.2' 'coincurve==15.0.0' +pip3 install 'aleph-client>=0.4.4' 'coincurve==15.0.0' # Compile all Python bytecode python3 -m compileall -f /usr/local/lib/python3.9 From 7cda855ff7389c213e89374054ca4bd2d713196d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 4 Nov 2021 12:49:48 +0100 Subject: [PATCH 175/990] Fix: Subscription is now a model --- vm_supervisor/reactor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/reactor.py b/vm_supervisor/reactor.py index 0038c7cc2..814c2ee60 100644 --- a/vm_supervisor/reactor.py +++ b/vm_supervisor/reactor.py @@ -2,6 +2,8 @@ import logging from typing import List, Dict, Coroutine +from aleph_message.models.program import Subscription + from aleph_message.models import Message, ProgramMessage from vm_supervisor.pubsub import PubSub from vm_supervisor.run import run_code_on_event @@ -23,11 +25,11 @@ def is_equal_or_includes(value, compare_to) -> bool: raise ValueError("Unsupported value") -def subscription_matches(subscription: Dict, message: ProgramMessage) -> bool: +def subscription_matches(subscription: Subscription, message: ProgramMessage) -> bool: if not subscription: # Require at least one value to match return False - for key, value in subscription.items(): + for key, value in subscription.dict().items(): if not is_equal_or_includes(value, getattr(message, key)): return False return True From 5c45d363458bbdd8a8d4035ea2ea9b0ccfd2b6b6 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 9 Nov 2021 17:00:15 +0100 Subject: [PATCH 176/990] Update the tutorial, add macOS instructions using Vagrant --- tutorials/README.md | 11 +++++++++ tutorials/REQUIREMENTS.md | 47 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/tutorials/README.md b/tutorials/README.md index 5f9056c30..a40614ccb 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -141,6 +141,17 @@ Test your progam locally using uvicorn, an ASGI server: uvicorn main:app --reload ``` +If you are on Mac OS test your program locally by starting to run +```shell +vagrant ssh +``` + +Then go to your working repository and launch: + +```shell +python3 -m uvicorn main:app --reload --host=0.0.0.0 +``` + Then open http://127.0.0.1:8000 . The `--reload` option will automatically reload your app when the code changes. diff --git a/tutorials/REQUIREMENTS.md b/tutorials/REQUIREMENTS.md index ea74ad9d4..c31b009d2 100644 --- a/tutorials/REQUIREMENTS.md +++ b/tutorials/REQUIREMENTS.md @@ -1,6 +1,6 @@ # Tutorial: Adding Python libraries to an Aleph VM -## 0. Setup your environment +## 0.a Setup your environment (Debian/Ubuntu Linux) ```shell sudo apt install python3-pip python3-venv squashfs-tools ``` @@ -9,6 +9,50 @@ sudo apt install python3-pip python3-venv squashfs-tools pip3 install aleph-client ``` +## 0.b Quick install (macOS using Vagrant) + +For starting to run aleph-vm on mac you have to initialize a VM. + +### Install VirtualBox +You will need VirtualBox, a free and open-source hosted hypervisor (or virtual machine manager) for the next step. + +You can download and install it here . + +### Install Vagrant +Vagrant is an open-source software product for building and maintaining portable virtual software development environments based on VirtualBox. + +Run following command for installing it (before make sure [homebrew](brew.sh) is installed on your mac). + +```shell +brew install vagrant +``` + +Once Vagrant is installed, go to your working repository and initialize vagrant + +```shell +vagrant init boxomatic/debian-11 +``` + +A `Vagrantfile` (in Ruby) will be created, you can consult it if you wish. + +Now in order to instantiate a new virtual machine, run the following command: + +```shell +vagrant up +``` + +If this does not work, check out you System Preferences > Security and Privacy and allow the "System software from developer" in the bottom of the window. + +Once the command is down, your virtual machine will be booted and ready! + +### Set Vagrantfile configuration + +Open the vagranfile and add following `config.vm.box`` + +```shell +config.vm.network "forwarded_port", guest:8000, host:8000 +``` + ### 1. Install the packages in a directory ```shell @@ -19,6 +63,7 @@ pip install -t /opt/packages -r requirements.txt mksquashfs /opt/packages packages.squashfs ``` + ## 2. Upload the packages ### 2.a. Without IPFS (small size) From 66b7ac8738e9b35257e773f154f3cfa40891635d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 15 Nov 2021 15:18:05 +0100 Subject: [PATCH 177/990] Fix: HTTP partial transfer caused issues --- vm_supervisor/run.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py index 8278b31e3..10564840b 100644 --- a/vm_supervisor/run.py +++ b/vm_supervisor/run.py @@ -97,6 +97,10 @@ async def run_code_on_request(vm_hash: VmHash, path: str, request: web.Request) headers = { key.decode(): value.decode() for key, value in result["headers"]["headers"] } + headers["Content-Length"] = str(len(result["body"]["body"])) + for header in ["Content-Encoding", "Transfer-Encoding", "Vary"]: + if header in headers: + del headers[header] return web.Response( status=result["headers"]["status"], From d7f7be510ce9478444a23d64b75cf3cd75194557 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 15 Nov 2021 15:34:13 +0100 Subject: [PATCH 178/990] Fix: Pipe to set root password did not work --- runtimes/aleph-debian-11-python/create_disk_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index b6b9a1fe2..3ae16c34d 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -33,7 +33,7 @@ pip3 install 'aleph-client>=0.4.4' 'coincurve==15.0.0' # Compile all Python bytecode python3 -m compileall -f /usr/local/lib/python3.9 -#echo -e "toor\ntoor" | passwd root +echo "root:toor" | chpasswd mkdir -p /overlay From 26d7643515d74877f8f02bd3efcda239718b7522 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 15 Nov 2021 15:34:54 +0100 Subject: [PATCH 179/990] Fix: Example index was missing /environ endpoint --- examples/example_fastapi_2/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example_fastapi_2/main.py b/examples/example_fastapi_2/main.py index dfb4477e2..91611a972 100644 --- a/examples/example_fastapi_2/main.py +++ b/examples/example_fastapi_2/main.py @@ -36,7 +36,7 @@ async def index(): opt_venv = [] return { "Example": "example_fastapi_2", - "endpoints": ["/messages", "/internet", "/post_a_message", + "endpoints": ["/environ", "/messages", "/internet", "/post_a_message", "/state/increment", "/wait-for/{delay}"], "files_in_volumes": { "/opt/venv": opt_venv, From 0c4a03d1cde2d9625a634ad026a3aac79226e039 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 16:05:55 +0100 Subject: [PATCH 180/990] Fix: Using coroutine() is deprecated Solution: Replace it with an `async def`. --- vm_supervisor/__main__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index e96ddb58d..41446f38b 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -119,8 +119,9 @@ class FakeRequest: (name.encode(), value.encode()) for name, value in fake_request.headers.items() ] - # noinspection PyDeprecation - fake_request.read = coroutine(lambda: b"") + async def fake_read() -> bytes: + return b"" + fake_request.read = fake_read logger.info("--- Start benchmark ---") From f860ffcb133b8727795ab9b562b30b69f5df5610 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 16:09:14 +0100 Subject: [PATCH 181/990] Internal: os.system did not raise an error on failure --- vm_supervisor/vm/firecracker_microvm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index 50f21a313..dcf42df6f 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -1,10 +1,10 @@ import asyncio import dataclasses import logging +import subprocess from dataclasses import dataclass, field from enum import Enum from multiprocessing import Process, set_start_method -from os import system from os.path import isfile, exists from typing import Optional, Dict, List @@ -390,7 +390,7 @@ async def start_guest_api(self): self.guest_api_process.start() while not exists(vsock_path): await asyncio.sleep(0.01) - system(f"chown jailman:jailman {vsock_path}") + subprocess.run(f"chown jailman:jailman {vsock_path}", shell=True, check=True) logger.debug(f"started guest API for {self.vm_id}") async def stop_guest_api(self): From 29d54fde958b4fbfd41b9a39a476b283604e96a0 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 16:10:36 +0100 Subject: [PATCH 182/990] Logging: setfacl can fail without being problematic When used in a Podman container for example --- firecracker/microvm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 4cbfef826..8415b9ab5 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -50,11 +50,11 @@ async def setfacl(): if proc.returncode == 0: return - logger.error(f"[{cmd!r} exited with {[proc.returncode]}]") + logger.warning(f"[{cmd!r} exited with {[proc.returncode]}]") if stdout: - logger.error(f"[stdout]\n{stdout.decode()}") + logger.warning(f"[stdout]\n{stdout.decode()}") if stderr: - logger.error(f"[stderr]\n{stderr.decode()}") + logger.warning(f"[stderr]\n{stderr.decode()}") class MicroVM: From 2725ce673d3525521525ff93bbe7afd887441ec3 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 16:11:05 +0100 Subject: [PATCH 183/990] Fix: Expiration coroutine was not cancelled on VM stop --- vm_supervisor/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index 420a4a11a..0991805d3 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -139,6 +139,7 @@ async def stop(self): self.times.stopping_at = datetime.now() await self.vm.teardown() self.times.stopped_at = datetime.now() + self.cancel_expiration() def start_watching_for_updates(self, pubsub: PubSub): pool = asyncio.get_running_loop() From 963ef7eeae1dea5f09f94483b88c26e160fd6450 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 16:11:15 +0100 Subject: [PATCH 184/990] Cleanup: Add type annotation --- vm_supervisor/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index 0991805d3..521221bff 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -168,7 +168,7 @@ async def all_runs_complete(self): logger.debug("Stop: waiting for runs to complete...") await self.runs_done_event.wait() - async def run_code(self, scope: dict = None): + async def run_code(self, scope: dict = None) -> bytes: self.concurrent_runs += 1 self.runs_done_event.clear() try: From 276555a00a9ea932933461cceab37689842858e9 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 16:11:37 +0100 Subject: [PATCH 185/990] Internal: There was no convenient way to stop all VMs in a pool --- vm_supervisor/pool.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py index f380d3f85..032052033 100644 --- a/vm_supervisor/pool.py +++ b/vm_supervisor/pool.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, Optional +from typing import Dict, Optional, List from aleph_message.models import ProgramContent, ProgramMessage from .conf import settings @@ -47,3 +47,13 @@ async def get_running_vm(self, vm_hash: VmHash) -> Optional[VmExecution]: def forget_vm(self, vm_hash: VmHash) -> None: self.executions.pop(vm_hash) + + async def stop(self): + """Stop all VMs in the pool.""" + hashes_to_forget: List[VmHash] = [] + for vm_hash, execution in self.executions.items(): + await execution.stop() + hashes_to_forget.append(vm_hash) + + for vm_hash in hashes_to_forget: + self.forget_vm(vm_hash) From 097a5ce6aac3bae7cb049640ca765934899a43ca Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 16:12:53 +0100 Subject: [PATCH 186/990] Fix: Networking can work inside a Podman container Not sure about Docker containers --- docker/vm_supervisor.dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile index a7dfe0dc0..0620d3bb8 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor.dockerfile @@ -25,8 +25,7 @@ RUN mkdir /srv/jailer ENV PYTHONPATH /mnt -# Networking does not work in Docker containers -ENV ALEPH_VM_ALLOW_VM_NETWORKING False +ENV ALEPH_VM_NETWORK_INTERFACE "tap0" # Jailer does not work in Docker containers ENV ALEPH_VM_USE_JAILER False # Use fake test data From 6a3922d84ba3bc52153e12fcc34163cef7e5d455 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 16:14:02 +0100 Subject: [PATCH 187/990] Comment: Explain why variable stays empty --- runtimes/aleph-alpine-3.13-python/init1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 918c1b8ab..483b80832 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -319,7 +319,7 @@ async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[by raise await asyncio.sleep(.05) - output = "" + output = "" # Process stdout is not captured per request output_data = None logger.debug("Returning result") return headers, body, output, output_data From 39d087114d386bc13e5ea7639d0421fdab3a080a Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 16:14:22 +0100 Subject: [PATCH 188/990] Fix: Shared cache should be enabled in example message --- examples/message_from_aleph.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index 36b12fa67..5a96022a0 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -39,7 +39,7 @@ "reproducible": true, "internet": true, "aleph_api": true, - "shared_cache": false + "shared_cache": true }, "resources": { "vcpus": 1, From 5960918d0edece2e298b1be770acbd9530d7148e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 18:00:43 +0100 Subject: [PATCH 189/990] Fix: Content-length header could appear twice, raising an error --- vm_supervisor/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py index 10564840b..b61706a8f 100644 --- a/vm_supervisor/run.py +++ b/vm_supervisor/run.py @@ -97,7 +97,8 @@ async def run_code_on_request(vm_hash: VmHash, path: str, request: web.Request) headers = { key.decode(): value.decode() for key, value in result["headers"]["headers"] } - headers["Content-Length"] = str(len(result["body"]["body"])) + if "content-length" not in headers: + headers["Content-Length".lower()] = str(len(result["body"]["body"])) for header in ["Content-Encoding", "Transfer-Encoding", "Vary"]: if header in headers: del headers[header] From bf47f012752d475c9b4cdba45939930e2cb58e16 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 18:01:25 +0100 Subject: [PATCH 190/990] Fix: Exceptions catching in example was too broad --- examples/example_fastapi_2/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/example_fastapi_2/main.py b/examples/example_fastapi_2/main.py index 91611a972..15945135c 100644 --- a/examples/example_fastapi_2/main.py +++ b/examples/example_fastapi_2/main.py @@ -120,12 +120,12 @@ async def keys_from_cache(pattern: str = '*'): @app.get("/state/increment") async def increment(): - path = "/var/lib/sqlite/mydb" + path = "/var/lib/example/storage.json" try: with open(path) as fd: data = json.load(fd) data["counter"] += 1 - except: + except FileNotFoundError: data = {"counter": 0} with open(path, 'w') as fd: json.dump(data, fd) From a1c9948d4c5ed05632158bda2d5d4f436869c7eb Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 16 Nov 2021 18:02:19 +0100 Subject: [PATCH 191/990] Feature: Add page to check if VM works on deployment --- vm_supervisor/conf.py | 2 ++ vm_supervisor/status.py | 70 +++++++++++++++++++++++++++++++++++++ vm_supervisor/supervisor.py | 2 ++ vm_supervisor/views.py | 15 ++++++++ 4 files changed, 89 insertions(+) create mode 100644 vm_supervisor/status.py diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 5ddfb70f0..1eb720439 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -96,6 +96,8 @@ class Settings(BaseSettings): FAKE_DATA_EXAMPLE: str = "example_fastapi_2" # FAKE_DATA_EXAMPLE: str = "example_django" + CHECK_FASTAPI_VM_ID: str = "bbd7f6e2ce72104a334f22e4b29f0ebeb96af3179167521788bce80754f3c58a" + def update(self, **kwargs): for key, value in kwargs.items(): if key != key.upper(): diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py new file mode 100644 index 000000000..3afed3128 --- /dev/null +++ b/vm_supervisor/status.py @@ -0,0 +1,70 @@ +""" +Used to check that the example_fastapi_2 program works as expected +in a deployed supervisor. +""" + +from typing import Dict, Any, List + +from aiohttp import ClientSession + +from vm_supervisor.conf import settings + +CHECK_VM_URL = f"http://{settings.SUPERVISOR_HOST}:{settings.SUPERVISOR_PORT}/vm/{settings.CHECK_FASTAPI_VM_ID}" + + +async def get_json_from_vm(session: ClientSession, suffix: str) -> Any: + url = f"{CHECK_VM_URL}{suffix}" + async with session.get(url) as resp: + resp.raise_for_status() + return await resp.json() + + +async def check_index(session: ClientSession) -> bool: + result: Dict = await get_json_from_vm(session, "/") + assert result["Example"] == "example_fastapi_2" + return True + + +async def check_environ(session: ClientSession) -> bool: + result: Dict = await get_json_from_vm(session, "/environ") + assert "ALEPH_API_HOST" in result + assert "ALEPH_API_UNIX_SOCKET" in result + assert "ALEPH_REMOTE_CRYPTO_HOST" in result + assert "ALEPH_REMOTE_CRYPTO_UNIX_SOCKET" in result + assert "ALEPH_ADDRESS_TO_USE" in result + return True + + +async def check_messages(session: ClientSession) -> bool: + result: Dict = await get_json_from_vm(session, "/messages") + assert "Messages" in result + assert "messages" in result["Messages"] + assert "item_hash" in result["Messages"]["messages"][0] + return True + + +async def check_internet(session: ClientSession) -> bool: + result: Dict = await get_json_from_vm(session, "/internet") + assert result["result"] == 200 + assert "Server" in result["headers"] + return True + + +async def check_cache(session: ClientSession) -> bool: + result1: bool = await get_json_from_vm(session, "/cache/set/a/42") + assert result1 == True + result2: int = await get_json_from_vm(session, "/cache/get/a") + assert result2 == "42" + keys: List[str] = await get_json_from_vm(session, "/cache/keys") + print("KEYS", keys) + assert "a" in keys + return True + + +async def check_persistent_storage(session: ClientSession) -> bool: + result: Dict = await get_json_from_vm(session, "/state/increment") + counter = result["counter"] + result_2: Dict = await get_json_from_vm(session, "/state/increment") + counter_2 = result_2["counter"] + assert counter_2 == counter + 1 + return True diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 93c257b2b..89f6908c1 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -18,6 +18,7 @@ about_login, about_executions, about_config, + status_check_fastapi, ) logger = logging.getLogger(__name__) @@ -29,6 +30,7 @@ web.get("/about/login", about_login), web.get("/about/executions", about_executions), web.get("/about/config", about_config), + web.get("/status/check/fastapi", status_check_fastapi), web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path), web.route("*", "/{suffix:.*}", run_code_from_hostname), ] diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py index 7f57f3c75..907e5f0fd 100644 --- a/vm_supervisor/views.py +++ b/vm_supervisor/views.py @@ -3,9 +3,11 @@ from typing import Awaitable import aiodns +import aiohttp from aiohttp import web from aiohttp.web_exceptions import HTTPNotFound +from . import status from .conf import settings from .models import VmHash from .run import run_code_on_request, pool @@ -94,3 +96,16 @@ async def about_config(request: web.Request): async def index(request: web.Request): assert request.method == "GET" return web.Response(text="Server: Aleph VM Supervisor") + + +async def status_check_fastapi(request: web.Request): + async with aiohttp.ClientSession() as session: + result = { + "index": await status.check_index(session), + "environ": await status.check_environ(session), + "messages": await status.check_messages(session), + "internet": await status.check_internet(session), + "cache": await status.check_cache(session), + "persistent_storage": await status.check_persistent_storage(session), + } + return web.json_response(result, status=200 if all(result.values()) else 503) \ No newline at end of file From b22f99b971a2d4b662091e1d6954c2401ea364bf Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 17 Nov 2021 14:36:39 +0100 Subject: [PATCH 192/990] Feature: Improve fake data configuration Add CLI argument --fake-data-program (-f) to pass a custom fake program directory. When used, all hashes used to reference programs and volumes will be ignored and fake data defined in the settings will be used instead. Patch Patch --- examples/message_from_aleph.json | 2 +- vm_supervisor/__main__.py | 13 ++++++++++-- vm_supervisor/conf.py | 19 +++++++++++++---- vm_supervisor/storage.py | 36 ++++++++++++-------------------- vm_supervisor/views.py | 4 ++-- 5 files changed, 42 insertions(+), 32 deletions(-) diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json index 5a96022a0..217e5a96e 100644 --- a/examples/message_from_aleph.json +++ b/examples/message_from_aleph.json @@ -3,7 +3,7 @@ "$oid": "6080402d7f44efefd611dc1e" }, "chain": "ETH", - "item_hash": "TEST_HASH", + "item_hash": "fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-hash", "sender": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", "type": "PROGRAM", "channel": "Fun-dApps", diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 41446f38b..645cd4920 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -3,7 +3,6 @@ import logging import sys import time -from asyncio import coroutine from statistics import mean from typing import List, Tuple, Dict @@ -96,6 +95,14 @@ def parse_args(args): default=0, help="Number of benchmarks to run", ) + parser.add_argument( + "-f", + "--fake-data-program", + dest="fake_data_program", + type=str, + default=None, + help="Path to project containing fake data", + ) return parser.parse_args(args) @@ -103,7 +110,8 @@ async def benchmark(runs: int): """Measure performance by immediately running the supervisor with fake requests. """ - ref = VmHash("cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e") + ref = VmHash("fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-hash") + settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM class FakeRequest: headers: Dict[str, str] @@ -189,6 +197,7 @@ def main(): PRINT_SYSTEM_LOGS=args.system_logs, PREALLOC_VM_COUNT=args.prealloc_vm_count, ALLOW_VM_NETWORKING=args.allow_vm_networking, + FAKE_DATA_PROGRAM=args.fake_data_program, ) settings.setup() if args.print_settings: diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 1eb720439..65825800f 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -2,7 +2,7 @@ import os import re from enum import Enum -from os.path import isfile, join, exists +from os.path import isfile, join, exists, abspath, isdir from subprocess import check_output from typing import NewType, Optional, List @@ -92,9 +92,13 @@ class Settings(BaseSettings): join("/var/tmp/aleph", "volumes", "persistent") ) - FAKE_DATA: bool = False - FAKE_DATA_EXAMPLE: str = "example_fastapi_2" - # FAKE_DATA_EXAMPLE: str = "example_django" + FAKE_DATA_PROGRAM: Optional[FilePath] = None + BENCHMARK_FAKE_DATA_PROGRAM: str = abspath(join(__file__, "../../examples/example_fastapi_2")) + + FAKE_DATA_MESSAGE: FilePath = abspath(join(__file__, "../../examples/message_from_aleph.json")) + FAKE_DATA_DATA: Optional[FilePath] = abspath(join(__file__, "../../examples/data/")) + FAKE_DATA_RUNTIME: FilePath = abspath(join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs")) + FAKE_DATA_VOLUME: Optional[FilePath] = abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs")) CHECK_FASTAPI_VM_ID: str = "bbd7f6e2ce72104a334f22e4b29f0ebeb96af3179167521788bce80754f3c58a" @@ -117,6 +121,13 @@ def check(self): if self.ALLOW_VM_NETWORKING: assert exists(f"/sys/class/net/{self.NETWORK_INTERFACE}") + if self.FAKE_DATA_PROGRAM: + assert isdir(self.FAKE_DATA_PROGRAM), "Local fake program directory is missing" + assert isfile(self.FAKE_DATA_MESSAGE), "Local fake message is missing" + assert isdir(self.FAKE_DATA_DATA), "Local fake data directory is missing" + assert isfile(self.FAKE_DATA_RUNTIME), "Local runtime .squashfs build is missing" + assert isfile(self.FAKE_DATA_VOLUME), "Local data volume .squashfs is missing" + def setup(self): os.makedirs(self.MESSAGE_CACHE, exist_ok=True) os.makedirs(self.CODE_CACHE, exist_ok=True) diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py index 984689ffb..5f6efeef8 100644 --- a/vm_supervisor/storage.py +++ b/vm_supervisor/storage.py @@ -10,7 +10,7 @@ import logging import os import re -from os.path import isfile, join, abspath +from os.path import isfile, join, dirname from shutil import make_archive import aiohttp @@ -53,7 +53,7 @@ async def download_file(url: str, local_path: FilePath) -> None: async def get_latest_amend(item_hash: str) -> str: - if settings.FAKE_DATA: + if settings.FAKE_DATA_PROGRAM: return item_hash else: url = f"{settings.CONNECTOR_URL}/compute/latest_amend/{item_hash}" @@ -66,10 +66,8 @@ async def get_latest_amend(item_hash: str) -> str: async def get_message(ref: str) -> ProgramMessage: - if settings.FAKE_DATA: - cache_path = os.path.abspath( - join(__file__, "../../examples/message_from_aleph.json") - ) + if settings.FAKE_DATA_PROGRAM: + cache_path = settings.FAKE_DATA_MESSAGE else: cache_path = FilePath(join(settings.MESSAGE_CACHE, ref) + ".json") url = f"{settings.CONNECTOR_URL}/download/message/{ref}" @@ -77,7 +75,7 @@ async def get_message(ref: str) -> ProgramMessage: with open(cache_path, "r") as cache_file: msg = json.load(cache_file) - if settings.FAKE_DATA: + if settings.FAKE_DATA_PROGRAM: msg["item_content"] = json.dumps(msg["content"]) msg["item_hash"] = hashlib.sha256( msg["item_content"].encode("utf-8") @@ -86,9 +84,8 @@ async def get_message(ref: str) -> ProgramMessage: async def get_code_path(ref: str) -> FilePath: - if settings.FAKE_DATA: - root_dir = abspath(join(__file__, "../../examples/")) - archive_path = join(root_dir, settings.FAKE_DATA_EXAMPLE) + if settings.FAKE_DATA_PROGRAM: + archive_path = settings.FAKE_DATA_PROGRAM encoding: Encoding = ( await get_message(ref="fake-message") @@ -111,8 +108,8 @@ async def get_code_path(ref: str) -> FilePath: async def get_data_path(ref: str) -> FilePath: - if settings.FAKE_DATA: - data_dir = abspath(join(__file__, "../../examples/data")) + if settings.FAKE_DATA_PROGRAM and settings.FAKE_DATA_DATA: + data_dir = settings.FAKE_DATA_DATA make_archive(data_dir, "zip", data_dir) return FilePath(f"{data_dir}.zip") @@ -123,12 +120,8 @@ async def get_data_path(ref: str) -> FilePath: async def get_runtime_path(ref: str) -> FilePath: - if settings.FAKE_DATA: - return FilePath( - os.path.abspath( - join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs") - ) - ) + if settings.FAKE_DATA_PROGRAM: + return FilePath(settings.FAKE_DATA_RUNTIME) cache_path = FilePath(join(settings.RUNTIME_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" @@ -151,11 +144,8 @@ def create_ext4(path: FilePath, size_mib: int) -> bool: async def get_volume_path(volume: MachineVolume, namespace: str) -> FilePath: if isinstance(volume, ImmutableVolume): ref = volume.ref - if settings.FAKE_DATA: - data_dir = abspath( - join(__file__, "../../examples/volumes/volume-venv.squashfs") - ) - return FilePath(data_dir) + if settings.FAKE_DATA_PROGRAM and settings.FAKE_DATA_VOLUME: + return FilePath(settings.FAKE_DATA_VOLUME) cache_path = FilePath(join(settings.DATA_CACHE, ref)) url = f"{settings.CONNECTOR_URL}/download/data/{ref}" diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py index 907e5f0fd..c4fd1b986 100644 --- a/vm_supervisor/views.py +++ b/vm_supervisor/views.py @@ -43,8 +43,8 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: path = path if path.startswith("/") else f"/{path}" message_ref_base32 = request.host.split(".")[0] - if settings.FAKE_DATA: - message_ref = "TEST_HASH" + if settings.FAKE_DATA_PROGRAM: + message_ref = "fake-hash" else: try: message_ref = b32_to_b16(message_ref_base32).decode() From 8a4cb1716b46bb9748e6f3b0d49a37d6f1b6991c Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 17 Nov 2021 14:37:29 +0100 Subject: [PATCH 193/990] Feature: Add setting to avoid watching for new messages --- vm_supervisor/__main__.py | 1 + vm_supervisor/conf.py | 1 + vm_supervisor/supervisor.py | 6 ++++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 645cd4920..4fedcf46c 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -136,6 +136,7 @@ async def fake_read() -> bytes: bench: List[float] = [] # Does not make sense in benchmarks + settings.WATCH_FOR_MESSAGES = False settings.WATCH_FOR_UPDATES = False # First test all methods diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 65825800f..09c67205e 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -64,6 +64,7 @@ class Settings(BaseSettings): START_ID_INDEX: int = 4 PREALLOC_VM_COUNT: int = 0 REUSE_TIMEOUT: float = 60 * 60.0 + WATCH_FOR_MESSAGES: bool = True WATCH_FOR_UPDATES: bool = True NETWORK_INTERFACE: str = "eth0" DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index 89f6908c1..c54eb9a28 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -46,7 +46,9 @@ def run(): app["secret_token"] = secret_token print(f"Login to /about pages /about/login?token={secret_token}") - app.on_startup.append(start_watch_for_messages_task) - app.on_cleanup.append(stop_watch_for_messages_task) + if settings.WATCH_FOR_MESSAGES: + app.on_startup.append(start_watch_for_messages_task) + app.on_cleanup.append(stop_watch_for_messages_task) + web.run_app(app, host=settings.SUPERVISOR_HOST, port=settings.SUPERVISOR_PORT) From af3cfe859c770bfe9b04e0554dad5f76b3f603ab Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 17 Nov 2021 14:39:36 +0100 Subject: [PATCH 194/990] Fix: chpasswd was not found, hidden in /usr/sbin --- runtimes/aleph-debian-11-python/create_disk_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh index 3ae16c34d..f30005f43 100644 --- a/runtimes/aleph-debian-11-python/create_disk_image.sh +++ b/runtimes/aleph-debian-11-python/create_disk_image.sh @@ -33,7 +33,7 @@ pip3 install 'aleph-client>=0.4.4' 'coincurve==15.0.0' # Compile all Python bytecode python3 -m compileall -f /usr/local/lib/python3.9 -echo "root:toor" | chpasswd +echo "root:toor" | /usr/sbin/chpasswd mkdir -p /overlay From 24ba884740b8ecf50a226b2930ca1b48812791e3 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 17 Nov 2021 15:10:45 +0100 Subject: [PATCH 195/990] Security: Limit max size of inline zip archives These are passed in memory via the configuration. A very large size could cause performance issues on the supervisor. --- vm_supervisor/conf.py | 3 +++ vm_supervisor/run.py | 6 +++++- vm_supervisor/vm/firecracker_microvm.py | 12 ++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 09c67205e..9aaf7dbb0 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -93,6 +93,9 @@ class Settings(BaseSettings): join("/var/tmp/aleph", "volumes", "persistent") ) + MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000 # 10 MB + MAX_DATA_ARCHIVE_SIZE = 10_000_000 # 10 MB + FAKE_DATA_PROGRAM: Optional[FilePath] = None BENCHMARK_FAKE_DATA_PROGRAM: str = abspath(join(__file__, "../../examples/example_fastapi_2")) diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py index b61706a8f..debcf566b 100644 --- a/vm_supervisor/run.py +++ b/vm_supervisor/run.py @@ -12,7 +12,7 @@ from .models import VmHash, VmExecution from .pool import VmPool from .pubsub import PubSub -from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError +from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError, FileTooLargeError logger = logging.getLogger(__name__) @@ -60,6 +60,8 @@ async def run_code_on_request(vm_hash: VmHash, path: str, request: web.Request) logger.exception(error) pool.forget_vm(vm_hash=vm_hash) raise HTTPBadRequest(reason="Code, runtime or data not available") + except FileTooLargeError as error: + raise HTTPInternalServerError(reason=error.args[0]) except VmSetupError as error: logger.exception(error) pool.forget_vm(vm_hash=vm_hash) @@ -145,6 +147,8 @@ async def run_code_on_event(vm_hash: VmHash, event, pubsub: PubSub): logger.exception(error) pool.forget_vm(vm_hash=vm_hash) raise HTTPBadRequest(reason="Code, runtime or data not available") + except FileTooLargeError as error: + raise HTTPInternalServerError(reason=error.args[0]) except VmSetupError as error: logger.exception(error) pool.forget_vm(vm_hash=vm_hash) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index dcf42df6f..bf29a4c95 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -1,6 +1,7 @@ import asyncio import dataclasses import logging +import os.path import subprocess from dataclasses import dataclass, field from enum import Enum @@ -44,6 +45,11 @@ def load_file_content(path: FilePath) -> bytes: return b"" +class FileTooLargeError(Exception): + pass + + + class ResourceDownloadError(ClientResponseError): """An error occurred while downloading a VM resource file""" @@ -324,6 +330,9 @@ async def start(self): async def configure(self): """Configure the VM by sending configuration info to it's init""" + if self.resources.data_path and os.path.getsize(self.resources.data_path) > settings.MAX_DATA_ARCHIVE_SIZE: + raise FileTooLargeError(f"Data file too large to pass as an inline zip") + input_data: bytes = load_file_content(self.resources.data_path) interface = ( @@ -344,6 +353,9 @@ async def configure(self): for index, volume in enumerate(self.resources.volumes) ] else: + if self.resources.data_path and os.path.getsize(self.resources.code_path) > settings.MAX_PROGRAM_ARCHIVE_SIZE: + raise FileTooLargeError(f"Program file too large to pass as an inline zip") + code: bytes = load_file_content(self.resources.code_path) volumes = [ Volume( From 9143d3dc29bbd9e5d4b5442cf83cf9757ec96a8e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 17 Nov 2021 17:09:12 +0100 Subject: [PATCH 196/990] Change: Improve running supervisor in a container --- docker/run_vm_supervisor.sh | 4 +++- docker/vm_supervisor.dockerfile | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/run_vm_supervisor.sh b/docker/run_vm_supervisor.sh index a7c7a69cb..ae065b1f3 100755 --- a/docker/run_vm_supervisor.sh +++ b/docker/run_vm_supervisor.sh @@ -4,5 +4,7 @@ podman build -ti -t aleph-vm-supervisor -f docker/vm_supervisor.dockerfile . podman run -ti --rm \ -v $(pwd):/root/aleph-vm \ --device /dev/kvm \ + -p 4020:4020 \ aleph-vm-supervisor \ - python3 -m vm_supervisor -p -vv --system-logs --benchmark 1 --profile + bash +# python3 -m vm_supervisor -p -vv --system-logs --profile -f ./examples/example_fastapi_2 diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor.dockerfile index 0620d3bb8..799004348 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor.dockerfile @@ -25,13 +25,17 @@ RUN mkdir /srv/jailer ENV PYTHONPATH /mnt +# Networking only works in privileged containers +ENV ALEPH_VM_ALLOW_VM_NETWORKING False ENV ALEPH_VM_NETWORK_INTERFACE "tap0" # Jailer does not work in Docker containers ENV ALEPH_VM_USE_JAILER False # Use fake test data ENV ALEPH_VM_FAKE_DATA True +# Allow connections from host +ENV ALEPH_VM_SUPERVISOR_HOST "0.0.0.0" # Make it easy to enter this command from a shell script -RUN echo "python3 -m vm_supervisor -p -vv --system-logs --benchmark 1 --profile" >> /root/.bash_history +RUN echo "python3 -m vm_supervisor --print-settings --very-verbose --system-logs --profile -f ./examples/example_fastapi_2" >> /root/.bash_history WORKDIR /root/aleph-vm From 131fc763768cc78d640aec5408df32bc8d757371 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 17 Nov 2021 17:10:35 +0100 Subject: [PATCH 197/990] Problem: Podman was not supported in Docker launch scripts --- docker/run_vm_connector.sh | 12 ++++++++++-- docker/run_vm_supervisor.sh | 13 +++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docker/run_vm_connector.sh b/docker/run_vm_connector.sh index 5a4811dd5..fc2c60845 100755 --- a/docker/run_vm_connector.sh +++ b/docker/run_vm_connector.sh @@ -2,9 +2,17 @@ set -euf -docker build -t aleph-connector -f docker/vm_connector.dockerfile . +# Use Podman if installed, else use Docker +if hash podman 2> /dev/null +then + DOCKER_COMMAND=podman +else + DOCKER_COMMAND=docker +fi -docker run -ti --rm -p 8000:8000/tcp \ +$DOCKER_COMMAND build -t aleph-connector -f docker/vm_connector.dockerfile . + +$DOCKER_COMMAND run -ti --rm -p 4021:4021/tcp \ -v "$(pwd)/kernels:/opt/kernels:ro" \ -v "$(pwd)/vm_connector:/opt/vm_connector:ro" \ --name aleph-connector \ diff --git a/docker/run_vm_supervisor.sh b/docker/run_vm_supervisor.sh index ae065b1f3..d34e4510e 100755 --- a/docker/run_vm_supervisor.sh +++ b/docker/run_vm_supervisor.sh @@ -1,7 +1,16 @@ #!/bin/sh -podman build -ti -t aleph-vm-supervisor -f docker/vm_supervisor.dockerfile . -podman run -ti --rm \ +# Use Podman if installed, else use Docker +if hash podman 2> /dev/null +then + DOCKER_COMMAND=podman +else + DOCKER_COMMAND=docker +fi + +$DOCKER_COMMAND build -ti -t aleph-vm-supervisor -f docker/vm_supervisor.dockerfile . + +$DOCKER_COMMAND run -ti --rm \ -v $(pwd):/root/aleph-vm \ --device /dev/kvm \ -p 4020:4020 \ From 3a08b3d746b96b38221f3780c019457ac79a076f Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 17 Nov 2021 17:12:47 +0100 Subject: [PATCH 198/990] Feature: Improve developer tools with Docker supervisor image --- .dockerignore | 2 + .gitignore | 2 + docker/publish_vm_connector.sh | 2 +- docker/publish_vm_supervisor_dev.sh | 18 +++++ docker/run_vm_supervisor.sh | 9 ++- ...ockerfile => vm_supervisor-dev.dockerfile} | 13 +++- tutorials/TESTING.md | 65 +++++++++++++++++++ 7 files changed, 103 insertions(+), 8 deletions(-) mode change 100644 => 100755 docker/publish_vm_connector.sh create mode 100755 docker/publish_vm_supervisor_dev.sh rename docker/{vm_supervisor.dockerfile => vm_supervisor-dev.dockerfile} (83%) create mode 100644 tutorials/TESTING.md diff --git a/.dockerignore b/.dockerignore index 64b7aba61..c9e440ce8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,5 +9,7 @@ **/*.ext4 **/*.zip **/*.pyz +**/*.rdb +**/*.key **/data.tgz /pydantic/ diff --git a/.gitignore b/.gitignore index 98069599f..72f7e3b2c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ __pycache__ *.zip *.pyz *.tgz +*.rdb +*.key /pydantic/ node_modules *.squashfs diff --git a/docker/publish_vm_connector.sh b/docker/publish_vm_connector.sh old mode 100644 new mode 100755 index cf5de87c4..4101400bf --- a/docker/publish_vm_connector.sh +++ b/docker/publish_vm_connector.sh @@ -15,4 +15,4 @@ $DOCKER_COMMAND build -t alephim/vm-connector -f docker/vm_connector.dockerfile $DOCKER_COMMAND tag alephim/vm-connector alephim/vm-connector:$VERSION $DOCKER_COMMAND push alephim/vm-connector:$VERSION docker.io/alephim/vm-connector:$VERSION -echo docker.io/alephim/pyaleph-node:$VERSION +echo docker.io/alephim/vm-connector:$VERSION diff --git a/docker/publish_vm_supervisor_dev.sh b/docker/publish_vm_supervisor_dev.sh new file mode 100755 index 000000000..69827f2de --- /dev/null +++ b/docker/publish_vm_supervisor_dev.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -euf -o pipefail + +if hash docker 2> /dev/null +then + DOCKER_COMMAND=docker +else + DOCKER_COMMAND=podman +fi + +#VERSION=$(git describe --tags)-alpha +VERSION=alpha + +$DOCKER_COMMAND build -t alephim/vm-supervisor-dev -f docker/vm_supervisor-dev.dockerfile . + +$DOCKER_COMMAND tag alephim/vm-supervisor-dev alephim/vm-supervisor-dev:$VERSION +$DOCKER_COMMAND push alephim/vm-supervisor-dev:$VERSION docker.io/alephim/vm-supervisor-dev:$VERSION +echo docker.io/alephim/vm-supervisor-dev:$VERSION diff --git a/docker/run_vm_supervisor.sh b/docker/run_vm_supervisor.sh index d34e4510e..8ec235009 100755 --- a/docker/run_vm_supervisor.sh +++ b/docker/run_vm_supervisor.sh @@ -8,12 +8,11 @@ else DOCKER_COMMAND=docker fi -$DOCKER_COMMAND build -ti -t aleph-vm-supervisor -f docker/vm_supervisor.dockerfile . +$DOCKER_COMMAND build -t alephim/vm-supervisor-dev -f docker/vm_supervisor-dev.dockerfile . $DOCKER_COMMAND run -ti --rm \ - -v $(pwd):/root/aleph-vm \ + -v "$(pwd)/runtimes/aleph-debian-11-python/rootfs.squashfs:/opt/aleph-vm/runtimes/aleph-debian-11-python/rootfs.squashfs:ro" \ + -v "$(pwd)/examples/volumes/volume-venv.squashfs:/opt/aleph-vm/examples/volumes/volume-venv.squashfs:ro" \ --device /dev/kvm \ -p 4020:4020 \ - aleph-vm-supervisor \ - bash -# python3 -m vm_supervisor -p -vv --system-logs --profile -f ./examples/example_fastapi_2 + alephim/vm-supervisor-dev diff --git a/docker/vm_supervisor.dockerfile b/docker/vm_supervisor-dev.dockerfile similarity index 83% rename from docker/vm_supervisor.dockerfile rename to docker/vm_supervisor-dev.dockerfile index 799004348..e2f96ad2f 100644 --- a/docker/vm_supervisor.dockerfile +++ b/docker/vm_supervisor-dev.dockerfile @@ -1,6 +1,6 @@ # This is mainly a copy of the installation instructions from [vm_supervisor/README.md] -FROM debian:buster +FROM debian:bullseye RUN apt-get update && apt-get -y upgrade && apt-get install -y \ sudo acl curl systemd-container \ @@ -38,4 +38,13 @@ ENV ALEPH_VM_SUPERVISOR_HOST "0.0.0.0" # Make it easy to enter this command from a shell script RUN echo "python3 -m vm_supervisor --print-settings --very-verbose --system-logs --profile -f ./examples/example_fastapi_2" >> /root/.bash_history -WORKDIR /root/aleph-vm +RUN mkdir /opt/aleph-vm/ +COPY ./vm_supervisor /opt/aleph-vm/vm_supervisor +COPY ./firecracker /opt/aleph-vm/firecracker +COPY ./guest_api /opt/aleph-vm/guest_api +COPY ./examples /opt/aleph-vm/examples +COPY ./runtimes /opt/aleph-vm/runtimes + +WORKDIR /opt/aleph-vm + +CMD "bash" diff --git a/tutorials/TESTING.md b/tutorials/TESTING.md new file mode 100644 index 000000000..0061e162f --- /dev/null +++ b/tutorials/TESTING.md @@ -0,0 +1,65 @@ +# Testing your VMs locally + +You can test your VM locally without uploading each version on the Aleph network. + +To do this, you'll want to use the `--fake-data-program` or `-f` argument of the VM Supervisor. + +## 0. Build the required squashfs volumes + +Build or download the required squashfs volumes: + +```shell +cd ./runtimes/aleph-debian-11-python/ +sudo bash ./create_disk_image.sh + +cd ../.. +``` +> ℹ️ This does not work in a container since debootstrap requires mounting volumes. + +This will create a local runtime root filesystem in `./runtimes/aleph-debian-11-python/rootfs.squashfs`. + +```shell +cd ./examples/volumes/ +bash ./build_squashfs.sh + +cd ../.. +``` +This will create a local example read-only volume named `./example/volumes/volume-venv.squashfs`. + +## 1. In a Docker container + +Run the developer image, mounting the two generated volumes: +```shell +run -ti --rm \ + -v "$(pwd)/runtimes/aleph-debian-11-python/rootfs.squashfs:/opt/aleph-vm/runtimes/aleph-debian-11-python/rootfs.squashfs:ro" \ + -v "$(pwd)/examples/volumes/volume-venv.squashfs:/opt/aleph-vm/examples/volumes/volume-venv.squashfs:ro" \ + --device /dev/kvm \ + -p 4020:4020 \ + docker.io/alephim/vm-supervisor-dev +``` + +Or launch this command using: +```shell +bash ./docker/run_vm_supervisor.sh +``` + + +Within the container, run the supervisor with fake data: +```shell +python3 -m vm_supervisor --print-settings --very-verbose --system-logs --fake-data-program ./examples/example_fastapi_2 +``` + +> ℹ️ The command is in your .bash_history, press key up to skip typing it. + +## 2. On your system + +### 2.a. Install the system requirements + +See [../vm_supervisor/README.md](../vm_supervisor/README.md) to install the system requirements. + +### 2.b. Run the supervisor with fake data: + +```shell +python3 -m vm_supervisor --print-settings --very-verbose --system-logs --fake-data-program ./examples/example_fastapi_2 +``` + From a4ae50845933bebe30d5ee5d6a9290eb4ac36f32 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 19 Nov 2021 13:59:49 +0100 Subject: [PATCH 199/990] Fix: Typo in TESTING.md --- tutorials/TESTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/TESTING.md b/tutorials/TESTING.md index 0061e162f..894bbfcd1 100644 --- a/tutorials/TESTING.md +++ b/tutorials/TESTING.md @@ -30,7 +30,7 @@ This will create a local example read-only volume named `./example/volumes/volum Run the developer image, mounting the two generated volumes: ```shell -run -ti --rm \ +docker run -ti --rm \ -v "$(pwd)/runtimes/aleph-debian-11-python/rootfs.squashfs:/opt/aleph-vm/runtimes/aleph-debian-11-python/rootfs.squashfs:ro" \ -v "$(pwd)/examples/volumes/volume-venv.squashfs:/opt/aleph-vm/examples/volumes/volume-venv.squashfs:ro" \ --device /dev/kvm \ From cc50f4b0a9e31338fd0badd5ce2142a27ad12246 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 19 Nov 2021 15:23:41 +0100 Subject: [PATCH 200/990] Fix: Handle task cancellation error properly --- vm_supervisor/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py index ca50d2e11..7bc67c52a 100644 --- a/vm_supervisor/tasks.py +++ b/vm_supervisor/tasks.py @@ -85,4 +85,7 @@ async def start_watch_for_messages_task(app: web.Application): async def stop_watch_for_messages_task(app: web.Application): app["messages_listener"].cancel() - await app["messages_listener"] + try: + await app["messages_listener"] + except asyncio.CancelledError: + logger.debug("Task messages_listener is cancelled now") From 9e8821ad45456c59f689e22688ace9b88bef6669 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 2 Dec 2021 17:46:46 +0100 Subject: [PATCH 201/990] Feature: User could not see which version a VM is running Solution: Add headers that display the program item_hash and current code ref. --- vm_supervisor/run.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py index debcf566b..070870014 100644 --- a/vm_supervisor/run.py +++ b/vm_supervisor/run.py @@ -105,6 +105,12 @@ async def run_code_on_request(vm_hash: VmHash, path: str, request: web.Request) if header in headers: del headers[header] + headers.update({ + "Aleph-Program-ItemHash": execution.vm_hash, + "Aleph-Program-Code-Ref": execution.program.code.ref, + # "Aleph-Compute-Vm-Id": str(execution.vm.vm_id), + }) + return web.Response( status=result["headers"]["status"], body=result["body"]["body"], From d1ea5915228e26f9bd5184fcb7132fb3616b8128 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 13 Dec 2021 12:14:02 +0100 Subject: [PATCH 202/990] Fix: Unhandled errors would crash the websocket subscription --- vm_supervisor/tasks.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py index 7bc67c52a..9f25cc9e3 100644 --- a/vm_supervisor/tasks.py +++ b/vm_supervisor/tasks.py @@ -34,9 +34,15 @@ async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]: try: yield Message(**data) except pydantic.error_wrappers.ValidationError as error: - print(error.json()) - print(error.raw_errors) - raise + logger.error(f"Invalid Aleph message: \n {error.json()}\n {error.raw_errors}", + exc_info=True) + continue + except KeyError: + logger.exception(f"Invalid Aleph message could not be parsed", exc_info=True) + continue + except Exception: + logger.exception(f"Unknown error when parsing Aleph message", exc_info=True) + continue elif msg.type == aiohttp.WSMsgType.ERROR: break From ee28d8a65246055454f80d8c64e2db579a32e314 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 13 Dec 2021 13:07:14 +0100 Subject: [PATCH 203/990] Fix: Watch for message ref==None was not properly handled --- vm_supervisor/tasks.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py index 9f25cc9e3..f84faca85 100644 --- a/vm_supervisor/tasks.py +++ b/vm_supervisor/tasks.py @@ -58,11 +58,10 @@ async def watch_for_messages(dispatcher: PubSub, reactor: Reactor): logger.info(f"Websocket received message: {message.item_hash}") # Dispatch update to running VMs - ref = ( - message.content.ref - if hasattr(message.content, "ref") - else message.item_hash - ) + if hasattr(message.content, "ref") and message.content.ref: + ref = message.content.ref + else: + ref = message.item_hash await dispatcher.publish(key=ref, value=message) # Register new VM to run on future messages: From 884c90fa36377abeed6964c91ab61dcd26f2dc0a Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 13 Dec 2021 13:08:10 +0100 Subject: [PATCH 204/990] Fix: JSON decoding errors were not handled in websocket subscription --- vm_supervisor/tasks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py index f84faca85..16cf7862a 100644 --- a/vm_supervisor/tasks.py +++ b/vm_supervisor/tasks.py @@ -28,9 +28,12 @@ async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]: async for msg in ws: logger.debug(f"Websocket received data...") if msg.type == aiohttp.WSMsgType.TEXT: - data = json.loads(msg.data) - # Patch data format to match HTTP GET format - data["_id"] = {"$oid": data["_id"]} + try: + data = json.loads(msg.data) + # Patch data format to match HTTP GET format + data["_id"] = {"$oid": data["_id"]} + except json.JSONDecodeError: + logger.error(f"Invalid JSON from websocket subscription {msg.data}", exc_info=True) try: yield Message(**data) except pydantic.error_wrappers.ValidationError as error: From 8103af4af734c03b179153550b0d9f8c2829ba29 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 13 Dec 2021 13:09:00 +0100 Subject: [PATCH 205/990] Logging: Lack of details made exceptions difficult to analyze --- vm_supervisor/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py index 16cf7862a..0434662ff 100644 --- a/vm_supervisor/tasks.py +++ b/vm_supervisor/tasks.py @@ -41,10 +41,10 @@ async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]: exc_info=True) continue except KeyError: - logger.exception(f"Invalid Aleph message could not be parsed", exc_info=True) + logger.exception(f"Invalid Aleph message could not be parsed '{data}'", exc_info=True) continue except Exception: - logger.exception(f"Unknown error when parsing Aleph message", exc_info=True) + logger.exception(f"Unknown error when parsing Aleph message {data}", exc_info=True) continue elif msg.type == aiohttp.WSMsgType.ERROR: break From 1a2251c5a8f9f64ec8784e3c91bbfc1f880756eb Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 13 Dec 2021 13:10:06 +0100 Subject: [PATCH 206/990] Fix: Websocket subscription would not reconnect if disconnected --- vm_supervisor/tasks.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py index 0434662ff..4aaa05774 100644 --- a/vm_supervisor/tasks.py +++ b/vm_supervisor/tasks.py @@ -3,7 +3,7 @@ import logging import math import time -from typing import AsyncIterable +from typing import AsyncIterable, TypeVar import aiohttp import pydantic @@ -19,6 +19,17 @@ logger = logging.getLogger(__name__) +Value = TypeVar('Value') + +async def retry_generator(generator: AsyncIterable[Value], max_seconds: int = 8) -> AsyncIterable[Value]: + retry_delay = 0.1 + while True: + async for value in generator: + yield value + + await asyncio.sleep(retry_delay) + retry_delay = max(retry_delay * 2, max_seconds) + async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]: logger.debug("subscribe_via_ws()") @@ -57,7 +68,7 @@ async def watch_for_messages(dispatcher: PubSub, reactor: Reactor): {"startDate": math.floor(time.time())} ) - async for message in subscribe_via_ws(url): + async for message in retry_generator(subscribe_via_ws(url)): logger.info(f"Websocket received message: {message.item_hash}") # Dispatch update to running VMs From fd7037abcb934445befadb074f52f6d7399344ee Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 13 Dec 2021 16:11:20 +0100 Subject: [PATCH 207/990] Fix: Mypy rejected some annotations --- docker/run_benchmark_in_docker.sh | 0 firecracker/config.py | 4 ++-- firecracker/microvm.py | 2 +- guest_api/__main__.py | 27 +++++++++++++------------ vm_supervisor/__main__.py | 17 ++++++++++------ vm_supervisor/conf.py | 10 ++++----- vm_supervisor/models.py | 19 ++++++++++------- vm_supervisor/run.py | 12 ++++++++--- vm_supervisor/tasks.py | 3 ++- vm_supervisor/views.py | 8 ++++---- vm_supervisor/vm/firecracker_microvm.py | 4 +++- 11 files changed, 63 insertions(+), 43 deletions(-) delete mode 100644 docker/run_benchmark_in_docker.sh diff --git a/docker/run_benchmark_in_docker.sh b/docker/run_benchmark_in_docker.sh deleted file mode 100644 index e69de29bb..000000000 diff --git a/firecracker/config.py b/firecracker/config.py index 439f93c7a..4c906e8b6 100644 --- a/firecracker/config.py +++ b/firecracker/config.py @@ -7,7 +7,7 @@ class BootSource(BaseModel): - kernel_image_path: FilePath = "vmlinux.bin" + kernel_image_path: FilePath = FilePath("vmlinux.bin") boot_args: str = ( "console=ttyS0 reboot=k panic=1 pci=off " "ro noapic nomodules random.trust_cpu=on" @@ -24,7 +24,7 @@ def args(enable_console: bool = True): class Drive(BaseModel): drive_id: str = "rootfs" - path_on_host: FilePath = "./runtimes/aleph-alpine-3.13-python/rootfs.ext4" + path_on_host: FilePath = FilePath("./runtimes/aleph-alpine-3.13-python/rootfs.ext4") is_root_device: bool = True is_read_only: bool = True diff --git a/firecracker/microvm.py b/firecracker/microvm.py index 8415b9ab5..f1c283112 100644 --- a/firecracker/microvm.py +++ b/firecracker/microvm.py @@ -68,7 +68,7 @@ class MicroVM: stdout_task: Optional[Task] = None stderr_task: Optional[Task] = None config_file = None - drives: List[Drive] = None + drives: List[Drive] init_timeout: float @property diff --git a/guest_api/__main__.py b/guest_api/__main__.py index a8b799d3f..cd442b650 100644 --- a/guest_api/__main__.py +++ b/guest_api/__main__.py @@ -15,7 +15,8 @@ async def proxy(request: web.Request): - path = request.match_info.get('tail').lstrip('/') + tail: str = request.match_info.get('tail') or "" + path: str = tail.lstrip('/') query_string = request.rel_url.query_string url = f"{ALEPH_API_SERVER}/{path}?{query_string}" @@ -70,7 +71,7 @@ async def properties(request: web.Request): async def sign(request: web.Request): - vm_hash = request.app.meta_vm_hash + vm_hash = request.app['meta_vm_hash'] message = await request.json() # Ensure that the hash of the VM is used as sending address @@ -90,9 +91,9 @@ async def sign(request: web.Request): async def get_from_cache(request: web.Request): - prefix: str = request.app.meta_vm_hash - key: str = request.match_info.get('key') - if not re.match(r'^\w+$', key): + prefix: str = request.app['meta_vm_hash'] + key: Optional[str] = request.match_info.get('key') + if not (key and re.match(r'^\w+$', key)): return web.HTTPBadRequest(text="Invalid key") redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost") @@ -104,9 +105,9 @@ async def get_from_cache(request: web.Request): async def put_in_cache(request: web.Request): - prefix: str = request.app.meta_vm_hash - key: str = request.match_info.get('key') - if not re.match(r'^\w+$', key): + prefix: str = request.app['meta_vm_hash'] + key: Optional[str] = request.match_info.get('key') + if not (key and re.match(r'^\w+$', key)): return web.HTTPBadRequest(text="Invalid key") value: bytes = await request.read() @@ -117,9 +118,9 @@ async def put_in_cache(request: web.Request): async def delete_from_cache(request: web.Request): - prefix: str = request.app.meta_vm_hash - key: str = request.match_info.get('key') - if not re.match(r'^\w+$', key): + prefix: str = request.app['meta_vm_hash'] + key: Optional[str] = request.match_info.get('key') + if not (key and re.match(r'^\w+$', key)): return web.HTTPBadRequest(text="Invalid key") redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost") @@ -128,7 +129,7 @@ async def delete_from_cache(request: web.Request): async def list_keys_from_cache(request: web.Request): - prefix: str = request.app.meta_vm_hash + prefix: str = request.app['meta_vm_hash'] pattern: str = request.rel_url.query.get('pattern', '*') if not re.match(r'^[\w?*^\-]+$', pattern): return web.HTTPBadRequest(text="Invalid key") @@ -144,7 +145,7 @@ async def list_keys_from_cache(request: web.Request): def run_guest_api(unix_socket_path, vm_hash: Optional[str] = None): app = web.Application() - app.meta_vm_hash = vm_hash or '_' + app['meta_vm_hash'] = vm_hash or '_' app.router.add_route(method='GET', path='/properties', handler=properties) app.router.add_route(method='POST', path='/sign', handler=sign) diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 4fedcf46c..5173565e2 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -4,9 +4,9 @@ import sys import time from statistics import mean -from typing import List, Tuple, Dict +from typing import List, Tuple, Dict, Callable -from aiohttp.web import Response +from aiohttp.web import Response, Request from vm_supervisor.pubsub import PubSub from .run import run_code_on_request, run_code_on_event @@ -113,11 +113,16 @@ async def benchmark(runs: int): ref = VmHash("fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-hash") settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM - class FakeRequest: + FakeRequest: Request + class FakeRequest: # type: ignore[no-redef] headers: Dict[str, str] raw_headers: List[Tuple[bytes, bytes]] + match_info: Dict + method: str + query_string: str + read: Callable - fake_request = FakeRequest() + fake_request = FakeRequest() # type: ignore[operator] fake_request.match_info = {"ref": ref, "suffix": "/"} fake_request.method = "GET" fake_request.query_string = "" @@ -163,10 +168,10 @@ async def fake_read() -> bytes: for run in range(runs): t0 = time.time() fake_request.match_info["suffix"] = path - response: Response = await run_code_on_request( + response2: Response = await run_code_on_request( vm_hash=ref, path=path, request=fake_request ) - assert response.status == 200 + assert response2.status == 200 bench.append(time.time() - t0) logger.info( diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 9aaf7dbb0..a67c43e6d 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -97,12 +97,12 @@ class Settings(BaseSettings): MAX_DATA_ARCHIVE_SIZE = 10_000_000 # 10 MB FAKE_DATA_PROGRAM: Optional[FilePath] = None - BENCHMARK_FAKE_DATA_PROGRAM: str = abspath(join(__file__, "../../examples/example_fastapi_2")) + BENCHMARK_FAKE_DATA_PROGRAM: FilePath = FilePath(abspath(join(__file__, "../../examples/example_fastapi_2"))) - FAKE_DATA_MESSAGE: FilePath = abspath(join(__file__, "../../examples/message_from_aleph.json")) - FAKE_DATA_DATA: Optional[FilePath] = abspath(join(__file__, "../../examples/data/")) - FAKE_DATA_RUNTIME: FilePath = abspath(join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs")) - FAKE_DATA_VOLUME: Optional[FilePath] = abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs")) + FAKE_DATA_MESSAGE: FilePath = FilePath(abspath(join(__file__, "../../examples/message_from_aleph.json"))) + FAKE_DATA_DATA: Optional[FilePath] = FilePath(abspath(join(__file__, "../../examples/data/"))) + FAKE_DATA_RUNTIME: FilePath = FilePath(abspath(join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs"))) + FAKE_DATA_VOLUME: Optional[FilePath] = FilePath(abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs"))) CHECK_FASTAPI_VM_ID: str = "bbd7f6e2ce72104a334f22e4b29f0ebeb96af3179167521788bce80754f3c58a" diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py index 521221bff..48adb5873 100644 --- a/vm_supervisor/models.py +++ b/vm_supervisor/models.py @@ -18,7 +18,7 @@ @dataclass class VmExecutionTimes: - defined_at: datetime = None + defined_at: datetime preparing_at: Optional[datetime] = None prepared_at: Optional[datetime] = None starting_at: Optional[datetime] = None @@ -40,14 +40,14 @@ class VmExecution: vm_hash: VmHash original: ProgramContent program: ProgramContent - resources: Optional[AlephFirecrackerResources] - vm: AlephFirecrackerVM = None + resources: Optional[AlephFirecrackerResources] = None + vm: Optional[AlephFirecrackerVM] = None times: VmExecutionTimes - ready_event: asyncio.Event = None - concurrent_runs: int = None - runs_done_event: asyncio.Event = None + ready_event: asyncio.Event + concurrent_runs: int + runs_done_event: asyncio.Event expire_task: Optional[asyncio.Task] = None @property @@ -84,6 +84,8 @@ async def prepare(self): self.resources = resources async def create(self, address: int) -> AlephFirecrackerVM: + if not self.resources: + raise ValueError("Execution resources must be configured first") self.times.starting_at = datetime.now() self.vm = vm = AlephFirecrackerVM( vm_id=address, @@ -112,8 +114,9 @@ def stop_after_timeout(self, timeout: float = 5.0) -> Task: loop = asyncio.get_event_loop() if sys.version_info.major >= 3 and sys.version_info.minor >= 8: # Task can be named + vm_id: str = str(self.vm.vm_id if self.vm else None) self.expire_task = loop.create_task( - self.expire(timeout), name=f"expire {self.vm.vm_id}" + self.expire(timeout), name=f"expire {vm_id}" ) else: self.expire_task = loop.create_task(self.expire(timeout)) @@ -169,6 +172,8 @@ async def all_runs_complete(self): await self.runs_done_event.wait() async def run_code(self, scope: dict = None) -> bytes: + if not self.vm: + raise ValueError("The VM has not been created yet") self.concurrent_runs += 1 self.runs_done_event.clear() try: diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py index 070870014..36b09186a 100644 --- a/vm_supervisor/run.py +++ b/vm_supervisor/run.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, Any +from typing import Dict, Any, Optional import msgpack from aiohttp import web @@ -44,7 +44,7 @@ async def run_code_on_request(vm_hash: VmHash, path: str, request: web.Request) Execute the code corresponding to the 'code id' in the path. """ - execution: VmExecution = await pool.get_running_vm(vm_hash=vm_hash) + execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash) if not execution: message, original_message = await load_updated_message(vm_hash) @@ -71,6 +71,9 @@ async def run_code_on_request(vm_hash: VmHash, path: str, request: web.Request) pool.forget_vm(vm_hash=vm_hash) raise HTTPInternalServerError(reason="Error during runtime initialisation") + if not execution.vm: + raise ValueError("The VM has not been created") + logger.debug(f"Using vm={execution.vm.vm_id}") scope: Dict = await build_asgi_scope(path, request) @@ -134,7 +137,7 @@ async def run_code_on_event(vm_hash: VmHash, event, pubsub: PubSub): """ try: - execution: VmExecution = await pool.get_running_vm(vm_hash=vm_hash) + execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash) except Exception as error: logger.exception(error) raise @@ -164,6 +167,9 @@ async def run_code_on_event(vm_hash: VmHash, event, pubsub: PubSub): pool.forget_vm(vm_hash=vm_hash) raise HTTPInternalServerError(reason="Error during runtime initialisation") + if not execution.vm: + raise ValueError("The VM has not been created") + logger.debug(f"Using vm={execution.vm.vm_id}") scope: Dict = await build_event_scope(event) diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py index 4aaa05774..dbf6b00b3 100644 --- a/vm_supervisor/tasks.py +++ b/vm_supervisor/tasks.py @@ -14,6 +14,7 @@ from aleph_message.models import BaseMessage, ProgramMessage from .conf import settings from .messages import load_updated_message +from .models import VmHash from .pubsub import PubSub from .reactor import Reactor @@ -93,7 +94,7 @@ async def start_watch_for_messages_task(app: web.Application): # Register an hardcoded initial program # TODO: Register all programs with subscriptions sample_message, _ = await load_updated_message( - ref="cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e") + ref=VmHash("cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e")) assert sample_message.content.on.message, sample_message reactor.register(sample_message) diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py index c4fd1b986..9f11078ef 100644 --- a/vm_supervisor/views.py +++ b/vm_supervisor/views.py @@ -25,7 +25,7 @@ def run_code_from_path(request: web.Request) -> Awaitable[web.Response]: path = request.match_info["suffix"] path = path if path.startswith("/") else f"/{path}" - message_ref: VmHash = request.match_info["ref"] + message_ref = VmHash(request.match_info["ref"]) return run_code_on_request(message_ref, path, request) @@ -44,16 +44,16 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: message_ref_base32 = request.host.split(".")[0] if settings.FAKE_DATA_PROGRAM: - message_ref = "fake-hash" + message_ref = VmHash("fake-hash") else: try: - message_ref = b32_to_b16(message_ref_base32).decode() + message_ref = VmHash(b32_to_b16(message_ref_base32).decode()) logger.debug( f"Using base32 message id from hostname to obtain '{message_ref}" ) except binascii.Error: try: - message_ref = await get_ref_from_dns(domain=f"_aleph-id.{request.host}") + message_ref = VmHash(await get_ref_from_dns(domain=f"_aleph-id.{request.host}")) logger.debug(f"Using DNS TXT record to obtain '{message_ref}'") except aiodns.error.DNSError: raise HTTPNotFound(reason="Invalid message reference") diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index bf29a4c95..fa88b2a6d 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -206,7 +206,7 @@ class AlephFirecrackerVM: enable_console: bool enable_networking: bool hardware_resources: MachineResources - fvm: MicroVM = None + fvm: Optional[MicroVM] = None guest_api_process: Optional[Process] = None def __init__( @@ -418,6 +418,8 @@ async def run_code( self, scope: dict = None, ): + if not self.fvm: + raise ValueError("MicroVM must be created first") logger.debug("running code") scope = scope or {} reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path) From db901dda8262306165ff4ab359627efba806e6ad Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 13 Dec 2021 16:13:33 +0100 Subject: [PATCH 208/990] Cleanup: Use black --- guest_api/__main__.py | 96 +++++++++++++------------ vm_supervisor/__main__.py | 2 + vm_supervisor/conf.py | 28 ++++++-- vm_supervisor/reactor.py | 9 ++- vm_supervisor/run.py | 24 ++++--- vm_supervisor/supervisor.py | 3 +- vm_supervisor/tasks.py | 31 +++++--- vm_supervisor/views.py | 6 +- vm_supervisor/vm/firecracker_microvm.py | 20 ++++-- 9 files changed, 137 insertions(+), 82 deletions(-) diff --git a/guest_api/__main__.py b/guest_api/__main__.py index cd442b650..40c828390 100644 --- a/guest_api/__main__.py +++ b/guest_api/__main__.py @@ -15,17 +15,17 @@ async def proxy(request: web.Request): - tail: str = request.match_info.get('tail') or "" - path: str = tail.lstrip('/') + tail: str = request.match_info.get("tail") or "" + path: str = tail.lstrip("/") query_string = request.rel_url.query_string url = f"{ALEPH_API_SERVER}/{path}?{query_string}" async with aiohttp.ClientSession() as session: async with session.request(method=request.method, url=url) as response: data = await response.read() - return web.Response(body=data, - status=response.status, - content_type=response.content_type) + return web.Response( + body=data, status=response.status, content_type=response.content_type + ) async def repost(request: web.Request): @@ -49,9 +49,9 @@ async def repost(request: web.Request): async with aiohttp.ClientSession() as session: async with session.post(url=url, json=new_data) as response: data = await response.read() - return web.Response(body=data, - status=response.status, - content_type=response.content_type) + return web.Response( + body=data, status=response.status, content_type=response.content_type + ) # async def decrypt_secret(request: web.Request): @@ -65,13 +65,13 @@ async def properties(request: web.Request): async with aiohttp.ClientSession() as session: async with session.get(url=url) as response: data = await response.read() - return web.Response(body=data, - status=response.status, - content_type=response.content_type) + return web.Response( + body=data, status=response.status, content_type=response.content_type + ) async def sign(request: web.Request): - vm_hash = request.app['meta_vm_hash'] + vm_hash = request.app["meta_vm_hash"] message = await request.json() # Ensure that the hash of the VM is used as sending address @@ -85,15 +85,17 @@ async def sign(request: web.Request): async with aiohttp.ClientSession() as session: async with session.post(url=url, json=message) as response: signed_message = await response.read() - return web.Response(body=signed_message, - status=response.status, - content_type=response.content_type) + return web.Response( + body=signed_message, + status=response.status, + content_type=response.content_type, + ) async def get_from_cache(request: web.Request): - prefix: str = request.app['meta_vm_hash'] - key: Optional[str] = request.match_info.get('key') - if not (key and re.match(r'^\w+$', key)): + prefix: str = request.app["meta_vm_hash"] + key: Optional[str] = request.match_info.get("key") + if not (key and re.match(r"^\w+$", key)): return web.HTTPBadRequest(text="Invalid key") redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost") @@ -105,22 +107,23 @@ async def get_from_cache(request: web.Request): async def put_in_cache(request: web.Request): - prefix: str = request.app['meta_vm_hash'] - key: Optional[str] = request.match_info.get('key') - if not (key and re.match(r'^\w+$', key)): + prefix: str = request.app["meta_vm_hash"] + key: Optional[str] = request.match_info.get("key") + if not (key and re.match(r"^\w+$", key)): return web.HTTPBadRequest(text="Invalid key") value: bytes = await request.read() redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost") - return web.json_response(await redis.set(f"{prefix}:{key}", value, - expire=CACHE_EXPIRES_AFTER)) + return web.json_response( + await redis.set(f"{prefix}:{key}", value, expire=CACHE_EXPIRES_AFTER) + ) async def delete_from_cache(request: web.Request): - prefix: str = request.app['meta_vm_hash'] - key: Optional[str] = request.match_info.get('key') - if not (key and re.match(r'^\w+$', key)): + prefix: str = request.app["meta_vm_hash"] + key: Optional[str] = request.match_info.get("key") + if not (key and re.match(r"^\w+$", key)): return web.HTTPBadRequest(text="Invalid key") redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost") @@ -129,42 +132,41 @@ async def delete_from_cache(request: web.Request): async def list_keys_from_cache(request: web.Request): - prefix: str = request.app['meta_vm_hash'] - pattern: str = request.rel_url.query.get('pattern', '*') - if not re.match(r'^[\w?*^\-]+$', pattern): + prefix: str = request.app["meta_vm_hash"] + pattern: str = request.rel_url.query.get("pattern", "*") + if not re.match(r"^[\w?*^\-]+$", pattern): return web.HTTPBadRequest(text="Invalid key") redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost") result = await redis.keys(f"{prefix}:{pattern}") - keys = [ - key.decode()[len(prefix)+1:] - for key in result - ] + keys = [key.decode()[len(prefix) + 1 :] for key in result] return web.json_response(keys) def run_guest_api(unix_socket_path, vm_hash: Optional[str] = None): app = web.Application() - app['meta_vm_hash'] = vm_hash or '_' + app["meta_vm_hash"] = vm_hash or "_" - app.router.add_route(method='GET', path='/properties', handler=properties) - app.router.add_route(method='POST', path='/sign', handler=sign) + app.router.add_route(method="GET", path="/properties", handler=properties) + app.router.add_route(method="POST", path="/sign", handler=sign) - app.router.add_route(method='GET', path='/cache/', handler=list_keys_from_cache) - app.router.add_route(method='GET', path='/cache/{key:.*}', handler=get_from_cache) - app.router.add_route(method='PUT', path='/cache/{key:.*}', handler=put_in_cache) - app.router.add_route(method='DELETE', path='/cache/{key:.*}', handler=delete_from_cache) + app.router.add_route(method="GET", path="/cache/", handler=list_keys_from_cache) + app.router.add_route(method="GET", path="/cache/{key:.*}", handler=get_from_cache) + app.router.add_route(method="PUT", path="/cache/{key:.*}", handler=put_in_cache) + app.router.add_route( + method="DELETE", path="/cache/{key:.*}", handler=delete_from_cache + ) - app.router.add_route(method='GET', path='/{tail:.*}', handler=proxy) - app.router.add_route(method='HEAD', path='/{tail:.*}', handler=proxy) - app.router.add_route(method='OPTIONS', path='/{tail:.*}', handler=proxy) + app.router.add_route(method="GET", path="/{tail:.*}", handler=proxy) + app.router.add_route(method="HEAD", path="/{tail:.*}", handler=proxy) + app.router.add_route(method="OPTIONS", path="/{tail:.*}", handler=proxy) - app.router.add_route(method='POST', path='/api/v0/ipfs/pubsub/pub', handler=repost) - app.router.add_route(method='POST', path='/api/v0/p2p/pubsub/pub', handler=repost) + app.router.add_route(method="POST", path="/api/v0/ipfs/pubsub/pub", handler=repost) + app.router.add_route(method="POST", path="/api/v0/p2p/pubsub/pub", handler=repost) # web.run_app(app=app, port=9000) web.run_app(app=app, path=unix_socket_path) -if __name__ == '__main__': - run_guest_api("/tmp/guest-api", vm_hash='vm') +if __name__ == "__main__": + run_guest_api("/tmp/guest-api", vm_hash="vm") diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py index 5173565e2..98547fcef 100644 --- a/vm_supervisor/__main__.py +++ b/vm_supervisor/__main__.py @@ -114,6 +114,7 @@ async def benchmark(runs: int): settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM FakeRequest: Request + class FakeRequest: # type: ignore[no-redef] headers: Dict[str, str] raw_headers: List[Tuple[bytes, bytes]] @@ -134,6 +135,7 @@ class FakeRequest: # type: ignore[no-redef] async def fake_read() -> bytes: return b"" + fake_request.read = fake_read logger.info("--- Start benchmark ---") diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index a67c43e6d..aea82a528 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -97,14 +97,26 @@ class Settings(BaseSettings): MAX_DATA_ARCHIVE_SIZE = 10_000_000 # 10 MB FAKE_DATA_PROGRAM: Optional[FilePath] = None - BENCHMARK_FAKE_DATA_PROGRAM: FilePath = FilePath(abspath(join(__file__, "../../examples/example_fastapi_2"))) + BENCHMARK_FAKE_DATA_PROGRAM: FilePath = FilePath( + abspath(join(__file__, "../../examples/example_fastapi_2")) + ) - FAKE_DATA_MESSAGE: FilePath = FilePath(abspath(join(__file__, "../../examples/message_from_aleph.json"))) - FAKE_DATA_DATA: Optional[FilePath] = FilePath(abspath(join(__file__, "../../examples/data/"))) - FAKE_DATA_RUNTIME: FilePath = FilePath(abspath(join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs"))) - FAKE_DATA_VOLUME: Optional[FilePath] = FilePath(abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs"))) + FAKE_DATA_MESSAGE: FilePath = FilePath( + abspath(join(__file__, "../../examples/message_from_aleph.json")) + ) + FAKE_DATA_DATA: Optional[FilePath] = FilePath( + abspath(join(__file__, "../../examples/data/")) + ) + FAKE_DATA_RUNTIME: FilePath = FilePath( + abspath(join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs")) + ) + FAKE_DATA_VOLUME: Optional[FilePath] = FilePath( + abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs")) + ) - CHECK_FASTAPI_VM_ID: str = "bbd7f6e2ce72104a334f22e4b29f0ebeb96af3179167521788bce80754f3c58a" + CHECK_FASTAPI_VM_ID: str = ( + "bbd7f6e2ce72104a334f22e4b29f0ebeb96af3179167521788bce80754f3c58a" + ) def update(self, **kwargs): for key, value in kwargs.items(): @@ -126,7 +138,9 @@ def check(self): assert exists(f"/sys/class/net/{self.NETWORK_INTERFACE}") if self.FAKE_DATA_PROGRAM: - assert isdir(self.FAKE_DATA_PROGRAM), "Local fake program directory is missing" + assert isdir( + self.FAKE_DATA_PROGRAM + ), "Local fake program directory is missing" assert isfile(self.FAKE_DATA_MESSAGE), "Local fake message is missing" assert isdir(self.FAKE_DATA_DATA), "Local fake data directory is missing" assert isfile(self.FAKE_DATA_RUNTIME), "Local runtime .squashfs build is missing" diff --git a/vm_supervisor/reactor.py b/vm_supervisor/reactor.py index 814c2ee60..db604a2d6 100644 --- a/vm_supervisor/reactor.py +++ b/vm_supervisor/reactor.py @@ -49,8 +49,10 @@ async def trigger(self, message: Message): for listener in self.listeners: if not listener.content.on.message: - logger.warning("Program with no subscription was registered in reactor listeners: " - f"{listener.item_hash}") + logger.warning( + "Program with no subscription was registered in reactor listeners: " + f"{listener.item_hash}" + ) continue for subscription in listener.content.on.message: @@ -72,4 +74,5 @@ def register(self, message: ProgramMessage): else: logger.debug( "Program with no subscription cannot be registered in reactor listeners: " - f"{message.item_hash}") + f"{message.item_hash}" + ) diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py index 36b09186a..cae8a101f 100644 --- a/vm_supervisor/run.py +++ b/vm_supervisor/run.py @@ -12,7 +12,11 @@ from .models import VmHash, VmExecution from .pool import VmPool from .pubsub import PubSub -from .vm.firecracker_microvm import ResourceDownloadError, VmSetupError, FileTooLargeError +from .vm.firecracker_microvm import ( + ResourceDownloadError, + VmSetupError, + FileTooLargeError, +) logger = logging.getLogger(__name__) @@ -39,7 +43,9 @@ async def build_event_scope(event) -> Dict[str, Any]: } -async def run_code_on_request(vm_hash: VmHash, path: str, request: web.Request) -> web.Response: +async def run_code_on_request( + vm_hash: VmHash, path: str, request: web.Request +) -> web.Response: """ Execute the code corresponding to the 'code id' in the path. """ @@ -108,11 +114,13 @@ async def run_code_on_request(vm_hash: VmHash, path: str, request: web.Request) if header in headers: del headers[header] - headers.update({ - "Aleph-Program-ItemHash": execution.vm_hash, - "Aleph-Program-Code-Ref": execution.program.code.ref, - # "Aleph-Compute-Vm-Id": str(execution.vm.vm_id), - }) + headers.update( + { + "Aleph-Program-ItemHash": execution.vm_hash, + "Aleph-Program-Code-Ref": execution.program.code.ref, + # "Aleph-Compute-Vm-Id": str(execution.vm.vm_id), + } + ) return web.Response( status=result["headers"]["status"], @@ -196,7 +204,7 @@ async def run_code_on_event(vm_hash: VmHash, event, pubsub: PubSub): ) logger.info(f"Result: {result['body']}") - return result['body'] + return result["body"] except UnpackValueError as error: logger.exception(error) diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py index c54eb9a28..1566b12c2 100644 --- a/vm_supervisor/supervisor.py +++ b/vm_supervisor/supervisor.py @@ -50,5 +50,4 @@ def run(): app.on_startup.append(start_watch_for_messages_task) app.on_cleanup.append(stop_watch_for_messages_task) - web.run_app(app, host=settings.SUPERVISOR_HOST, - port=settings.SUPERVISOR_PORT) + web.run_app(app, host=settings.SUPERVISOR_HOST, port=settings.SUPERVISOR_PORT) diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py index dbf6b00b3..35ec271d5 100644 --- a/vm_supervisor/tasks.py +++ b/vm_supervisor/tasks.py @@ -20,9 +20,12 @@ logger = logging.getLogger(__name__) -Value = TypeVar('Value') +Value = TypeVar("Value") -async def retry_generator(generator: AsyncIterable[Value], max_seconds: int = 8) -> AsyncIterable[Value]: + +async def retry_generator( + generator: AsyncIterable[Value], max_seconds: int = 8 +) -> AsyncIterable[Value]: retry_delay = 0.1 while True: async for value in generator: @@ -45,18 +48,29 @@ async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]: # Patch data format to match HTTP GET format data["_id"] = {"$oid": data["_id"]} except json.JSONDecodeError: - logger.error(f"Invalid JSON from websocket subscription {msg.data}", exc_info=True) + logger.error( + f"Invalid JSON from websocket subscription {msg.data}", + exc_info=True, + ) try: yield Message(**data) except pydantic.error_wrappers.ValidationError as error: - logger.error(f"Invalid Aleph message: \n {error.json()}\n {error.raw_errors}", - exc_info=True) + logger.error( + f"Invalid Aleph message: \n {error.json()}\n {error.raw_errors}", + exc_info=True, + ) continue except KeyError: - logger.exception(f"Invalid Aleph message could not be parsed '{data}'", exc_info=True) + logger.exception( + f"Invalid Aleph message could not be parsed '{data}'", + exc_info=True, + ) continue except Exception: - logger.exception(f"Unknown error when parsing Aleph message {data}", exc_info=True) + logger.exception( + f"Unknown error when parsing Aleph message {data}", + exc_info=True, + ) continue elif msg.type == aiohttp.WSMsgType.ERROR: break @@ -94,7 +108,8 @@ async def start_watch_for_messages_task(app: web.Application): # Register an hardcoded initial program # TODO: Register all programs with subscriptions sample_message, _ = await load_updated_message( - ref=VmHash("cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e")) + ref=VmHash("cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e") + ) assert sample_message.content.on.message, sample_message reactor.register(sample_message) diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py index 9f11078ef..f0570beee 100644 --- a/vm_supervisor/views.py +++ b/vm_supervisor/views.py @@ -53,7 +53,9 @@ async def run_code_from_hostname(request: web.Request) -> web.Response: ) except binascii.Error: try: - message_ref = VmHash(await get_ref_from_dns(domain=f"_aleph-id.{request.host}")) + message_ref = VmHash( + await get_ref_from_dns(domain=f"_aleph-id.{request.host}") + ) logger.debug(f"Using DNS TXT record to obtain '{message_ref}'") except aiodns.error.DNSError: raise HTTPNotFound(reason="Invalid message reference") @@ -108,4 +110,4 @@ async def status_check_fastapi(request: web.Request): "cache": await status.check_cache(session), "persistent_storage": await status.check_persistent_storage(session), } - return web.json_response(result, status=200 if all(result.values()) else 503) \ No newline at end of file + return web.json_response(result, status=200 if all(result.values()) else 503) diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py index fa88b2a6d..d069ab317 100644 --- a/vm_supervisor/vm/firecracker_microvm.py +++ b/vm_supervisor/vm/firecracker_microvm.py @@ -49,7 +49,6 @@ class FileTooLargeError(Exception): pass - class ResourceDownloadError(ClientResponseError): """An error occurred while downloading a VM resource file""" @@ -297,7 +296,8 @@ async def setup(self): NetworkInterface( iface_id="eth0", host_dev_name=await fvm.create_network_interface( - interface=settings.NETWORK_INTERFACE), + interface=settings.NETWORK_INTERFACE + ), ) ] if self.enable_networking @@ -330,7 +330,11 @@ async def start(self): async def configure(self): """Configure the VM by sending configuration info to it's init""" - if self.resources.data_path and os.path.getsize(self.resources.data_path) > settings.MAX_DATA_ARCHIVE_SIZE: + if ( + self.resources.data_path + and os.path.getsize(self.resources.data_path) + > settings.MAX_DATA_ARCHIVE_SIZE + ): raise FileTooLargeError(f"Data file too large to pass as an inline zip") input_data: bytes = load_file_content(self.resources.data_path) @@ -353,8 +357,14 @@ async def configure(self): for index, volume in enumerate(self.resources.volumes) ] else: - if self.resources.data_path and os.path.getsize(self.resources.code_path) > settings.MAX_PROGRAM_ARCHIVE_SIZE: - raise FileTooLargeError(f"Program file too large to pass as an inline zip") + if ( + self.resources.data_path + and os.path.getsize(self.resources.code_path) + > settings.MAX_PROGRAM_ARCHIVE_SIZE + ): + raise FileTooLargeError( + f"Program file too large to pass as an inline zip" + ) code: bytes = load_file_content(self.resources.code_path) volumes = [ From b89dcb81417455f157456dcc0b8ac21a9d53680e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 13 Dec 2021 16:37:36 +0100 Subject: [PATCH 209/990] Fix: Confirmation messages always displayed an exception with traceback --- vm_supervisor/tasks.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py index 35ec271d5..f8ff3c112 100644 --- a/vm_supervisor/tasks.py +++ b/vm_supervisor/tasks.py @@ -52,6 +52,18 @@ async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]: f"Invalid JSON from websocket subscription {msg.data}", exc_info=True, ) + + # Chain confirmation messages are published in the WS subscription + # but do not contain the fields "item_type" or "content, hence they + # are not valid Messages. + if "item_type" not in data: + assert "content" not in data + assert "confirmation" in data + logger.info( + f"Ignoring confirmation message '{data['item_hash']}'" + ) + continue + try: yield Message(**data) except pydantic.error_wrappers.ValidationError as error: From 67900311c9440ce86f5d7bdce6d52423d74e3417 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 14 Dec 2021 10:17:25 +0100 Subject: [PATCH 210/990] Refactor: Rename `example_fastapi_2` -> `example_fastapi` --- docker/vm_supervisor-dev.dockerfile | 2 +- examples/Makefile | 8 ++++---- examples/README.md | 10 +++++----- .../{example_fastapi_2 => example_fastapi}/main.py | 2 +- tutorials/TESTING.md | 4 ++-- vm_connector/tests/test_message.json | 2 +- vm_supervisor/conf.py | 2 +- vm_supervisor/status.py | 4 ++-- 8 files changed, 17 insertions(+), 17 deletions(-) rename examples/{example_fastapi_2 => example_fastapi}/main.py (99%) diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile index e2f96ad2f..ab8f5210c 100644 --- a/docker/vm_supervisor-dev.dockerfile +++ b/docker/vm_supervisor-dev.dockerfile @@ -36,7 +36,7 @@ ENV ALEPH_VM_FAKE_DATA True ENV ALEPH_VM_SUPERVISOR_HOST "0.0.0.0" # Make it easy to enter this command from a shell script -RUN echo "python3 -m vm_supervisor --print-settings --very-verbose --system-logs --profile -f ./examples/example_fastapi_2" >> /root/.bash_history +RUN echo "python3 -m vm_supervisor --print-settings --very-verbose --system-logs --profile -f ./examples/example_fastapi" >> /root/.bash_history RUN mkdir /opt/aleph-vm/ COPY ./vm_supervisor /opt/aleph-vm/vm_supervisor diff --git a/examples/Makefile b/examples/Makefile index d477eaed4..b148ee309 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -1,11 +1,11 @@ -all: example_fastapi_2.zip data.tgz +all: example_fastapi.zip data.tgz clean: - rm example_fastapi_2.zip + rm example_fastapi.zip rm data.tgz -example_fastapi_2.zip: - zip -r example_fastapi_2.zip example_fastapi_2 +example_fastapi.zip: + zip -r example_fastapi.zip example_fastapi data.tgz: tar -cvzf data.tgz data diff --git a/examples/README.md b/examples/README.md index 04b0a620f..e76f3986d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -6,11 +6,11 @@ on [FastAPI](https://fastapi.tiangolo.com/). ## Initial setup Let's start by creating a package for our app: -Create a directory named `example_fastapi_2` +Create a directory named `example_fastapi` and an empty file named `__init__.py` file within the directory. ``` -example_fastapi_2/ -example_fastapi_2/__init__.py +example_fastapi/ +example_fastapi/__init__.py ``` The copy the example from the FastAPI tutorial in `__init__.py`: @@ -41,7 +41,7 @@ Uvicorn is used to run ASGI compatible web applications, such as the `app` web application from the example above. You need to specify it the name of the Python module to use and the name of the app: ```shell -uvicorn example_fastapi_2:app --reload +uvicorn example_fastapi:app --reload ``` Then open the app in a web browser on http://localhost:8000 @@ -59,7 +59,7 @@ To achieve this, we need to follow the following steps: ### 1. Create a zip archive containing the app ```shell -zip -r example_fastapi_2.zip example_fastapi_2 +zip -r example_fastapi.zip example_fastapi ``` ### 2. Store the zip archive on Aleph diff --git a/examples/example_fastapi_2/main.py b/examples/example_fastapi/main.py similarity index 99% rename from examples/example_fastapi_2/main.py rename to examples/example_fastapi/main.py index 15945135c..9464837e1 100644 --- a/examples/example_fastapi_2/main.py +++ b/examples/example_fastapi/main.py @@ -35,7 +35,7 @@ async def index(): else: opt_venv = [] return { - "Example": "example_fastapi_2", + "Example": "example_fastapi", "endpoints": ["/environ", "/messages", "/internet", "/post_a_message", "/state/increment", "/wait-for/{delay}"], "files_in_volumes": { diff --git a/tutorials/TESTING.md b/tutorials/TESTING.md index 894bbfcd1..a222fd042 100644 --- a/tutorials/TESTING.md +++ b/tutorials/TESTING.md @@ -46,7 +46,7 @@ bash ./docker/run_vm_supervisor.sh Within the container, run the supervisor with fake data: ```shell -python3 -m vm_supervisor --print-settings --very-verbose --system-logs --fake-data-program ./examples/example_fastapi_2 +python3 -m vm_supervisor --print-settings --very-verbose --system-logs --fake-data-program ./examples/example_fastapi ``` > ℹ️ The command is in your .bash_history, press key up to skip typing it. @@ -60,6 +60,6 @@ See [../vm_supervisor/README.md](../vm_supervisor/README.md) to install the syst ### 2.b. Run the supervisor with fake data: ```shell -python3 -m vm_supervisor --print-settings --very-verbose --system-logs --fake-data-program ./examples/example_fastapi_2 +python3 -m vm_supervisor --print-settings --very-verbose --system-logs --fake-data-program ./examples/example_fastapi ``` diff --git a/vm_connector/tests/test_message.json b/vm_connector/tests/test_message.json index ca24dfd16..b3079f130 100644 --- a/vm_connector/tests/test_message.json +++ b/vm_connector/tests/test_message.json @@ -4,7 +4,7 @@ "content": { "code": { "encoding": "zip", - "entrypoint": "example_fastapi_2:app", + "entrypoint": "example_fastapi:app", "ref": "7eb2eca2378ea8855336ed76c8b26219f1cb90234d04441de9cf8cb1c649d003", "latest_amend": true }, diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index aea82a528..5a898ba61 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -98,7 +98,7 @@ class Settings(BaseSettings): FAKE_DATA_PROGRAM: Optional[FilePath] = None BENCHMARK_FAKE_DATA_PROGRAM: FilePath = FilePath( - abspath(join(__file__, "../../examples/example_fastapi_2")) + abspath(join(__file__, "../../examples/example_fastapi")) ) FAKE_DATA_MESSAGE: FilePath = FilePath( diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py index 3afed3128..4fab06160 100644 --- a/vm_supervisor/status.py +++ b/vm_supervisor/status.py @@ -1,5 +1,5 @@ """ -Used to check that the example_fastapi_2 program works as expected +Used to check that the example_fastapi program works as expected in a deployed supervisor. """ @@ -21,7 +21,7 @@ async def get_json_from_vm(session: ClientSession, suffix: str) -> Any: async def check_index(session: ClientSession) -> bool: result: Dict = await get_json_from_vm(session, "/") - assert result["Example"] == "example_fastapi_2" + assert result["Example"] == "example_fastapi" return True From a283a861870de76f9887d90927b3e14509be64b2 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 14 Dec 2021 10:40:46 +0100 Subject: [PATCH 211/990] Feature: Add Rust example program --- examples/example_http_rust/Dockerfile | 14 +++++++++++ examples/example_http_rust/Makefile | 18 ++++++++++++++ examples/example_http_rust/README.md | 33 ++++++++++++++++++++++++++ examples/example_http_rust/src/main.rs | 32 +++++++++++++++++++++++++ 4 files changed, 97 insertions(+) create mode 100644 examples/example_http_rust/Dockerfile create mode 100644 examples/example_http_rust/Makefile create mode 100644 examples/example_http_rust/README.md create mode 100644 examples/example_http_rust/src/main.rs diff --git a/examples/example_http_rust/Dockerfile b/examples/example_http_rust/Dockerfile new file mode 100644 index 000000000..7144e041e --- /dev/null +++ b/examples/example_http_rust/Dockerfile @@ -0,0 +1,14 @@ +FROM rust:bullseye + +RUN apt-get update && apt-get -y upgrade && apt-get install -y \ + libsecp256k1-dev \ + python3-pip \ + squashfs-tools \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install aleph-client + +WORKDIR /usr/src/example_http_rust +COPY . . + +RUN cargo install --path . diff --git a/examples/example_http_rust/Makefile b/examples/example_http_rust/Makefile new file mode 100644 index 000000000..0f82bdd02 --- /dev/null +++ b/examples/example_http_rust/Makefile @@ -0,0 +1,18 @@ + +podman-prepare: + podman build -t aleph-example-rust . + +podman-publish: + podman run --rm -ti aleph-example-rust make publish + +docker-prepare: + docker build -t aleph-example-rust . + +docker-publish: + docker run --rm -ti aleph-example-rust make publish + +publish: + cargo build --release + mkdir -p ./dist + cp target/release/example_http_rust ./dist/ + aleph program ./dist example_http_rust diff --git a/examples/example_http_rust/README.md b/examples/example_http_rust/README.md new file mode 100644 index 000000000..144aa189b --- /dev/null +++ b/examples/example_http_rust/README.md @@ -0,0 +1,33 @@ +# Aleph VM Rust Example + +A simple example program written in Rust that +can run in an Aleph VM. + +## About + +This example is a simple HTTP server listening on port 8080. +It does not depend on third-party libraries. + +Test it on http://localhost:8080. + +## Publish the program + +### Locally + +```shell +make publish +``` + +### Using Podman + +```shell +make podman-prepare +make podman-publish +``` + +### Using Docker + +```shell +make prepare +make publish +``` diff --git a/examples/example_http_rust/src/main.rs b/examples/example_http_rust/src/main.rs new file mode 100644 index 000000000..b354b6cff --- /dev/null +++ b/examples/example_http_rust/src/main.rs @@ -0,0 +1,32 @@ +use std::io::prelude::*; +use std::net::TcpListener; +use std::net::TcpStream; + +fn main() { + + let listener = TcpListener::bind("0.0.0.0:8080").unwrap(); + println!("Running on 0.0.0.0:8080"); + for stream in listener.incoming() { + let stream = stream.unwrap(); + handle_connection(stream); + } +} + + +fn handle_connection(mut stream: TcpStream) { + println!("handling connection"); + + const MSG: &str = "helloworld"; + let msg = MSG.as_bytes(); + + let response = format!("{:x?}", msg); + + let mut buffer = [0; 1024]; + + stream.read(&mut buffer).unwrap(); + + let response = format!("HTTP/1.1 200 OK\nContent-Type: text/plain\n\nOKIDOK\n{}", response); + + stream.write(response.as_bytes()).unwrap(); + stream.flush().unwrap(); +} From 11813b095a1640b300fbfbcda806753500dcbb01 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 14 Dec 2021 12:41:54 +0100 Subject: [PATCH 212/990] Feature: Add Node JS example program --- examples/example_http_js/.dockerignore | 3 +++ examples/example_http_js/Dockerfile | 15 ++++++++++++ examples/example_http_js/Makefile | 19 +++++++++++++++ examples/example_http_js/README.md | 33 ++++++++++++++++++++++++++ examples/example_http_js/package.json | 1 + examples/example_http_js/src/run.sh | 3 +++ examples/example_http_js/src/server.js | 7 ++++++ 7 files changed, 81 insertions(+) create mode 100644 examples/example_http_js/.dockerignore create mode 100644 examples/example_http_js/Dockerfile create mode 100644 examples/example_http_js/Makefile create mode 100644 examples/example_http_js/README.md create mode 100644 examples/example_http_js/package.json create mode 100755 examples/example_http_js/src/run.sh create mode 100644 examples/example_http_js/src/server.js diff --git a/examples/example_http_js/.dockerignore b/examples/example_http_js/.dockerignore new file mode 100644 index 000000000..c6d35cc33 --- /dev/null +++ b/examples/example_http_js/.dockerignore @@ -0,0 +1,3 @@ +*.zip +*.squashfs +*.key diff --git a/examples/example_http_js/Dockerfile b/examples/example_http_js/Dockerfile new file mode 100644 index 000000000..4916b01b1 --- /dev/null +++ b/examples/example_http_js/Dockerfile @@ -0,0 +1,15 @@ +FROM node:16-bullseye + +RUN apt-get update && apt-get -y upgrade && apt-get install -y \ + libsecp256k1-dev \ + squashfs-tools \ + python3-pip \ + git \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install aleph-client + +WORKDIR /usr/src/example_http_js +COPY . . + +RUN npm i diff --git a/examples/example_http_js/Makefile b/examples/example_http_js/Makefile new file mode 100644 index 000000000..3b2ac89e8 --- /dev/null +++ b/examples/example_http_js/Makefile @@ -0,0 +1,19 @@ + +podman-prepare: + podman build -t aleph-example-js . + +podman-publish: + podman run --rm -ti aleph-example-js make publish + +podman-client: + podman rmi aleph-example-js + +docker-prepare: + docker build -t aleph-example-js . + +docker-publish: + docker run --rm -ti aleph-example-js make publish + +publish: + chmod +x ./src/run.sh + aleph program ./src "run.sh" diff --git a/examples/example_http_js/README.md b/examples/example_http_js/README.md new file mode 100644 index 000000000..144aa189b --- /dev/null +++ b/examples/example_http_js/README.md @@ -0,0 +1,33 @@ +# Aleph VM Rust Example + +A simple example program written in Rust that +can run in an Aleph VM. + +## About + +This example is a simple HTTP server listening on port 8080. +It does not depend on third-party libraries. + +Test it on http://localhost:8080. + +## Publish the program + +### Locally + +```shell +make publish +``` + +### Using Podman + +```shell +make podman-prepare +make podman-publish +``` + +### Using Docker + +```shell +make prepare +make publish +``` diff --git a/examples/example_http_js/package.json b/examples/example_http_js/package.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/examples/example_http_js/package.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/examples/example_http_js/src/run.sh b/examples/example_http_js/src/run.sh new file mode 100755 index 000000000..d56a2caf4 --- /dev/null +++ b/examples/example_http_js/src/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh +cd /opt/code +node /opt/code/server.js diff --git a/examples/example_http_js/src/server.js b/examples/example_http_js/src/server.js new file mode 100644 index 000000000..2dc7189a5 --- /dev/null +++ b/examples/example_http_js/src/server.js @@ -0,0 +1,7 @@ +const http = require('http'); +const requestListener = function (req, res) { + res.writeHead(200, {'Content-Type': 'text/plain'}); + res.end('Hello, World!'); +} +const server = http.createServer(requestListener); +server.listen(8080); From 4b8ae193d41f56987b7859a5730275ed464a1873 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 14 Dec 2021 14:10:25 +0100 Subject: [PATCH 213/990] Fix: Init timeout for HTTP executables was reached regularly on slow servers --- runtimes/aleph-alpine-3.13-python/init1.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 483b80832..7d42c70e8 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -309,7 +309,8 @@ async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[by headers = None body = None - async with aiohttp.ClientSession(conn_timeout=.05) as session: + timeout = aiohttp.ClientTimeout(total=5) + async with aiohttp.ClientSession(timeout=timeout) as session: while not body: try: tries += 1 From 31c01c09945d1e58bb3f4dc4cdf718bdf8ceb422 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 14 Dec 2021 16:06:28 +0100 Subject: [PATCH 214/990] Feature: Build Debian package within a container using Make --- .dockerignore | 1 + packaging/Dockerfile | 15 +++++++++++++++ packaging/Makefile | 15 +++++++++++++-- packaging/aleph-vm/DEBIAN/control | 2 +- 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 packaging/Dockerfile diff --git a/.dockerignore b/.dockerignore index c9e440ce8..584a8c6b5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,4 @@ **/*.key **/data.tgz /pydantic/ +**/target diff --git a/packaging/Dockerfile b/packaging/Dockerfile new file mode 100644 index 000000000..6a804780e --- /dev/null +++ b/packaging/Dockerfile @@ -0,0 +1,15 @@ +FROM debian:bullseye + +RUN apt-get update && apt-get -y upgrade && apt-get install -y \ + make \ + git \ + curl \ + sudo \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt +COPY ../vm_supervisor ./vm_supervisor +COPY ../guest_api ./guest_api +COPY ../firecracker ./firecracker +COPY ../packaging ./packaging +COPY ../kernels ./kernels diff --git a/packaging/Makefile b/packaging/Makefile index 9247d0951..60f1f50de 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -20,7 +20,7 @@ debian-package-resources: firecracker-bins vmlinux firecracker-bins: target-dir build-dir mkdir -p ./build/firecracker-release # Download latest release - curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v0.24.2/firecracker-v0.24.2-x86_64.tgz | tar -xz --directory ./build/firecracker-release + curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v0.25.2/firecracker-v0.25.2-x86_64.tgz | tar -xz --directory ./build/firecracker-release # Copy binaries: cp ./build/firecracker-release/firecracker-v* ./target/firecracker cp ./build/firecracker-release/jailer-v* ./target/jailer @@ -29,7 +29,8 @@ firecracker-bins: target-dir build-dir vmlinux: #curl -fsSL -o ./target/vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin - cp ../kernels/vmlinux.bin ./target/vmlinux.bin + curl -fsSL -o ./target/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/vmlinux.bin + #cp ../kernels/vmlinux.bin ./target/vmlinux.bin build-dir: mkdir -p target @@ -40,3 +41,13 @@ target-dir: clean: rm -fr ./target/* rm -fr ./build/* + +all-podman: + cd .. && podman build -t aleph-vm-packaging -f ./packaging/Dockerfile . + mkdir -p ./target + podman run --rm -ti \ + -w /opt/packaging \ + -v ./target:/opt/packaging/target \ + localhost/aleph-vm-packaging:latest \ + make + file target/aleph-vm.deb diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control index ceb118ee3..3855df848 100644 --- a/packaging/aleph-vm/DEBIAN/control +++ b/packaging/aleph-vm/DEBIAN/control @@ -1,5 +1,5 @@ Package: aleph-vm -Version: 0.1.0-8 +Version: 0.1.1-0 Architecture: all Maintainer: Aleph.im Description: Aleph.im VM execution engine From 72047ac6a7052bda66f0fe33874a6856934d61b4 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 14 Dec 2021 16:13:26 +0100 Subject: [PATCH 215/990] Feature: Build Debian package on GitHub Actions --- .github/workflows/build-deb-package.yml | 15 +++++++++++++++ packaging/Makefile | 6 +++--- 2 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/build-deb-package.yml diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml new file mode 100644 index 000000000..0c4ead63f --- /dev/null +++ b/.github/workflows/build-deb-package.yml @@ -0,0 +1,15 @@ +name: "Build Packages" +on: + push + +jobs: + build_deb: + name: "Build Debian Package" + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - run: | + cd packaging && make all-podman diff --git a/packaging/Makefile b/packaging/Makefile index 60f1f50de..5942db367 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -22,8 +22,8 @@ firecracker-bins: target-dir build-dir # Download latest release curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v0.25.2/firecracker-v0.25.2-x86_64.tgz | tar -xz --directory ./build/firecracker-release # Copy binaries: - cp ./build/firecracker-release/firecracker-v* ./target/firecracker - cp ./build/firecracker-release/jailer-v* ./target/jailer + cp ./build/firecracker-release/release-v*/firecracker-v* ./target/firecracker + cp ./build/firecracker-release/release-v*/jailer-v* ./target/jailer chmod +x ./target/firecracker chmod +x ./target/jailer @@ -43,7 +43,7 @@ clean: rm -fr ./build/* all-podman: - cd .. && podman build -t aleph-vm-packaging -f ./packaging/Dockerfile . + cd .. && podman build -t localhost/aleph-vm-packaging:latest -f ./packaging/Dockerfile . mkdir -p ./target podman run --rm -ti \ -w /opt/packaging \ From 3a15922ee59fa4b7a175214809d455c04b30ca2f Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 28 Dec 2021 10:32:57 +0100 Subject: [PATCH 216/990] Doc: Update README quick install to version 0.1.6 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cfe3dafed..49916247c 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ for production purposes: sudo apt update sudo apt install -y docker.io sudo docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha -wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/aleph-vm-0.1.0.deb -sudo apt install ./aleph-vm-0.1.0.deb +wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.6/aleph-vm-0.1.6.deb +sudo apt install ./aleph-vm-0.1.6.deb ``` ### Reverse Proxy From 3a69a3eb115f33e461f801635cbf57f6517978b9 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 28 Dec 2021 10:35:01 +0100 Subject: [PATCH 217/990] Doc: Update recommended OS to Debian 11 --- vm_supervisor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md index 1ced9bee8..5bf2bcfa1 100644 --- a/vm_supervisor/README.md +++ b/vm_supervisor/README.md @@ -28,7 +28,7 @@ Intel Skylake, Intel Cascade Lake, AMD Zen2 and ARM64 Neoverse N1. ### Operating System -These instructions have been tested on Debian 10 Buster, and should work on recent versions +These instructions have been tested on Debian 11 Bullseye, and should work on recent versions of Ubuntu as well (20.04+). ### Hosting providers From c67674c1b14d0a320a34ff683781b2dbd82c5e8b Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 14 Dec 2021 12:46:11 +0100 Subject: [PATCH 218/990] Fix: Runtime code directory was inconsistent depending on Zip/Squashfs/plain A program uploaded in a Zip archive or as plain was extracted in `/opt`, while a Squashfs archive was mounted on `/opt/code`. This solves the issue by always extracting code in `/opt/code`. --- runtimes/aleph-alpine-3.13-python/init1.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py index 7d42c70e8..3519bff32 100644 --- a/runtimes/aleph-alpine-3.13-python/init1.py +++ b/runtimes/aleph-alpine-3.13-python/init1.py @@ -199,14 +199,16 @@ def setup_code_executable(code: bytes, encoding: Encoding, entrypoint: str) -> s elif encoding == Encoding.zip: open("/opt/archive.zip", "wb").write(code) logger.debug("Run unzip") - os.system("unzip /opt/archive.zip -d /opt") - path = f"/opt/{entrypoint}" + os.makedirs("/opt/code", exist_ok=True) + os.system("unzip /opt/archive.zip -d /opt/code") + path = f"/opt/code/{entrypoint}" if not os.path.isfile(path): - os.system("find /opt") + os.system("find /opt/code") raise FileNotFoundError(f"No such file: {path}") os.system(f"chmod +x {path}") elif encoding == Encoding.plain: - path = f"/opt/executable {entrypoint}" + os.makedirs("/opt/code", exist_ok=True) + path = f"/opt/code/executable {entrypoint}" open(path, "wb").write(code) os.system(f"chmod +x {path}") else: From 7aadcac46ca2d641b9e2886090e6bea3163d7e6d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 12 Jan 2022 15:37:37 +0100 Subject: [PATCH 219/990] Doc: Fix README, creating Caddyfile requires sudo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 49916247c..911990b00 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo sudo apt update sudo apt install caddy -cat >/etc/caddy/Caddyfile </etc/caddy/Caddyfile < Date: Wed, 12 Jan 2022 17:52:00 +0100 Subject: [PATCH 220/990] Fix: Deb Package missed fake data for diagnostic This prevented the supervisor from launching --- .dockerignore | 2 +- .github/workflows/build-deb-package.yml | 4 ++++ examples/volumes/build_squashfs.sh | 0 packaging/Dockerfile | 6 ++++++ packaging/Makefile | 12 ++++++++++++ packaging/aleph-vm/DEBIAN/control | 2 +- packaging/aleph-vm/DEBIAN/postinst | 3 --- runtimes/aleph-debian-11-python/create_disk_image.sh | 0 8 files changed, 24 insertions(+), 5 deletions(-) mode change 100644 => 100755 examples/volumes/build_squashfs.sh mode change 100644 => 100755 runtimes/aleph-debian-11-python/create_disk_image.sh diff --git a/.dockerignore b/.dockerignore index 584a8c6b5..6937fd0ff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,7 +4,7 @@ **/rootfs/ **/*.sqlite3 -**/*.squashfs +# **/*.squashfs **/*.bin **/*.ext4 **/*.zip diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml index 0c4ead63f..e1aee2d02 100644 --- a/.github/workflows/build-deb-package.yml +++ b/.github/workflows/build-deb-package.yml @@ -12,4 +12,8 @@ jobs: uses: actions/checkout@v2 - run: | + sudo apt update + sudo apt install -y debootstrap + cd runtimes/aleph-debian-11-python && sudo ./create_disk_image.sh && cd ../.. + cd examples/volumes && sudo ./build_squashfs.sh && cd ../.. cd packaging && make all-podman diff --git a/examples/volumes/build_squashfs.sh b/examples/volumes/build_squashfs.sh old mode 100644 new mode 100755 diff --git a/packaging/Dockerfile b/packaging/Dockerfile index 6a804780e..b1ba77531 100644 --- a/packaging/Dockerfile +++ b/packaging/Dockerfile @@ -5,6 +5,7 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \ git \ curl \ sudo \ + python3-pip \ && rm -rf /var/lib/apt/lists/* WORKDIR /opt @@ -13,3 +14,8 @@ COPY ../guest_api ./guest_api COPY ../firecracker ./firecracker COPY ../packaging ./packaging COPY ../kernels ./kernels + +COPY ../examples/ ./examples + +RUN mkdir -p ./runtimes/aleph-debian-11-python +COPY ../runtimes/aleph-debian-11-python/rootfs.squashfs ./runtimes/aleph-debian-11-python/rootfs.squashfs diff --git a/packaging/Makefile b/packaging/Makefile index 5942db367..1cb71c14a 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -10,6 +10,18 @@ debian-package-code: cp -r ../guest_api ./aleph-vm/opt/aleph-vm/ cp -r ../firecracker ./aleph-vm/opt/aleph-vm/ + # Fake data for diagnostic and benchmarks + mkdir -p ./aleph-vm/opt/aleph-vm/examples/ + cp -r ../examples/example_fastapi ./aleph-vm/opt/aleph-vm/examples/example_fastapi + cp ../examples/message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/message_from_aleph.json + cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data + mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes + cp ../examples/volumes/volume-venv.squashfs ./aleph-vm/opt/aleph-vm/examples/volumes/volume-venv.squashfs + mkdir -p ./aleph-vm/opt/aleph-vm/runtimes/aleph-debian-11-python/ + cp ../runtimes/aleph-debian-11-python/rootfs.squashfs ./aleph-vm/opt/aleph-vm/runtimes/aleph-debian-11-python/rootfs.squashfs + + pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message>=0.1.18' + debian-package-resources: firecracker-bins vmlinux rm -fr ./aleph-vm/opt/firecracker mkdir -p ./aleph-vm/opt/firecracker diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control index 3855df848..8e7d14111 100644 --- a/packaging/aleph-vm/DEBIAN/control +++ b/packaging/aleph-vm/DEBIAN/control @@ -1,5 +1,5 @@ Package: aleph-vm -Version: 0.1.1-0 +Version: 0.1.6-1 Architecture: all Maintainer: Aleph.im Description: Aleph.im VM execution engine diff --git a/packaging/aleph-vm/DEBIAN/postinst b/packaging/aleph-vm/DEBIAN/postinst index 1bac33236..4bf426224 100755 --- a/packaging/aleph-vm/DEBIAN/postinst +++ b/packaging/aleph-vm/DEBIAN/postinst @@ -5,9 +5,6 @@ if ! id -u jailman > /dev/null 2>&1; then useradd jailman fi -# No suggestions since only pure Python dependencies will be required: -pip3 install 'aleph-message==0.1.12' - mkdir -p /srv/jailer systemctl daemon-reload diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh old mode 100644 new mode 100755 From e037c04cb6c91f0d2d3202b63304a53632c8cdb8 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 13 Jan 2022 21:26:31 +0100 Subject: [PATCH 221/990] GitHub: Add issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 38 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..dd84ea782 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Smartphone (please complete the following information):** + - Device: [e.g. iPhone6] + - OS: [e.g. iOS8.1] + - Browser [e.g. stock browser, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..bbcbbe7d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From ff9473f87bf22c8a8d1f0fb6538cb14baa68b52a Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 13 Jan 2022 17:49:27 +0100 Subject: [PATCH 222/990] Feature: Build Ubuntu 20.04 Debian packages In addition to Debian packages. --- .github/workflows/build-deb-package.yml | 3 ++- packaging/Makefile | 16 ++++++++++++-- .../{Dockerfile => debian-11.dockerfile} | 0 packaging/ubuntu-20.04.dockerfile | 21 +++++++++++++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) rename packaging/{Dockerfile => debian-11.dockerfile} (100%) create mode 100644 packaging/ubuntu-20.04.dockerfile diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml index e1aee2d02..81bc7bf26 100644 --- a/.github/workflows/build-deb-package.yml +++ b/.github/workflows/build-deb-package.yml @@ -16,4 +16,5 @@ jobs: sudo apt install -y debootstrap cd runtimes/aleph-debian-11-python && sudo ./create_disk_image.sh && cd ../.. cd examples/volumes && sudo ./build_squashfs.sh && cd ../.. - cd packaging && make all-podman + cd packaging && make all-podman-debian-11 && cd .. + cd packaging && make all-podman-ubuntu-2004 && cd .. diff --git a/packaging/Makefile b/packaging/Makefile index 1cb71c14a..571668cda 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -54,8 +54,8 @@ clean: rm -fr ./target/* rm -fr ./build/* -all-podman: - cd .. && podman build -t localhost/aleph-vm-packaging:latest -f ./packaging/Dockerfile . +all-podman-debian-11: + cd .. && podman build -t localhost/aleph-vm-packaging:latest -f ./packaging/debian-11.dockerfile . mkdir -p ./target podman run --rm -ti \ -w /opt/packaging \ @@ -63,3 +63,15 @@ all-podman: localhost/aleph-vm-packaging:latest \ make file target/aleph-vm.deb + mv target/aleph-vm.deb target/aleph-vm.debian-11.deb + +all-podman-ubuntu-2004: + cd .. && podman build -t localhost/aleph-vm-packaging:latest -f ./packaging/ubuntu-20.04.dockerfile . + mkdir -p ./target + podman run --rm -ti \ + -w /opt/packaging \ + -v ./target:/opt/packaging/target \ + localhost/aleph-vm-packaging:latest \ + make + file target/aleph-vm.deb + mv target/aleph-vm.deb target/aleph-vm.ubuntu-20.04.deb diff --git a/packaging/Dockerfile b/packaging/debian-11.dockerfile similarity index 100% rename from packaging/Dockerfile rename to packaging/debian-11.dockerfile diff --git a/packaging/ubuntu-20.04.dockerfile b/packaging/ubuntu-20.04.dockerfile new file mode 100644 index 000000000..90e209557 --- /dev/null +++ b/packaging/ubuntu-20.04.dockerfile @@ -0,0 +1,21 @@ +FROM ubuntu:20.04 + +RUN apt-get update && apt-get -y upgrade && apt-get install -y \ + make \ + git \ + curl \ + sudo \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt +COPY ../vm_supervisor ./vm_supervisor +COPY ../guest_api ./guest_api +COPY ../firecracker ./firecracker +COPY ../packaging ./packaging +COPY ../kernels ./kernels + +COPY ../examples/ ./examples + +RUN mkdir -p ./runtimes/aleph-debian-11-python +COPY ../runtimes/aleph-debian-11-python/rootfs.squashfs ./runtimes/aleph-debian-11-python/rootfs.squashfs From 64772b82bb381b295a2667b20b737737952c6d46 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 13 Jan 2022 20:52:49 +0100 Subject: [PATCH 223/990] Feature: Add Debian repository management for Bullseye and Focal --- .gitignore | 3 +++ packaging/Makefile | 12 ++++++++++++ packaging/aleph-vm/DEBIAN/control | 2 ++ packaging/repositories/bullseye/conf/distributions | 13 +++++++++++++ packaging/repositories/focal/conf/distributions | 13 +++++++++++++ 5 files changed, 43 insertions(+) create mode 100644 packaging/repositories/bullseye/conf/distributions create mode 100644 packaging/repositories/focal/conf/distributions diff --git a/.gitignore b/.gitignore index 72f7e3b2c..49fefe952 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ node_modules /runtimes/aleph-debian-11-python/rootfs/ /packaging/aleph-vm/opt/ /packaging/target/ +/packaging/repositories/*/db/ +/packaging/repositories/*/dists/ +/packaging/repositories/*/pool/ diff --git a/packaging/Makefile b/packaging/Makefile index 571668cda..703203313 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -75,3 +75,15 @@ all-podman-ubuntu-2004: make file target/aleph-vm.deb mv target/aleph-vm.deb target/aleph-vm.ubuntu-20.04.deb + +# run on host in order to sign with GPG +repository-bullseye: + cd ./repositories/bullseye && reprepro -Vb . includedeb bullseye ../../target/aleph-vm.debian-11.deb && cd .. + +# run on host in order to sign with GPG +repository-focal: + cd ./repositories/focal && reprepro -Vb . includedeb focal ../../target/aleph-vm.ubuntu-20.04.deb && cd .. + +repositories: repository-bullseye repository-focal + +all-podman: all-podman-debian-11 all-podman-ubuntu-2004 repositories diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control index 8e7d14111..167e3e5a7 100644 --- a/packaging/aleph-vm/DEBIAN/control +++ b/packaging/aleph-vm/DEBIAN/control @@ -4,3 +4,5 @@ Architecture: all Maintainer: Aleph.im Description: Aleph.im VM execution engine Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap +Section: aleph-im +Priority: Extra diff --git a/packaging/repositories/bullseye/conf/distributions b/packaging/repositories/bullseye/conf/distributions new file mode 100644 index 000000000..a1d0ecfd6 --- /dev/null +++ b/packaging/repositories/bullseye/conf/distributions @@ -0,0 +1,13 @@ +Origin: Aleph-IM +Label: aleph-im +Suite: stable +Codename: bullseye +Version: 3.0 +Architectures: amd64 source +Components: contrib +#UDebComponents: main +Description: Aleph-im packages +SignWith: yes +#DebOverride: override +#UDebOverride: override +#DscOverride: srcoverride diff --git a/packaging/repositories/focal/conf/distributions b/packaging/repositories/focal/conf/distributions new file mode 100644 index 000000000..577ba950d --- /dev/null +++ b/packaging/repositories/focal/conf/distributions @@ -0,0 +1,13 @@ +Origin: Aleph-IM +Label: aleph-im +Suite: stable +Codename: focal +Version: 3.0 +Architectures: amd64 source +Components: contrib +#UDebComponents: main +Description: Aleph-im packages +SignWith: yes +#DebOverride: override +#UDebOverride: override +#DscOverride: srcoverride From e44ccd5b9d0a9dabe47fac986f647d35931faba8 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 13 Jan 2022 23:00:59 +0100 Subject: [PATCH 224/990] Update .deb URL in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 911990b00..a8826ff7a 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ for production purposes: sudo apt update sudo apt install -y docker.io sudo docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha -wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.6/aleph-vm-0.1.6.deb +wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.6-1/aleph-vm-0.1.6-1.deb sudo apt install ./aleph-vm-0.1.6.deb ``` From b8c448b68cf5557faebff1027dc1567b6ed5e29d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Thu, 13 Jan 2022 23:01:22 +0100 Subject: [PATCH 225/990] Fix: Typo in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a8826ff7a..d6353fc6f 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ sudo apt update sudo apt install -y docker.io sudo docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.6-1/aleph-vm-0.1.6-1.deb -sudo apt install ./aleph-vm-0.1.6.deb +sudo apt install ./aleph-vm-0.1.6-1.deb ``` ### Reverse Proxy From 0c7c94722d8ea5c9db4edd4edd8cebe83bcf6363 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 14 Jan 2022 00:17:01 +0100 Subject: [PATCH 226/990] Update README.md with network configuration --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index d6353fc6f..c14ab40cf 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,17 @@ wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.6-1/aleph-vm-0.1 sudo apt install ./aleph-vm-0.1.6-1.deb ``` +### Configuration + +You can update the configuration in `/etc/aleph-vm/supervisor.env`. + +On Ubuntu, the default network interface is not `eth0` and you will want to configure the default interface in the form of: +``` +ALEPH_VM_NETWORK_INTERFACE=enp0s1 +``` + +You can find all available options in [./vm_supervisor/conf.py](./vm_supervisor/conf.py). + ### Reverse Proxy We document how to use Caddy as a reverse proxy since it does automatic HTTPS certificates. From 142df69b438642783d7d24e92711e89f21655afa Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Fri, 14 Jan 2022 13:28:44 +0100 Subject: [PATCH 227/990] Feature: Add index page with diagnostic --- .../aleph-vm/etc/aleph-vm/supervisor.env | 1 + vm_supervisor/conf.py | 12 +- vm_supervisor/status.py | 86 ++++++---- vm_supervisor/templates/index.html | 152 ++++++++++++++++++ vm_supervisor/views.py | 19 ++- 5 files changed, 233 insertions(+), 37 deletions(-) create mode 100644 vm_supervisor/templates/index.html diff --git a/packaging/aleph-vm/etc/aleph-vm/supervisor.env b/packaging/aleph-vm/etc/aleph-vm/supervisor.env index d23371e08..d4fd45c2b 100644 --- a/packaging/aleph-vm/etc/aleph-vm/supervisor.env +++ b/packaging/aleph-vm/etc/aleph-vm/supervisor.env @@ -1,2 +1,3 @@ ALEPH_VM_PRINT_SYSTEM_LOGS=True ALEPH_VM_USE_JAILER=True +ALEPH_VM_DOMAIN_NAME=vm.example.org \ No newline at end of file diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py index 5a898ba61..e819ec05c 100644 --- a/vm_supervisor/conf.py +++ b/vm_supervisor/conf.py @@ -7,7 +7,7 @@ from typing import NewType, Optional, List from firecracker.models import FilePath -from pydantic import BaseSettings +from pydantic import BaseSettings, Field logger = logging.getLogger(__name__) @@ -61,6 +61,12 @@ class Settings(BaseSettings): SUPERVISOR_HOST: str = "127.0.0.1" SUPERVISOR_PORT: int = 4020 + # Public domain name + DOMAIN_NAME: Optional[str] = Field( + default="localhost", + description="Default public domain name", + ) + START_ID_INDEX: int = 4 PREALLOC_VM_COUNT: int = 0 REUSE_TIMEOUT: float = 60 * 60.0 @@ -135,7 +141,9 @@ def check(self): "http://" ) or self.CONNECTOR_URL.startswith("https://") if self.ALLOW_VM_NETWORKING: - assert exists(f"/sys/class/net/{self.NETWORK_INTERFACE}") + assert exists( + f"/sys/class/net/{self.NETWORK_INTERFACE}" + ), f"Network interface {self.NETWORK_INTERFACE} does not exist" if self.FAKE_DATA_PROGRAM: assert isdir( diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py index 4fab06160..c2d9a0c08 100644 --- a/vm_supervisor/status.py +++ b/vm_supervisor/status.py @@ -5,7 +5,7 @@ from typing import Dict, Any, List -from aiohttp import ClientSession +from aiohttp import ClientSession, ClientResponseError from vm_supervisor.conf import settings @@ -20,51 +20,69 @@ async def get_json_from_vm(session: ClientSession, suffix: str) -> Any: async def check_index(session: ClientSession) -> bool: - result: Dict = await get_json_from_vm(session, "/") - assert result["Example"] == "example_fastapi" - return True + try: + result: Dict = await get_json_from_vm(session, "/") + assert result["Example"] == "example_fastapi" + return True + except ClientResponseError: + return False async def check_environ(session: ClientSession) -> bool: - result: Dict = await get_json_from_vm(session, "/environ") - assert "ALEPH_API_HOST" in result - assert "ALEPH_API_UNIX_SOCKET" in result - assert "ALEPH_REMOTE_CRYPTO_HOST" in result - assert "ALEPH_REMOTE_CRYPTO_UNIX_SOCKET" in result - assert "ALEPH_ADDRESS_TO_USE" in result - return True + try: + result: Dict = await get_json_from_vm(session, "/environ") + assert "ALEPH_API_HOST" in result + assert "ALEPH_API_UNIX_SOCKET" in result + assert "ALEPH_REMOTE_CRYPTO_HOST" in result + assert "ALEPH_REMOTE_CRYPTO_UNIX_SOCKET" in result + assert "ALEPH_ADDRESS_TO_USE" in result + return True + except ClientResponseError: + return False async def check_messages(session: ClientSession) -> bool: - result: Dict = await get_json_from_vm(session, "/messages") - assert "Messages" in result - assert "messages" in result["Messages"] - assert "item_hash" in result["Messages"]["messages"][0] - return True + try: + result: Dict = await get_json_from_vm(session, "/messages") + assert "Messages" in result + assert "messages" in result["Messages"] + assert "item_hash" in result["Messages"]["messages"][0] + return True + except ClientResponseError: + return False async def check_internet(session: ClientSession) -> bool: - result: Dict = await get_json_from_vm(session, "/internet") - assert result["result"] == 200 - assert "Server" in result["headers"] - return True + try: + result: Dict = await get_json_from_vm(session, "/internet") + assert result["result"] == 200 + assert "Server" in result["headers"] + return True + except ClientResponseError: + return False async def check_cache(session: ClientSession) -> bool: - result1: bool = await get_json_from_vm(session, "/cache/set/a/42") - assert result1 == True - result2: int = await get_json_from_vm(session, "/cache/get/a") - assert result2 == "42" - keys: List[str] = await get_json_from_vm(session, "/cache/keys") - print("KEYS", keys) - assert "a" in keys - return True + try: + result1: bool = await get_json_from_vm(session, "/cache/set/a/42") + assert result1 == True + result2: int = await get_json_from_vm(session, "/cache/get/a") + assert result2 == "42" + keys: List[str] = await get_json_from_vm(session, "/cache/keys") + print("KEYS", keys) + assert "a" in keys + return True + except ClientResponseError: + return False async def check_persistent_storage(session: ClientSession) -> bool: - result: Dict = await get_json_from_vm(session, "/state/increment") - counter = result["counter"] - result_2: Dict = await get_json_from_vm(session, "/state/increment") - counter_2 = result_2["counter"] - assert counter_2 == counter + 1 - return True + try: + result: Dict = await get_json_from_vm(session, "/state/increment") + counter = result["counter"] + result_2: Dict = await get_json_from_vm(session, "/state/increment") + counter_2 = result_2["counter"] + assert counter_2 == counter + 1 + return True + except ClientResponseError: + return False diff --git a/vm_supervisor/templates/index.html b/vm_supervisor/templates/index.html new file mode 100644 index 000000000..920e0459b --- /dev/null +++ b/vm_supervisor/templates/index.html @@ -0,0 +1,152 @@ + + + + + Aleph.im Compute Node + + + +

    Aleph.im Compute Node

    + +
    +

    + This is an Aleph.im compute resource node. +

    +

    + It executes user programs stored on the Aleph network in Virtual Machines. +

    +

    + See the repository for more info. +

    + +
    + +
    + +
    +

    Multiaddr

    +

    + This node is exposed on the following addresses: +

    + + +
    + +
    + +
    +

    Diagnostic

    +

    + Virtualization is + + ... + + + + + + +

    + 
    
+    
    
+        Diagnostics checks |
+        Open diagnostic VM
+    
    
+
+
    
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index f0570beee..3f54de5c6 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -1,5 +1,7 @@
 import binascii
 import logging
+import os.path
+from string import Template
 from typing import Awaitable
 
 import aiodns
@@ -39,6 +41,10 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
     we expect the hash to be encoded in base32 instead of hexadecimal. Padding is added
     automatically.
     """
+    if request.host.split(':')[0] == settings.DOMAIN_NAME:
+        # Serve the index page
+        return await index(request=request)
+
     path = request.match_info["suffix"]
     path = path if path.startswith("/") else f"/{path}"
 
@@ -97,7 +103,18 @@ async def about_config(request: web.Request):
 
 async def index(request: web.Request):
     assert request.method == "GET"
-    return web.Response(text="Server: Aleph VM Supervisor")
+    path = os.path.join(os.path.dirname(__file__), 'templates/index.html')
+    with open(path, 'r') as template:
+        body = template.read()
+        s = Template(body)
+        body = s.substitute(
+            public_url=f'https://{settings.DOMAIN_NAME}/',
+            multiaddr_dns4=f'/dns4/{settings.DOMAIN_NAME}/tcp/443/https',
+            multiaddr_dns6=f'/dns6/{settings.DOMAIN_NAME}/tcp/443/https',
+            check_fastapi_vm_id=settings.CHECK_FASTAPI_VM_ID,
+        )
+    return web.Response(content_type="text/html",
+                        body=body)
 
 
 async def status_check_fastapi(request: web.Request):

From 1de4afdcf4703b8edb9beb0f9260a91021a085b7 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 14 Jan 2022 17:15:40 +0100
Subject: [PATCH 228/990] Fix: FastAPI example failed due to missing volume

---
 examples/message_from_aleph.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/message_from_aleph.json b/examples/message_from_aleph.json
index 217e5a96e..ae6b27489 100644
--- a/examples/message_from_aleph.json
+++ b/examples/message_from_aleph.json
@@ -59,8 +59,8 @@
             },
             {
                 "comment": "Working data persisted on the VM supervisor, not available on other nodes",
-                "mount": "/var/lib/sqlite",
-                "name": "database",
+                "mount": "/var/lib/example",
+                "name": "data",
                 "persistence": "host",
                 "size_mib": 5
             }

From 66e50a8d76de134b122a8a3f18eef793daea841a Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 14 Jan 2022 17:17:54 +0100
Subject: [PATCH 229/990] Internal: Mount vm_supervisor in container for faster
 development

---
 docker/run_vm_supervisor.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/run_vm_supervisor.sh b/docker/run_vm_supervisor.sh
index 8ec235009..a25f5c984 100755
--- a/docker/run_vm_supervisor.sh
+++ b/docker/run_vm_supervisor.sh
@@ -13,6 +13,7 @@ $DOCKER_COMMAND build -t alephim/vm-supervisor-dev -f docker/vm_supervisor-dev.d
 $DOCKER_COMMAND run -ti --rm \
   -v "$(pwd)/runtimes/aleph-debian-11-python/rootfs.squashfs:/opt/aleph-vm/runtimes/aleph-debian-11-python/rootfs.squashfs:ro" \
   -v "$(pwd)/examples/volumes/volume-venv.squashfs:/opt/aleph-vm/examples/volumes/volume-venv.squashfs:ro" \
+  -v "$(pwd)/vm_supervisor:/opt/aleph-vm/vm_supervisor:ro" \
   --device /dev/kvm \
   -p 4020:4020 \
   alephim/vm-supervisor-dev

From 09afeb26c553ed14439bb7368b9335a54f099190 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 14 Jan 2022 17:18:16 +0100
Subject: [PATCH 230/990] Fix: Freeze dependency versions in runtime

---
 runtimes/aleph-debian-11-python/create_disk_image.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
index f30005f43..7fbb6209a 100755
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-11-python/create_disk_image.sh
@@ -25,10 +25,10 @@ apt-get install -y --no-install-recommends --no-install-suggests \
   nodejs npm \
   build-essential python3-dev
 
-pip3 install fastapi django
+pip3 install 'fastapi~=0.71.0'
 
 echo "Pip installing aleph-client"
-pip3 install 'aleph-client>=0.4.4' 'coincurve==15.0.0'
+pip3 install 'aleph-client>=0.4.6' 'coincurve==15.0.0'
 
 # Compile all Python bytecode
 python3 -m compileall -f /usr/local/lib/python3.9

From 766f1243321e63a1e47d9acc0a5f69c1314b5404 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 17 Jan 2022 13:17:03 +0100
Subject: [PATCH 231/990] Fix: Old check VM was incompatible due to renaming

---
 vm_supervisor/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index e819ec05c..bd541232c 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -121,7 +121,7 @@ class Settings(BaseSettings):
     )
 
     CHECK_FASTAPI_VM_ID: str = (
-        "bbd7f6e2ce72104a334f22e4b29f0ebeb96af3179167521788bce80754f3c58a"
+        "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
     )
 
     def update(self, **kwargs):

From 48d2f0331fe4ecd0eb92929080fa874ab13d2761 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 17 Jan 2022 13:17:42 +0100
Subject: [PATCH 232/990] Version 0.1.7

---
 packaging/aleph-vm/DEBIAN/control | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index 167e3e5a7..fd0c6e1f1 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -1,5 +1,5 @@
 Package: aleph-vm
-Version: 0.1.6-1
+Version: 0.1.7-0
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine

From 352209aa73b78603e7588a6346cf43b7a1201ce4 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 17 Jan 2022 13:46:54 +0100
Subject: [PATCH 233/990] Fix: Better cleanup on Debian package removal

---
 packaging/Makefile                                          | 1 +
 packaging/aleph-vm/DEBIAN/postrm                            | 6 ++++++
 packaging/aleph-vm/DEBIAN/prerm                             | 5 +++++
 .../aleph-vm/etc/systemd/system/aleph-vm-supervisor.service | 1 +
 4 files changed, 13 insertions(+)
 create mode 100755 packaging/aleph-vm/DEBIAN/postrm
 create mode 100755 packaging/aleph-vm/DEBIAN/prerm

diff --git a/packaging/Makefile b/packaging/Makefile
index 703203313..28211fe10 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -21,6 +21,7 @@ debian-package-code:
 	cp ../runtimes/aleph-debian-11-python/rootfs.squashfs ./aleph-vm/opt/aleph-vm/runtimes/aleph-debian-11-python/rootfs.squashfs
 
 	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message>=0.1.18'
+	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux
 	rm -fr ./aleph-vm/opt/firecracker
diff --git a/packaging/aleph-vm/DEBIAN/postrm b/packaging/aleph-vm/DEBIAN/postrm
new file mode 100755
index 000000000..0f62a0480
--- /dev/null
+++ b/packaging/aleph-vm/DEBIAN/postrm
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -euf -o pipefail
+
+rm -r /srv/jailer
+
+systemctl daemon-reload
diff --git a/packaging/aleph-vm/DEBIAN/prerm b/packaging/aleph-vm/DEBIAN/prerm
new file mode 100755
index 000000000..4e88fe7fc
--- /dev/null
+++ b/packaging/aleph-vm/DEBIAN/prerm
@@ -0,0 +1,5 @@
+#!/bin/bash
+set -euf -o pipefail
+
+systemctl disable aleph-vm-supervisor.service
+systemctl stop aleph-vm-supervisor.service
diff --git a/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service b/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
index 6f5395176..4d5c41929 100644
--- a/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
+++ b/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
@@ -7,6 +7,7 @@ User=0
 Group=0
 WorkingDirectory=/opt/aleph-vm
 Environment=PYTHONPATH=/opt/aleph-vm/:$PYTHONPATH
+Environment=PYTHONDONTWRITEBYTECODE="enabled"
 EnvironmentFile=/etc/aleph-vm/supervisor.env
 ExecStart=python3 -m vm_supervisor --print-settings --very-verbose
 Restart=always

From 88831369c2554c7490a8795b8fe1f027a9538afb Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 17 Jan 2022 16:37:15 +0100
Subject: [PATCH 234/990] Doc: Update README quick install

---
 README.md | 60 +++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 43 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index c14ab40cf..8184ef5ce 100644
--- a/README.md
+++ b/README.md
@@ -1,46 +1,75 @@
 # Aleph-VM
 
-> Note: This project is still early prototyping.
-
 The Aleph-VM project allows you to run programs on [Aleph.im](https://aleph.im/).
 
-Programs can currently be written in Python using ASGI compatible frameworks (
+Aleph-VM is optimized to run programs on demand in a "function-as-as-service",
+as a response to HTTP requests.
+
+Programs can be written in any language as long as they can run a web server.
+They benefit from running in their own, customizable Linux virtual environment.
+
+Writing programs in Python using ASGI compatible frameworks (
 [FastAPI](https://github.com/tiangolo/fastapi), 
 [Django](https://docs.djangoproject.com/en/3.0/topics/async/),
-...) and respond to HTTP requests. 
-
-Alternatively, programs written in any language can listen to HTTP requests on port 8080.
+...) allows developers to use advanced functionnalities not yet available for other languages.
 
-### 1. Writing Aleph-VM programs
+## 1. Creating and running an Aleph Program 
 
 Have a look at [tutorials/README.md](tutorials/README.md) for a tutorial on how to program VMs
 as a user.
 
 The rest of this document focuses on how to run an Aleph-VM node that hosts and executes the programs. 
 
-## 1. Quick install
+## 2. Installing Aleph-VM on a server
+
+### 0. Requirements
+
+- A [supported Linux server](./vm_supervisor/README.md#1-supported-platforms)
+- A public domain name from a trusted registar and domain. 
+
+In order to run an Aleph.im Compute Resource Node, you will also need the following resources:
+
+- CPU (2 options):
+  - Min. 8 cores / 16 threads, 3.0 ghz+ CPU (gaming CPU for fast boot-up of microVMs)
+  - Min. 12 core / 24 threads, 2.4ghz+ CPU (datacenter CPU for multiple concurrent loads)
+- RAM: 64GB
+- STORAGE: 1TB (Nvme SSD prefered, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
+- BANDWIDTH: Minimum of 500 MB/s
+
+You will need a public domain name with access to add TXT and wildcard records.
+
+This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
+
+### 1. Quick install
 
 To quickly install Aleph-VM on a [supported Linux system](./vm_supervisor/README.md#1-supported-platforms)
 for production purposes:
 
 ```shell
 sudo apt update
+sudo apt upgrade
 sudo apt install -y docker.io
 sudo docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
-wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.6-1/aleph-vm-0.1.6-1.deb
-sudo apt install ./aleph-vm-0.1.6-1.deb
+wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.7/aleph-vm.debian-0.1.7-0.deb
+sudo apt install .//aleph-vm.debian-0.1.7-0.deb
 ```
 
 ### Configuration
 
-You can update the configuration in `/etc/aleph-vm/supervisor.env`. 
+Update the configuration in `/etc/aleph-vm/supervisor.env`. 
+
+You will want to insert your domain name in the for of:
+```
+ALEPH_VM_DOMAIN_NAME=vm.example.org
+```
 
 On Ubuntu, the default network interface is not `eth0` and you will want to configure the default interface in the form of:
 ```
 ALEPH_VM_NETWORK_INTERFACE=enp0s1
 ```
+(don't forget to replace `enp0s1` with the name of your default network interface).
 
-You can find all available options in [./vm_supervisor/conf.py](./vm_supervisor/conf.py).
+You can find all available options in [./vm_supervisor/conf.py](./vm_supervisor/conf.py). Prefix them with `ALEPH_VM_`.
 
 ### Reverse Proxy
 
@@ -64,7 +93,7 @@ sudo cat >/etc/caddy/Caddyfile <
Date: Mon, 17 Jan 2022 21:36:04 +0100
Subject: [PATCH 235/990] Doc: Ubuntu requires DNS using resolvectl

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 8184ef5ce..9057b4cdb 100644
--- a/README.md
+++ b/README.md
@@ -63,9 +63,10 @@ You will want to insert your domain name in the for of:
 ALEPH_VM_DOMAIN_NAME=vm.example.org
 ```
 
-On Ubuntu, the default network interface is not `eth0` and you will want to configure the default interface in the form of:
+On Ubuntu, the default network interface is not `eth0` and you will want to configure the default interface. Due to the DNS being handled by `systemd-resolved` on Ubuntu, you should also configure the DNS to use `resolvectl`.
 ```
 ALEPH_VM_NETWORK_INTERFACE=enp0s1
+ALEPH_VM_DNS_RESOLUTION=resolvectl
 ```
 (don't forget to replace `enp0s1` with the name of your default network interface).
 

From 5e0ff116b062c24cdc9a28b180862855d3fb40f6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 18 Jan 2022 15:21:27 +0100
Subject: [PATCH 236/990] Doc: Update README.md

---
 README.md | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9057b4cdb..e1b94eace 100644
--- a/README.md
+++ b/README.md
@@ -50,8 +50,8 @@ sudo apt update
 sudo apt upgrade
 sudo apt install -y docker.io
 sudo docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
-wget https://github.com/aleph-im/aleph-vm/releases/download/0.1.7/aleph-vm.debian-0.1.7-0.deb
-sudo apt install .//aleph-vm.debian-0.1.7-0.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.7/aleph-vm.debian-0.1.7-0.deb
+sudo apt install /opt/aleph-vm.debian-0.1.7-0.deb
 ```
 
 ### Configuration
@@ -72,6 +72,11 @@ ALEPH_VM_DNS_RESOLUTION=resolvectl
 
 You can find all available options in [./vm_supervisor/conf.py](./vm_supervisor/conf.py). Prefix them with `ALEPH_VM_`.
 
+Finally, restart the service:
+```shell
+systemctl restart aleph-vm-supervisor
+```
+
 ### Reverse Proxy
 
 We document how to use Caddy as a reverse proxy since it does automatic HTTPS certificates.

From 362cd1c73fe602b32c2576be9d36fd98823d898b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 19 Jan 2022 20:29:27 +0100
Subject: [PATCH 237/990] Doc: Update README.md to discourage editing conf.py

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e1b94eace..774ec253d 100644
--- a/README.md
+++ b/README.md
@@ -70,7 +70,7 @@ ALEPH_VM_DNS_RESOLUTION=resolvectl
 ```
 (don't forget to replace `enp0s1` with the name of your default network interface).
 
-You can find all available options in [./vm_supervisor/conf.py](./vm_supervisor/conf.py). Prefix them with `ALEPH_VM_`.
+You can view all available options in [./vm_supervisor/conf.py](./vm_supervisor/conf.py). Do not edit that file. Instead, add the settings you want to change in `/etc/aleph-vm/supervisor.env` with the prefix `ALEPH_VM_`, below or in replacement from the existing ones.
 
 Finally, restart the service:
 ```shell

From 470de9e9d129b26a276a15e0b2912164e92e93aa Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 19 Jan 2022 20:41:55 +0100
Subject: [PATCH 238/990] Doc: Remove advanced configuration, use diagnostic
 for check

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 774ec253d..6cdd83524 100644
--- a/README.md
+++ b/README.md
@@ -70,8 +70,6 @@ ALEPH_VM_DNS_RESOLUTION=resolvectl
 ```
 (don't forget to replace `enp0s1` with the name of your default network interface).
 
-You can view all available options in [./vm_supervisor/conf.py](./vm_supervisor/conf.py). Do not edit that file. Instead, add the settings you want to change in `/etc/aleph-vm/supervisor.env` with the prefix `ALEPH_VM_`, below or in replacement from the existing ones.
-
 Finally, restart the service:
 ```shell
 systemctl restart aleph-vm-supervisor
@@ -115,7 +113,9 @@ systemctl restart caddy
 
 ### Test
 
-https://vm.yourdomain.org/vm/17412050fa1c103c41f983fe305c1ce8c6a809040762cdc1614bc32a06a28a63/state/increment
+Open https://[YOUR DOMAIN] in a web browser, wait for diagnostic to complete and look for 
+
+> ![image](https://user-images.githubusercontent.com/404665/150202090-91a02536-4e04-4af2-967f-fe105d116e1f.png)
 
 ## 3. Architecture
 

From a6542c85ecb8c1904951093bcce62a227a1741ae Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 24 Jan 2022 16:34:43 +0100
Subject: [PATCH 239/990] Fix: Redis connections were accumulating

When a VM opens many connections to Redis (thousands) without restarting, the error "Too many open files" will start appearing in the logs.

This is due to new Redis connections being created but not closed, so they accumulate in memory.

Solution: Reuse a single Redis connection for all calls from the guest API.
---
 guest_api/__main__.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/guest_api/__main__.py b/guest_api/__main__.py
index 40c828390..ce1cc780d 100644
--- a/guest_api/__main__.py
+++ b/guest_api/__main__.py
@@ -12,6 +12,16 @@
 ALEPH_API_SERVER = "https://api2.aleph.im"
 ALEPH_VM_CONNECTOR = "http://localhost:4021"
 CACHE_EXPIRES_AFTER = 7 * 24 * 3600  # Seconds
+REDIS_ADDRESS = "redis://localhost"
+
+_redis: Optional[aioredis.Redis] = None
+
+
+async def get_redis(address: str = REDIS_ADDRESS) -> aioredis.Redis:
+    global _redis
+    if _redis is None:
+        _redis = await aioredis.create_redis(address=address)
+    return _redis
 
 
 async def proxy(request: web.Request):
@@ -98,7 +108,7 @@ async def get_from_cache(request: web.Request):
     if not (key and re.match(r"^\w+$", key)):
         return web.HTTPBadRequest(text="Invalid key")
 
-    redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost")
+    redis: aioredis.Redis = await get_redis()
     body = await redis.get(f"{prefix}:{key}")
     if body:
         return web.Response(body=body, status=200)
@@ -114,7 +124,7 @@ async def put_in_cache(request: web.Request):
 
     value: bytes = await request.read()
 
-    redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost")
+    redis: aioredis.Redis = await get_redis()
     return web.json_response(
         await redis.set(f"{prefix}:{key}", value, expire=CACHE_EXPIRES_AFTER)
     )
@@ -126,7 +136,7 @@ async def delete_from_cache(request: web.Request):
     if not (key and re.match(r"^\w+$", key)):
         return web.HTTPBadRequest(text="Invalid key")
 
-    redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost")
+    redis: aioredis.Redis = await get_redis()
     result = await redis.delete(f"{prefix}:{key}")
     return web.json_response(result)
 
@@ -137,7 +147,7 @@ async def list_keys_from_cache(request: web.Request):
     if not re.match(r"^[\w?*^\-]+$", pattern):
         return web.HTTPBadRequest(text="Invalid key")
 
-    redis: aioredis.Redis = await aioredis.create_redis(address="redis://localhost")
+    redis: aioredis.Redis = await get_redis()
     result = await redis.keys(f"{prefix}:{pattern}")
     keys = [key.decode()[len(prefix) + 1 :] for key in result]
     return web.json_response(keys)

From 8971680d9aba1c8701ab27a6a8bdbab2b4c995db Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 26 Jan 2022 17:25:41 +0100
Subject: [PATCH 240/990] Fix: A directory with the numerical id of the VM was
 not deleted

These directories were accumulating on disk.
---
 firecracker/microvm.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index f1c283112..17cfe12a3 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -72,9 +72,13 @@ class MicroVM:
     init_timeout: float
 
     @property
-    def jailer_path(self):
+    def namespace_path(self):
         firecracker_bin_name = os.path.basename(self.firecracker_bin_path)
-        return f"/srv/jailer/{firecracker_bin_name}/{self.vm_id}/root"
+        return f"/srv/jailer/{firecracker_bin_name}/{self.vm_id}"
+
+    @property
+    def jailer_path(self):
+        return os.path.join(self.namespace_path, "root")
 
     @property
     def socket_path(self):
@@ -399,7 +403,7 @@ async def teardown(self):
             )
 
         logger.debug("Removing files")
-        system(f"rm -fr {self.jailer_path}")
+        system(f"rm -fr {self.namespace_path}")
 
     def __del__(self):
         try:

From f385bfeb3e52ba116a17d97cb6702d9f80709611 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 26 Jan 2022 17:23:19 +0100
Subject: [PATCH 241/990] Feature: Admin could not distinguish guest_api among
 running processes

This adds the use of `setproctitle` to change the title of the process for easier diagnostic.
---
 guest_api/__main__.py             | 2 ++
 packaging/aleph-vm/DEBIAN/control | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/guest_api/__main__.py b/guest_api/__main__.py
index ce1cc780d..fec80dc4d 100644
--- a/guest_api/__main__.py
+++ b/guest_api/__main__.py
@@ -6,6 +6,7 @@
 import aiohttp
 from aiohttp import web
 import aioredis
+from setproctitle import setproctitle
 
 logger = logging.getLogger(__name__)
 
@@ -154,6 +155,7 @@ async def list_keys_from_cache(request: web.Request):
 
 
 def run_guest_api(unix_socket_path, vm_hash: Optional[str] = None):
+    setproctitle(f"aleph-vm guest_api on {unix_socket_path}")
     app = web.Application()
     app["meta_vm_hash"] = vm_hash or "_"
 
diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index fd0c6e1f1..325c5607f 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.7-0
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap
 Section: aleph-im
 Priority: Extra

From 9f48eb31da0e81e66c28c85cebaab79d2b514e12 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 26 Jan 2022 17:28:57 +0100
Subject: [PATCH 242/990] Internal: Package `firecracker` was not mounted in
 development container

---
 docker/run_vm_supervisor.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/run_vm_supervisor.sh b/docker/run_vm_supervisor.sh
index a25f5c984..22ca76db4 100755
--- a/docker/run_vm_supervisor.sh
+++ b/docker/run_vm_supervisor.sh
@@ -14,6 +14,7 @@ $DOCKER_COMMAND run -ti --rm \
   -v "$(pwd)/runtimes/aleph-debian-11-python/rootfs.squashfs:/opt/aleph-vm/runtimes/aleph-debian-11-python/rootfs.squashfs:ro" \
   -v "$(pwd)/examples/volumes/volume-venv.squashfs:/opt/aleph-vm/examples/volumes/volume-venv.squashfs:ro" \
   -v "$(pwd)/vm_supervisor:/opt/aleph-vm/vm_supervisor:ro" \
+  -v "$(pwd)/firecracker:/opt/aleph-vm/firecracker:ro" \
   --device /dev/kvm \
   -p 4020:4020 \
   alephim/vm-supervisor-dev

From 6cfbd537bf888121e2c185b044f293bf584c2c51 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 24 Jan 2022 13:05:55 +0100
Subject: [PATCH 243/990] Fix: VM were not properly shutdowm, persistent
 volumes could be corrupted

---
 firecracker/microvm.py                     | 31 +++++++-
 runtimes/aleph-alpine-3.13-python/init1.py | 84 +++++++++++++++++++---
 2 files changed, 103 insertions(+), 12 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 17cfe12a3..c7081ddc1 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -367,17 +367,46 @@ async def unix_client_connected(*_):
             logger.warning("Never received signal from init")
             raise MicroVMFailedInit()
 
+    async def shutdown(self):
+        logger.debug(f"Shutown vm={self.vm_id}")
+        reader, writer = await asyncio.open_unix_connection(path=self.vsock_path)
+        payload = b"halt"
+        writer.write(b"CONNECT 52\n" + payload)
+
+        await writer.drain()
+
+        ack: bytes = await reader.readline()
+        logger.debug(f"ack={ack.decode()}")
+
+        msg: bytes = await reader.readline()
+        logger.debug(f"msg={msg}")
+
+        msg2: bytes = await reader.readline()
+        logger.debug(f"msg2={msg2}")
+
+        if msg2 != b"STOPZ\n":
+            logger.error("Unexpected response from VM")
+
     async def stop(self):
         if self.proc:
+            logger.debug("Stopping firecracker process")
             try:
                 self.proc.terminate()
                 self.proc.kill()
             except ProcessLookupError:
-                pass
+                logger.debug(f"Firecracker process pid={self.proc.pid} not found")
             self.proc = None
+        else:
+            logger.debug("No firecracker process to stop")
 
     async def teardown(self):
         """Stop the VM, cleanup network interface and remove data directory."""
+        try:
+            await asyncio.wait_for(self.shutdown(), timeout=5)
+        except asyncio.TimeoutError:
+            logger.exception(f"Timeout during VM shutdown vm={self.vm_id}")
+        logger.debug("Waiting for one second for the process to shudown")
+        await asyncio.sleep(1)
         await self.stop()
 
         if self.stdout_task:
diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py
index 3519bff32..384e2845d 100644
--- a/runtimes/aleph-alpine-3.13-python/init1.py
+++ b/runtimes/aleph-alpine-3.13-python/init1.py
@@ -9,6 +9,7 @@
 
 logger.debug("Imports starting")
 
+import ctypes
 import asyncio
 import os
 import socket
@@ -42,6 +43,10 @@ class Interface(str, Enum):
     executable = "executable"
 
 
+class ShutdownException(Exception):
+    pass
+
+
 @dataclass
 class Volume:
     mount: str
@@ -329,13 +334,26 @@ async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[by
 
 
 async def process_instruction(
-        instruction: bytes, interface: Interface, application
+        instruction: bytes, interface: Interface, application: Union[ASGIApplication, subprocess.Popen]
 ) -> AsyncIterable[bytes]:
 
     if instruction == b"halt":
+        logger.info("Received halt command")
         system("sync")
+        logger.debug("Filesystems synced")
+        if isinstance(application, subprocess.Popen):
+            application.terminate()
+            logger.debug("Application terminated")
+            # application.communicate()
+        else:
+            # Close the cached session in aleph_client:
+            from aleph_client.asynchronous import get_fallback_session
+            session: aiohttp.ClientSession = get_fallback_session()
+            await session.close()
+            logger.debug("Aiohttp cached session closed")
         yield b"STOP\n"
-        sys.exit()
+        logger.debug("Supervisor informed of halt")
+        raise ShutdownException
     elif instruction.startswith(b"!"):
         # Execute shell commands in the form `!ls /`
         msg = instruction[1:].decode()
@@ -417,6 +435,14 @@ def setup_system(config: ConfigurationPayload):
     logger.debug("Setup finished")
 
 
+def umount_volumes(volumes: List[Volume]):
+    "Umount user related filesystems"
+    system("sync")
+    for volume in volumes:
+        logger.debug(f"Umounting /dev/{volume.device} on {volume.mount}")
+        system(f"umount {volume.mount}")
+
+
 async def main():
     client, addr = s.accept()
 
@@ -437,6 +463,11 @@ async def main():
         logger.exception("Program could not be started")
         raise
 
+    class ServerReference:
+        "Reference used to close the server from within `handle_instruction"
+        server: asyncio.AbstractServer
+    server_reference = ServerReference()
+
     async def handle_instruction(reader, writer):
         data = await reader.read(1000_1000)  # Max 1 Mo
 
@@ -445,23 +476,54 @@ async def handle_instruction(reader, writer):
             data_to_print = f"{data[:500]}..." if len(data) > 500 else data
             logger.debug(f"<<<\n\n{data_to_print}\n\n>>>")
 
-        async for result in process_instruction(instruction=data, interface=config.interface,
-                                                application=app):
-            writer.write(result)
+        try:
+            async for result in process_instruction(instruction=data, interface=config.interface,
+                                                    application=app):
+                writer.write(result)
+                await writer.drain()
+
+                logger.debug("Instruction processed")
+        except ShutdownException:
+            logger.info("Initiating shutdown")
+            writer.write(b"STOPZ\n")
             await writer.drain()
-
-        logger.debug("...DONE")
-        writer.close()
+            logger.debug("Shutdown confirmed to supervisor")
+            server_reference.server.close()
+            logger.debug("Supervisor socket server closed")
+        finally:
+            writer.close()
 
     server = await asyncio.start_server(handle_instruction, sock=s)
+    server_reference.server = server
 
     addr = server.sockets[0].getsockname()
     print(f'Serving on {addr}')
 
-    async with server:
-        await server.serve_forever()
-
+    try:
+        async with server:
+            await server.serve_forever()
+    except asyncio.CancelledError:
+        logger.debug("Server was properly cancelled")
+    finally:
+        logger.warning("System shutdown")
+        server.close()
+        logger.debug("Server closed")
+        umount_volumes(config.volumes)
+        logger.debug("User volumes unmounted")
 
 if __name__ == '__main__':
     logging.basicConfig(level=logging.DEBUG)
     asyncio.run(main())
+
+    logger.info("Unmounting system filesystems")
+    system("umount /dev/shm")
+    system("umount /dev/pts")
+    system("umount -a")
+
+    logger.info("Sending reboot syscall")
+    # Send reboot syscall, see man page
+    # https://man7.org/linux/man-pages/man2/reboot.2.html
+    libc = ctypes.CDLL(None)
+    libc.syscall(169, 0xfee1dead, 672274793, 0x4321fedc, None)
+    # The exit should not happen due to system halt.
+    sys.exit(0)

From 6a7e3e22556794eaf62ba4887035893a614f4e9c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 26 Jan 2022 19:48:01 +0100
Subject: [PATCH 244/990] Feature: Publish artifacts from GH actions

---
 .github/workflows/build-deb-package.yml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 81bc7bf26..1e06c9423 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -11,6 +11,11 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
+      - name: Unshallow
+        run: |
+          git fetch --prune --unshallow
+          git describe --tags
+
       - run: |
           sudo apt update
           sudo apt install -y debootstrap
@@ -18,3 +23,19 @@ jobs:
           cd examples/volumes && sudo ./build_squashfs.sh && cd ../..
           cd packaging && make all-podman-debian-11 && cd ..
           cd packaging && make all-podman-ubuntu-2004 && cd ..
+          ls packaging/target
+          
+      - uses: actions/upload-artifact@v2
+        with:
+          name: aleph-vm.debian-11.deb
+          path: packaging/target/aleph-vm.debian-11.deb
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: aleph-vm.ubuntu-20.04.deb
+          path: packaging/target/aleph-vm.ubuntu-20.04.deb
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: aleph-debian-11-python.squashfs
+          path: runtimes/aleph-debian-11-python/rootfs.squashfs

From 81794166e842418d9e066f112269572a09af5d15 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 26 Jan 2022 20:48:31 +0100
Subject: [PATCH 245/990] Fix: Runtime rootfs was making packages very heavy

Remove the runtime rootfs from Debian packages, but publish it in the job artifacts.
---
 .github/workflows/build-deb-package.yml | 21 ++++++++++++++++-----
 packaging/Makefile                      |  4 ----
 packaging/debian-11.dockerfile          |  3 ---
 packaging/ubuntu-20.04.dockerfile       |  3 ---
 vm_supervisor/conf.py                   |  8 ++++----
 5 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 1e06c9423..0c1ebd364 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -17,10 +17,6 @@ jobs:
           git describe --tags
 
       - run: |
-          sudo apt update
-          sudo apt install -y debootstrap
-          cd runtimes/aleph-debian-11-python && sudo ./create_disk_image.sh && cd ../..
-          cd examples/volumes && sudo ./build_squashfs.sh && cd ../..
           cd packaging && make all-podman-debian-11 && cd ..
           cd packaging && make all-podman-ubuntu-2004 && cd ..
           ls packaging/target
@@ -35,7 +31,22 @@ jobs:
           name: aleph-vm.ubuntu-20.04.deb
           path: packaging/target/aleph-vm.ubuntu-20.04.deb
 
+
+  build_rootfs:
+    name: "Build runtime aleph-debian-11-python"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - run: |
+          sudo apt update
+          sudo apt install -y debootstrap
+          cd runtimes/aleph-debian-11-python && sudo ./create_disk_image.sh && cd ../..
+          cd examples/volumes && sudo ./build_squashfs.sh && cd ../..
+
       - uses: actions/upload-artifact@v2
         with:
           name: aleph-debian-11-python.squashfs
-          path: runtimes/aleph-debian-11-python/rootfs.squashfs
+          path: runtimes/aleph-debian-11-python/rootfs.squashfs
\ No newline at end of file
diff --git a/packaging/Makefile b/packaging/Makefile
index 28211fe10..00da410c2 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -16,10 +16,6 @@ debian-package-code:
 	cp ../examples/message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	cp ../examples/volumes/volume-venv.squashfs ./aleph-vm/opt/aleph-vm/examples/volumes/volume-venv.squashfs
-	mkdir -p ./aleph-vm/opt/aleph-vm/runtimes/aleph-debian-11-python/
-	cp ../runtimes/aleph-debian-11-python/rootfs.squashfs ./aleph-vm/opt/aleph-vm/runtimes/aleph-debian-11-python/rootfs.squashfs
-
 	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message>=0.1.18'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
diff --git a/packaging/debian-11.dockerfile b/packaging/debian-11.dockerfile
index b1ba77531..7b5465b16 100644
--- a/packaging/debian-11.dockerfile
+++ b/packaging/debian-11.dockerfile
@@ -16,6 +16,3 @@ COPY ../packaging ./packaging
 COPY ../kernels ./kernels
 
 COPY ../examples/ ./examples
-
-RUN mkdir -p ./runtimes/aleph-debian-11-python
-COPY ../runtimes/aleph-debian-11-python/rootfs.squashfs ./runtimes/aleph-debian-11-python/rootfs.squashfs
diff --git a/packaging/ubuntu-20.04.dockerfile b/packaging/ubuntu-20.04.dockerfile
index 90e209557..794003824 100644
--- a/packaging/ubuntu-20.04.dockerfile
+++ b/packaging/ubuntu-20.04.dockerfile
@@ -16,6 +16,3 @@ COPY ../packaging ./packaging
 COPY ../kernels ./kernels
 
 COPY ../examples/ ./examples
-
-RUN mkdir -p ./runtimes/aleph-debian-11-python
-COPY ../runtimes/aleph-debian-11-python/rootfs.squashfs ./runtimes/aleph-debian-11-python/rootfs.squashfs
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index bd541232c..d9eb12e1d 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -149,10 +149,10 @@ def check(self):
             assert isdir(
                 self.FAKE_DATA_PROGRAM
             ), "Local fake program directory is missing"
-        assert isfile(self.FAKE_DATA_MESSAGE), "Local fake message is missing"
-        assert isdir(self.FAKE_DATA_DATA), "Local fake data directory is missing"
-        assert isfile(self.FAKE_DATA_RUNTIME), "Local runtime .squashfs build is missing"
-        assert isfile(self.FAKE_DATA_VOLUME), "Local data volume .squashfs is missing"
+            assert isfile(self.FAKE_DATA_MESSAGE), "Local fake message is missing"
+            assert isdir(self.FAKE_DATA_DATA), "Local fake data directory is missing"
+            assert isfile(self.FAKE_DATA_RUNTIME), "Local runtime .squashfs build is missing"
+            assert isfile(self.FAKE_DATA_VOLUME), "Local data volume .squashfs is missing"
 
     def setup(self):
         os.makedirs(self.MESSAGE_CACHE, exist_ok=True)

From 162515ae613ff809d6e703ec01e704e622ee6f96 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 26 Jan 2022 20:25:27 +0100
Subject: [PATCH 246/990] Packaging: Version had to be updated manually

Use git tags instead
---
 packaging/Makefile            |  7 +++--
 packaging/version_from_git.py | 57 +++++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+), 2 deletions(-)
 create mode 100755 packaging/version_from_git.py

diff --git a/packaging/Makefile b/packaging/Makefile
index 00da410c2..5ded65f57 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -41,6 +41,9 @@ vmlinux:
 	curl -fsSL -o ./target/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/vmlinux.bin
 	#cp ../kernels/vmlinux.bin ./target/vmlinux.bin
 
+version:
+	python3 ./version_from_git.py --inplace deb aleph-vm/DEBIAN/control
+
 build-dir:
 	mkdir -p target
 
@@ -51,7 +54,7 @@ clean:
 	rm -fr ./target/*
 	rm -fr ./build/*
 
-all-podman-debian-11:
+all-podman-debian-11: version
 	cd .. && podman build -t localhost/aleph-vm-packaging:latest -f ./packaging/debian-11.dockerfile .
 	mkdir -p ./target
 	podman run --rm -ti \
@@ -62,7 +65,7 @@ all-podman-debian-11:
 	file target/aleph-vm.deb
 	mv target/aleph-vm.deb target/aleph-vm.debian-11.deb
 
-all-podman-ubuntu-2004:
+all-podman-ubuntu-2004: version
 	cd .. && podman build -t localhost/aleph-vm-packaging:latest -f ./packaging/ubuntu-20.04.dockerfile .
 	mkdir -p ./target
 	podman run --rm -ti \
diff --git a/packaging/version_from_git.py b/packaging/version_from_git.py
new file mode 100755
index 000000000..0c60b0b43
--- /dev/null
+++ b/packaging/version_from_git.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+"""
+Set the version number of a package based on the current repository:
+
+Use the tag it one is available for the current commit.
+Else default to the short commit id, prefixed by the name of the current branch.
+
+Pass the path to the target file to edit in argument.
+"""
+
+import sys
+import os.path
+import subprocess
+import re
+
+script_path, *args, format_, target_file_path = sys.argv
+
+for arg in args:
+    if arg not in ('--inplace', '--stdout'):
+        print("Usage: version_from_git.py [target FILE PATH] [FORMAT] [OPTION...]\n\n"
+              "set the version number of a Debian package based on the current git commit\n\n"
+              "supported formats are 'deb' and 'setup.py'\n\n"
+              "  --help       print this message\n"
+              "  --inplace    edit file in place\n"
+              "  --inplace    edit file in place\n"
+              "  --stdout     print the result on stdout\n")
+        sys.exit(1)
+
+if not os.path.isfile(target_file_path):
+    print("No such file: '{}'".format(target_file_path))
+    sys.exit(2)
+
+
+def get_git_version():
+    output = subprocess.check_output(('git', 'describe', '--tags'))
+    return output.decode().strip()
+
+
+version = get_git_version()
+
+with open(target_file_path, 'r') as target_file:
+    target_content = target_file.read()
+
+if format_ == 'deb':
+    updated_content = re.sub(r"(Version:)\w*(.*)", "\\1 {}".format(version), target_content)
+elif format_ == 'setup.py':
+    updated_content = re.sub(r"(version)\w*=(.*)'", "\\1='{}'".format(version), target_content)
+else:
+    print("Format must be 'deb' or 'setup.py', not '{}'".format(format_))
+
+if '--inplace' in args:
+    with open(target_file_path, 'w') as target_file:
+        target_file.write(updated_content)
+
+if '--stdout' in args:
+    print(updated_content)

From 41c9d6c02135d32814db3754f914e585903a7dfd Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 26 Jan 2022 21:37:35 +0100
Subject: [PATCH 247/990] Fix: Config file was overwritten on package upgrade

---
 packaging/aleph-vm/DEBIAN/conffiles            | 1 +
 packaging/aleph-vm/DEBIAN/control              | 2 +-
 packaging/aleph-vm/etc/aleph-vm/supervisor.env | 3 +--
 3 files changed, 3 insertions(+), 3 deletions(-)
 create mode 100644 packaging/aleph-vm/DEBIAN/conffiles

diff --git a/packaging/aleph-vm/DEBIAN/conffiles b/packaging/aleph-vm/DEBIAN/conffiles
new file mode 100644
index 000000000..e1994d290
--- /dev/null
+++ b/packaging/aleph-vm/DEBIAN/conffiles
@@ -0,0 +1 @@
+/etc/aleph-vm/supervisor.env
diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index 325c5607f..5dbc339e8 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -1,5 +1,5 @@
 Package: aleph-vm
-Version: 0.1.7-0
+Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
diff --git a/packaging/aleph-vm/etc/aleph-vm/supervisor.env b/packaging/aleph-vm/etc/aleph-vm/supervisor.env
index d4fd45c2b..141fdcbbb 100644
--- a/packaging/aleph-vm/etc/aleph-vm/supervisor.env
+++ b/packaging/aleph-vm/etc/aleph-vm/supervisor.env
@@ -1,3 +1,2 @@
 ALEPH_VM_PRINT_SYSTEM_LOGS=True
-ALEPH_VM_USE_JAILER=True
-ALEPH_VM_DOMAIN_NAME=vm.example.org
\ No newline at end of file
+ALEPH_VM_DOMAIN_NAME=vm.example.org

From 68f0b4d764bebec2d806a8d3d8a29935bf68a2c6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 26 Jan 2022 21:57:15 +0100
Subject: [PATCH 248/990] Doc: Update quick install for version 0.1.9

---
 README.md | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 6cdd83524..13d764cc9 100644
--- a/README.md
+++ b/README.md
@@ -43,15 +43,24 @@ This documentation will use the invalid `vm.example.org` domain name. Replace it
 ### 1. Quick install
 
 To quickly install Aleph-VM on a [supported Linux system](./vm_supervisor/README.md#1-supported-platforms)
-for production purposes:
+for production purposes, run the following commands as `root`:
 
 ```shell
-sudo apt update
-sudo apt upgrade
-sudo apt install -y docker.io
-sudo docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.7/aleph-vm.debian-0.1.7-0.deb
-sudo apt install /opt/aleph-vm.debian-0.1.7-0.deb
+apt update
+apt upgrade
+apt install -y docker.io
+docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
+```
+
+On Debian 11:
+```shell
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.9/aleph-vm.debian-11.deb
+apt install /opt/aleph-vm.debian-0.1.9.deb
+```
+On Ubuntu 20.04:
+```shell
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.9/aleph-vm.ubuntu-20.04.deb
+apt install /opt/aleph-vm.ubuntu-20.04.deb
 ```
 
 ### Configuration
@@ -82,6 +91,8 @@ We document how to use Caddy as a reverse proxy since it does automatic HTTPS ce
 First, create a domain name that points to the server on IPv4 and IPv6.
 
 This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CONFIGURE_CADDY.md).
+
+Again, run these commands as `root` after replacing the domain `vm.example.org` with your own:
 ```shell
 sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https
 curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo apt-key add -

From a41303c08232cd10b8ff4c78aa6c9c715fe78733 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 8 Feb 2022 16:29:56 +0100
Subject: [PATCH 249/990] Fix: VMs were not properly shutdown on executor stop

Solution: Stop all VMs in the pool on aiohttp app cleanup.
---
 vm_supervisor/models.py     |  3 +++
 vm_supervisor/pool.py       | 12 ++++++------
 vm_supervisor/supervisor.py |  5 +++++
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 48adb5873..9b9966edf 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -138,6 +138,9 @@ def cancel_expiration(self) -> bool:
             return False
 
     async def stop(self):
+        if self.times.stopped_at is not None:
+            logger.debug(f"VM={self.vm.vm_id} already stopped")
+            return
         await self.all_runs_complete()
         self.times.stopping_at = datetime.now()
         await self.vm.teardown()
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 032052033..2a51a8143 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -1,3 +1,4 @@
+import asyncio
 import logging
 from typing import Dict, Optional, List
 
@@ -50,10 +51,9 @@ def forget_vm(self, vm_hash: VmHash) -> None:
 
     async def stop(self):
         """Stop all VMs in the pool."""
-        hashes_to_forget: List[VmHash] = []
-        for vm_hash, execution in self.executions.items():
-            await execution.stop()
-            hashes_to_forget.append(vm_hash)
 
-        for vm_hash in hashes_to_forget:
-            self.forget_vm(vm_hash)
+        # Stop executions in parallel:
+        await asyncio.gather(*(
+            execution.stop()
+            for vm_hash, execution in self.executions.items()
+        ))
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 1566b12c2..bcc71b5bb 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -20,6 +20,7 @@
     about_config,
     status_check_fastapi,
 )
+from .run import pool
 
 logger = logging.getLogger(__name__)
 
@@ -36,6 +37,9 @@
     ]
 )
 
+async def stop_all_vms(app: web.Application):
+    await pool.stop()
+
 
 def run():
     """Run the VM Supervisor."""
@@ -49,5 +53,6 @@ def run():
     if settings.WATCH_FOR_MESSAGES:
         app.on_startup.append(start_watch_for_messages_task)
         app.on_cleanup.append(stop_watch_for_messages_task)
+        app.on_cleanup.append(stop_all_vms)
 
     web.run_app(app, host=settings.SUPERVISOR_HOST, port=settings.SUPERVISOR_PORT)

From de54cc6efe3e26d2e96b1b96798b584d299a9986 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 8 Feb 2022 16:31:08 +0100
Subject: [PATCH 250/990] Fix: Shutdown procedure did not handle premature stop
 of the VM

Solution: Handle common errors in sending the shutdown signal to the VM
---
 firecracker/microvm.py | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index c7081ddc1..82e0a361f 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -369,23 +369,31 @@ async def unix_client_connected(*_):
 
     async def shutdown(self):
         logger.debug(f"Shutown vm={self.vm_id}")
-        reader, writer = await asyncio.open_unix_connection(path=self.vsock_path)
-        payload = b"halt"
-        writer.write(b"CONNECT 52\n" + payload)
+        try:
+            reader, writer = await asyncio.open_unix_connection(path=self.vsock_path)
+        except (FileNotFoundError, ConnectionResetError) as error:
+            logger.warning(f"VM={self.vm_id} cannot receive shutdown signal: {error.args}")
+            return
+
+        try:
+            payload = b"halt"
+            writer.write(b"CONNECT 52\n" + payload)
 
-        await writer.drain()
+            await writer.drain()
 
-        ack: bytes = await reader.readline()
-        logger.debug(f"ack={ack.decode()}")
+            ack: bytes = await reader.readline()
+            logger.debug(f"ack={ack.decode()}")
 
-        msg: bytes = await reader.readline()
-        logger.debug(f"msg={msg}")
+            msg: bytes = await reader.readline()
+            logger.debug(f"msg={msg}")
 
-        msg2: bytes = await reader.readline()
-        logger.debug(f"msg2={msg2}")
+            msg2: bytes = await reader.readline()
+            logger.debug(f"msg2={msg2}")
 
-        if msg2 != b"STOPZ\n":
-            logger.error("Unexpected response from VM")
+            if msg2 != b"STOPZ\n":
+                logger.error(f"Unexpected response from VM: {msg2[:20]}")
+        except ConnectionResetError as error:
+            logger.warning(f"ConnectionResetError in shutdown of {self.vm_id}: {error.args}")
 
     async def stop(self):
         if self.proc:

From 4f0dd35748c61d565ab0a6e3f74a45c68b60fefd Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 8 Feb 2022 17:11:21 +0100
Subject: [PATCH 251/990] Feature: Add support for error monitoring on Sentry

Fixes #132
---
 firecracker/microvm.py    |  2 +-
 vm_supervisor/__main__.py | 27 +++++++++++++++++++++++++--
 vm_supervisor/conf.py     |  2 ++
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 82e0a361f..97532bdfc 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -391,7 +391,7 @@ async def shutdown(self):
             logger.debug(f"msg2={msg2}")
 
             if msg2 != b"STOPZ\n":
-                logger.error(f"Unexpected response from VM: {msg2[:20]}")
+                logger.warning(f"Unexpected response from VM: {msg2[:20]}")
         except ConnectionResetError as error:
             logger.warning(f"ConnectionResetError in shutdown of {self.vm_id}: {error.args}")
 
diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index 98547fcef..361fd2ce2 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -8,11 +8,16 @@
 
 from aiohttp.web import Response, Request
 
+try:
+    import sentry_sdk
+except ImportError:
+    sentry_sdk = None
+
 from vm_supervisor.pubsub import PubSub
-from .run import run_code_on_request, run_code_on_event
-from .models import VmHash
 from . import supervisor
 from .conf import settings
+from .models import VmHash
+from .run import run_code_on_request, run_code_on_event
 
 logger = logging.getLogger(__name__)
 
@@ -207,6 +212,24 @@ def main():
         ALLOW_VM_NETWORKING=args.allow_vm_networking,
         FAKE_DATA_PROGRAM=args.fake_data_program,
     )
+
+    if sentry_sdk:
+        if settings.SENTRY_DSN:
+            sentry_sdk.init(
+                dsn=settings.SENTRY_DSN,
+                server_name=settings.DOMAIN_NAME,
+
+                # Set traces_sample_rate to 1.0 to capture 100%
+                # of transactions for performance monitoring.
+                # We recommend adjusting this value in production.
+                traces_sample_rate=1.0
+            )
+        else:
+            logger.debug("Sentry SDK found with no DNS configured.")
+    else:
+        logger.debug("Sentry SDK not found. \n"
+                     "Use `pip install sentry-sdk` and configure SENTRY_DSN if you'd like to monitor errors.")
+
     settings.setup()
     if args.print_settings:
         print(settings.display())
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index d9eb12e1d..6d513e7b7 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -124,6 +124,8 @@ class Settings(BaseSettings):
         "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
     )
 
+    SENTRY_DSN: Optional[str] = None
+
     def update(self, **kwargs):
         for key, value in kwargs.items():
             if key != key.upper():

From 4441867db4f588172aba2798ae83d996798c13b3 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 8 Feb 2022 17:48:36 +0100
Subject: [PATCH 252/990] Fix: Unhandled error class during shutdown

---
 firecracker/microvm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 97532bdfc..ce9763486 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -371,7 +371,7 @@ async def shutdown(self):
         logger.debug(f"Shutown vm={self.vm_id}")
         try:
             reader, writer = await asyncio.open_unix_connection(path=self.vsock_path)
-        except (FileNotFoundError, ConnectionResetError) as error:
+        except (FileNotFoundError, ConnectionResetError, ConnectionRefusedError) as error:
             logger.warning(f"VM={self.vm_id} cannot receive shutdown signal: {error.args}")
             return
 

From 6c3233ba7f9fa250407fdfa994ec5a5ea8804316 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Wed, 9 Feb 2022 09:54:33 +0100
Subject: [PATCH 253/990] [Doc] Fixed typos

---
 CONFIGURE_CADDY.md        | 2 +-
 README.md                 | 6 +++---
 firecracker/microvm.py    | 2 +-
 tutorials/ADVANCED.md     | 3 +--
 tutorials/README.md       | 2 +-
 tutorials/REQUIREMENTS.md | 2 +-
 vm_supervisor/README.md   | 2 +-
 7 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/CONFIGURE_CADDY.md b/CONFIGURE_CADDY.md
index 383d2d8ac..472e8fb41 100644
--- a/CONFIGURE_CADDY.md
+++ b/CONFIGURE_CADDY.md
@@ -25,7 +25,7 @@ sudo apt install -y certbot
 
 certbot certonly --manual --email email@yourdomain.org --preferred-challenges dns \
   --server https://acme-v02.api.letsencrypt.org/directory --agree-tos \
-  -d 'vm.yourdomain.org,*.vm.youdomain.org'
+  -d 'vm.yourdomain.org,*.vm.yourdomain.org'
 ```
 
 ## 2. Caddy Server
diff --git a/README.md b/README.md
index 13d764cc9..702b27ae9 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@ They benefit from running in their own, customizable Linux virtual environment.
 Writing programs in Python using ASGI compatible frameworks (
 [FastAPI](https://github.com/tiangolo/fastapi), 
 [Django](https://docs.djangoproject.com/en/3.0/topics/async/),
-...) allows developers to use advanced functionnalities not yet available for other languages.
+...) allows developers to use advanced functionalities not yet available for other languages.
 
 ## 1. Creating and running an Aleph Program 
 
@@ -25,7 +25,7 @@ The rest of this document focuses on how to run an Aleph-VM node that hosts and
 ### 0. Requirements
 
 - A [supported Linux server](./vm_supervisor/README.md#1-supported-platforms)
-- A public domain name from a trusted registar and domain. 
+- A public domain name from a trusted registrar and domain. 
 
 In order to run an Aleph.im Compute Resource Node, you will also need the following resources:
 
@@ -33,7 +33,7 @@ In order to run an Aleph.im Compute Resource Node, you will also need the follow
   - Min. 8 cores / 16 threads, 3.0 ghz+ CPU (gaming CPU for fast boot-up of microVMs)
   - Min. 12 core / 24 threads, 2.4ghz+ CPU (datacenter CPU for multiple concurrent loads)
 - RAM: 64GB
-- STORAGE: 1TB (Nvme SSD prefered, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
+- STORAGE: 1TB (Nvme SSD preferred, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
 - BANDWIDTH: Minimum of 500 MB/s
 
 You will need a public domain name with access to add TXT and wildcard records.
diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index ce9763486..6f3e8c646 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -368,7 +368,7 @@ async def unix_client_connected(*_):
             raise MicroVMFailedInit()
 
     async def shutdown(self):
-        logger.debug(f"Shutown vm={self.vm_id}")
+        logger.debug(f"Shutdown vm={self.vm_id}")
         try:
             reader, writer = await asyncio.open_unix_connection(path=self.vsock_path)
         except (FileNotFoundError, ConnectionResetError, ConnectionRefusedError) as error:
diff --git a/tutorials/ADVANCED.md b/tutorials/ADVANCED.md
index 344e7e7a6..4eb5cf06e 100644
--- a/tutorials/ADVANCED.md
+++ b/tutorials/ADVANCED.md
@@ -136,7 +136,7 @@ Unlike the cache, you can use these volumes to store any kind of files, includin
 There is no guarantee that these volumes will not be deleted anytime when the
 program is not running and important data must be persisted on the Aleph network.
 
-Host persistend volumes have a fixed size and must be named. The name will be used in the future
+Host persistent volumes have a fixed size and must be named. The name will be used in the future
 to allow changing the mount point of a volume.
 
 
@@ -154,4 +154,3 @@ records:
 `hosted-on-aleph.net IN CNAME aleph.sh`
 2. A `TXT` record to the VM hash with the prefix _aleph-id, for example:
 `_aleph-id.hosted-on-aleph.org 60 IN TXT "b34f193470c349b1d9b60903a6d172e8c335710736d4999ff05971692febe8bc"`
-
diff --git a/tutorials/README.md b/tutorials/README.md
index a40614ccb..49505662f 100644
--- a/tutorials/README.md
+++ b/tutorials/README.md
@@ -135,7 +135,7 @@ interface with programs written in Python. ASGI interfaces with many Python fram
 FastAPI but also [Django](https://www.djangoproject.com/) 
 or [Quart](https://github.com/pgjones/quart).
 
-Test your progam locally using uvicorn, an ASGI server:
+Test your program locally using uvicorn, an ASGI server:
 
 ```shell
 uvicorn main:app --reload
diff --git a/tutorials/REQUIREMENTS.md b/tutorials/REQUIREMENTS.md
index c31b009d2..1ddbdf6e7 100644
--- a/tutorials/REQUIREMENTS.md
+++ b/tutorials/REQUIREMENTS.md
@@ -47,7 +47,7 @@ Once the command is down, your virtual machine will be booted and ready!
 
 ### Set Vagrantfile configuration
 
-Open the vagranfile and add following `config.vm.box``
+Open the vagrantfile and add following `config.vm.box``
 
 ```shell
 config.vm.network "forwarded_port", guest:8000, host:8000
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index 5bf2bcfa1..90c291165 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -73,7 +73,7 @@ ln /opt/firecracker/firecracker-v* /opt/firecracker/firecracker
 ln /opt/firecracker/jailer-v* /opt/firecracker/jailer
 ```
 
-### 2.d. Clone this reposotiry on the host machine and enter it.
+### 2.d. Clone this repository on the host machine and enter it.
 
 ```shell
 git clone https://github.com/aleph-im/aleph-vm.git

From 361645d870adc1c6aa03901bc149e66ecc075e3a Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 14 Feb 2022 16:28:25 +0100
Subject: [PATCH 254/990] Fix: Users could now identify missing hash

The supervisor was returning 404 Hash not found when any dependency of the program was missing with no further detail.

This helps the user debugging by returning the missing hash.
---
 vm_supervisor/messages.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/vm_supervisor/messages.py b/vm_supervisor/messages.py
index e0610de65..8abaa186f 100644
--- a/vm_supervisor/messages.py
+++ b/vm_supervisor/messages.py
@@ -18,7 +18,8 @@ async def try_get_message(ref: str) -> ProgramMessage:
         raise HTTPServiceUnavailable(reason="Aleph Connector unavailable")
     except ClientResponseError as error:
         if error.status == 404:
-            raise HTTPNotFound(reason="Hash not found")
+            raise HTTPNotFound(reason="Hash not found",
+                               body=f"Hash not found: {ref}")
         else:
             raise
 
@@ -30,7 +31,8 @@ async def get_latest_ref(item_hash: str) -> str:
         raise HTTPServiceUnavailable(reason="Aleph Connector unavailable")
     except ClientResponseError as error:
         if error.status == 404:
-            raise HTTPNotFound(reason="Hash not found")
+            raise HTTPNotFound(reason="Hash not found",
+                               body=f"Hash not found: {item_hash}")
         else:
             raise
 

From eb2de3f70dca66bd227bf6ce8ffc4f61b356f191 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 14 Feb 2022 13:34:15 +0100
Subject: [PATCH 255/990] Fix: VM resource seconds field was ignored

Solution: Timeout the request after the number of seconds has passed.

Fixes #139
---
 vm_supervisor/run.py                    |  4 ++++
 vm_supervisor/vm/firecracker_microvm.py | 30 +++++++++++++++----------
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index cae8a101f..624b84f9a 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -1,3 +1,4 @@
+import asyncio
 import logging
 from typing import Dict, Any, Optional
 
@@ -87,6 +88,9 @@ async def run_code_on_request(
     try:
         await execution.becomes_ready()
         result_raw: bytes = await execution.run_code(scope=scope)
+    except asyncio.TimeoutError:
+        logger.warning(f"VM{execution.vm.vm_id} did not respond within `resource.seconds`")
+        return web.HTTPGatewayTimeout(body="Program did not respond within `resource.seconds`")
     except UnpackValueError as error:
         logger.exception(error)
         return web.Response(status=502, reason="Invalid response from VM")
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index d069ab317..04f3907a7 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -432,20 +432,26 @@ async def run_code(
             raise ValueError("MicroVM must be created first")
         logger.debug("running code")
         scope = scope or {}
-        reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
 
-        payload = RunCodePayload(scope=scope)
+        async def communicate(reader, writer, scope):
+            payload = RunCodePayload(scope=scope)
 
-        writer.write(b"CONNECT 52\n" + payload.as_msgpack())
-        await writer.drain()
+            writer.write(b"CONNECT 52\n" + payload.as_msgpack())
+            await writer.drain()
+
+            ack: bytes = await reader.readline()
+            logger.debug(f"ack={ack.decode()}")
 
-        ack: bytes = await reader.readline()
-        logger.debug(f"ack={ack.decode()}")
+            logger.debug("waiting for VM response")
+            response: bytes = await reader.read()
 
-        logger.debug("waiting for VM response")
-        response: bytes = await reader.read()
+            return response
 
-        logger.debug("cleaning VM resources")
-        writer.close()
-        await writer.wait_closed()
-        return response
+        reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
+        try:
+            return await asyncio.wait_for(communicate(reader, writer, scope),
+                                          timeout=self.hardware_resources.seconds)
+        finally:
+            logger.debug("Cleaning VM socket resources")
+            writer.close()
+            await writer.wait_closed()

From 9c9e1997c377b23bc935f0ba2ef83b999c394107 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 14 Feb 2022 13:25:37 +0100
Subject: [PATCH 256/990] Fix: Sentry DSN appeared in clear in the logs

Fixes #140
---
 vm_supervisor/conf.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 6d513e7b7..f0018bcdc 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -126,6 +126,10 @@ class Settings(BaseSettings):
 
     SENTRY_DSN: Optional[str] = None
 
+    # Fields
+    SENSITIVE_FIELDS: List[str] = Field(default=["SENTRY_DSN"],
+                                        description="Sensitive fields, redacted from `--print-settings`.")
+
     def update(self, **kwargs):
         for key, value in kwargs.items():
             if key != key.upper():
@@ -174,9 +178,17 @@ def setup(self):
                 assert "This should never happen"
 
     def display(self) -> str:
+        annotations = self.__annotations__.copy()
+
+        for attr in annotations.keys():
+            if attr in self.SENSITIVE_FIELDS:
+                annotations[attr] = ""
+            else:
+                annotations[attr] = getattr(self, attr)
+
         return "\n".join(
-            f"{annotation:<17} = {getattr(self, annotation)}"
-            for annotation, value in self.__annotations__.items()
+            f"{annotation:<27} = {value}"
+            for annotation, value in annotations.items()
         )
 
     class Config:

From 929919f4486801c3f0dab3c7dd0647e8c38dab55 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 14 Feb 2022 12:55:38 +0100
Subject: [PATCH 257/990] Doc: Package installation was difficult to follow

The documentation about the installation procedure in the main README.md was mixed with other topics and contained if-else for Debian/Ubuntu.

This splits the doc in different files for each supported host operating system.
---
 README.md                   | 122 ++++------------------------
 doc/INSTALL-Debian-11.md    | 153 ++++++++++++++++++++++++++++++++++++
 doc/INSTALL-Ubuntu-20.04.md | 152 +++++++++++++++++++++++++++++++++++
 doc/INSTALL.md              |   4 +
 4 files changed, 323 insertions(+), 108 deletions(-)
 create mode 100644 doc/INSTALL-Debian-11.md
 create mode 100644 doc/INSTALL-Ubuntu-20.04.md
 create mode 100644 doc/INSTALL.md

diff --git a/README.md b/README.md
index 702b27ae9..ac3253a7c 100644
--- a/README.md
+++ b/README.md
@@ -13,122 +13,29 @@ Writing programs in Python using ASGI compatible frameworks (
 [Django](https://docs.djangoproject.com/en/3.0/topics/async/),
 ...) allows developers to use advanced functionalities not yet available for other languages.
 
-## 1. Creating and running an Aleph Program 
+## 1. Install Aleph-VM from packages
 
-Have a look at [tutorials/README.md](tutorials/README.md) for a tutorial on how to program VMs
-as a user.
-
-The rest of this document focuses on how to run an Aleph-VM node that hosts and executes the programs. 
-
-## 2. Installing Aleph-VM on a server
-
-### 0. Requirements
-
-- A [supported Linux server](./vm_supervisor/README.md#1-supported-platforms)
-- A public domain name from a trusted registrar and domain. 
-
-In order to run an Aleph.im Compute Resource Node, you will also need the following resources:
-
-- CPU (2 options):
-  - Min. 8 cores / 16 threads, 3.0 ghz+ CPU (gaming CPU for fast boot-up of microVMs)
-  - Min. 12 core / 24 threads, 2.4ghz+ CPU (datacenter CPU for multiple concurrent loads)
-- RAM: 64GB
-- STORAGE: 1TB (Nvme SSD preferred, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
-- BANDWIDTH: Minimum of 500 MB/s
-
-You will need a public domain name with access to add TXT and wildcard records.
-
-This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
-
-### 1. Quick install
-
-To quickly install Aleph-VM on a [supported Linux system](./vm_supervisor/README.md#1-supported-platforms)
-for production purposes, run the following commands as `root`:
-
-```shell
-apt update
-apt upgrade
-apt install -y docker.io
-docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
-```
+Install Aleph-VM to run an Aleph.im Compute Resource Node easily from official pre-built packages.
 
-On Debian 11:
-```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.9/aleph-vm.debian-11.deb
-apt install /opt/aleph-vm.debian-0.1.9.deb
-```
-On Ubuntu 20.04:
-```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.9/aleph-vm.ubuntu-20.04.deb
-apt install /opt/aleph-vm.ubuntu-20.04.deb
-```
+- [On Debian 11](./doc/INSTALL-Debian-11.md)
+- [On Ubuntu 20.04](./doc/INSTALL-Ubuntu-20.04.md)
 
-### Configuration
+## 2. Install Aleph-VM from source
 
-Update the configuration in `/etc/aleph-vm/supervisor.env`. 
+For development and testing, install Aleph-VM from source.
 
-You will want to insert your domain name in the for of:
-```
-ALEPH_VM_DOMAIN_NAME=vm.example.org
-```
+1. Install the [VM-Connector](./vm_connector/README.md)
+2. Install the [VM-Supervisor](./vm_supervisor/README.md).
+3. Install and [configure a reverse-proxy such as [Caddy](./CONFIGURE_CADDY.md)
 
-On Ubuntu, the default network interface is not `eth0` and you will want to configure the default interface. Due to the DNS being handled by `systemd-resolved` on Ubuntu, you should also configure the DNS to use `resolvectl`.
-```
-ALEPH_VM_NETWORK_INTERFACE=enp0s1
-ALEPH_VM_DNS_RESOLUTION=resolvectl
-```
-(don't forget to replace `enp0s1` with the name of your default network interface).
+## 3. Create and run an Aleph Program 
 
-Finally, restart the service:
-```shell
-systemctl restart aleph-vm-supervisor
-```
-
-### Reverse Proxy
-
-We document how to use Caddy as a reverse proxy since it does automatic HTTPS certificates.
-
-First, create a domain name that points to the server on IPv4 and IPv6.
-
-This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CONFIGURE_CADDY.md).
-
-Again, run these commands as `root` after replacing the domain `vm.example.org` with your own:
-```shell
-sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo apt-key add -
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list
-sudo apt update
-sudo apt install caddy
-
-sudo cat >/etc/caddy/Caddyfile < ![image](https://user-images.githubusercontent.com/404665/150202090-91a02536-4e04-4af2-967f-fe105d116e1f.png)
-
-## 3. Architecture
+## 4. Architecture
 
 ![Aleph im VM - Details](https://user-images.githubusercontent.com/404665/127126908-3225a633-2c36-4129-8766-9810f2fcd7d6.png)
 
@@ -144,7 +51,6 @@ Assist with operations related to the Aleph network.
 
 See [vm_connector/README.md](./vm_connector/README.md).
 
-
 ---
 
 ![aleph.im logo](https://aleph.im/assets/img/logo-wide.1832dbae.svg)
diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
new file mode 100644
index 000000000..d1e13856d
--- /dev/null
+++ b/doc/INSTALL-Debian-11.md
@@ -0,0 +1,153 @@
+# Installing Aleph-VM on a server / Debian 11 Bullseye
+
+## 0. Introduction
+
+For production using official Debian packages.
+
+## 1. Requirements
+
+- A [supported Linux server](../vm_supervisor/README.md#1-supported-platforms)
+- A public domain name from a registrar and top level domain you trust. 
+
+In order to run an official Aleph.im Compute Resource Node (CRN), you will also need the following resources:
+
+- CPU (2 options):
+  - Min. 8 cores / 16 threads, 3.0 ghz+ CPU (gaming CPU for fast boot-up of microVMs)
+  - Min. 12 core / 24 threads, 2.4ghz+ CPU (datacenter CPU for multiple concurrent loads)
+- RAM: 64GB
+- STORAGE: 1TB (NVMe SSD preferred, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
+- BANDWIDTH: Minimum of 500 MB/s
+
+You will need a public domain name with access to add TXT and wildcard records.
+
+> 🛈 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
+
+## 2. Installation
+
+Run the following commands as `root`:
+
+First install the [VM-Connector](../vm_connector/README.md) using Docker:
+```shell
+apt update
+apt upgrade
+apt install -y docker.io apparmor-profiles
+docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
+```
+
+Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
+The procedure is similar for updates.
+```shell
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.9/aleph-vm.debian-11.deb
+apt install /opt/aleph-vm.debian-11.deb
+```
+
+### Configuration
+
+Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
+
+You will want to insert your domain name in the form of:
+```
+ALEPH_VM_DOMAIN_NAME=vm.example.org
+```
+
+On some systems, the default network interface is not `eth0` and you will want to configure the default interface
+by adding:
+```
+ALEPH_VM_NETWORK_INTERFACE=enp0s1
+```
+(don't forget to replace `enp0s1` with the name of your default network interface).
+
+Debian 11 by default uses `/etc/resolv.conf` for DNS resolution. The VM Supervisor uses this by default.
+If your system uses [systemd-resolved](https://manpages.debian.org/bullseye/systemd/systemd-resolved.8.en.html)
+instead, uncomment and add the following setting:
+```
+#ALEPH_VM_DNS_RESOLUTION=resolvctl
+```
+
+> 🛈 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
+
+Finally, restart the service:
+```shell
+systemctl restart aleph-vm-supervisor
+```
+
+## 3. Reverse Proxy
+
+We document how to use Caddy as a reverse proxy since it manages and renews HTTPS certificates automatically.
+
+Any other reverse-proxy (Nginx, HAProxy, Apache2, ...) should do the job as well, just make sure to renew the 
+HTTPS/TLS certificates on time.
+
+First, create a domain name that points to the server on IPv4 (A) and IPv6 (AAAA).
+
+This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CONFIGURE_CADDY.md).
+
+Again, run these commands as `root`:
+```shell
+apt install -y debian-keyring debian-archive-keyring apt-transport-https gnupg
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | apt-key add -
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list
+apt update
+apt install caddy
+```
+
+Then, after replacing the domain `vm.example.org` with your own, use configure Caddy:
+```shell
+cat >/etc/caddy/Caddyfile < ![image](https://user-images.githubusercontent.com/404665/150202090-91a02536-4e04-4af2-967f-fe105d116e1f.png)
+
+If you face an issue, check the logs of the different services for errors:
+
+VM-Supervisor:
+```shell
+journalctl -f -u aleph-vm-supervisor.service 
+```
+
+Caddy:
+```shell
+journalctl -f -u caddy.service 
+```
+
+VM-Connector:
+```shell
+docker logs -f vm-connector
+```
+
+### Common errors
+
+#### "Network interface eth0 does not exist"
+
+Did you update the configuration file `/etc/aleph-vm/vm-supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
+the default network interface of your server ?
+
+#### "Aleph Connector unavailable"
+
+Investigate the installation of the VM-Connector using Docker in step 2.
+
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
new file mode 100644
index 000000000..f85dcadbb
--- /dev/null
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -0,0 +1,152 @@
+# Installing Aleph-VM on a server / Ubuntu 20.04 Focal Fossa
+
+## 0. Introduction
+
+For production using official Debian packages.
+
+## 1. Requirements
+
+- A [supported Linux server](../vm_supervisor/README.md#1-supported-platforms)
+- A public domain name from a registrar and top level domain you trust. 
+
+In order to run an official Aleph.im Compute Resource Node (CRN), you will also need the following resources:
+
+- CPU (2 options):
+  - Min. 8 cores / 16 threads, 3.0 ghz+ CPU (gaming CPU for fast boot-up of microVMs)
+  - Min. 12 core / 24 threads, 2.4ghz+ CPU (datacenter CPU for multiple concurrent loads)
+- RAM: 64GB
+- STORAGE: 1TB (NVMe SSD preferred, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
+- BANDWIDTH: Minimum of 500 MB/s
+
+You will need a public domain name with access to add TXT and wildcard records.
+
+> 🛈 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
+
+## 2. Installation
+
+Run the following commands:
+
+First install the [VM-Connector](../vm_connector/README.md) using Docker:
+```shell
+sudo apt update
+sudo apt upgrade
+sudo apt install -y docker.io
+docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
+```
+
+Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
+The procedure is similar for updates.
+```shell
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.9/aleph-vm.ubuntu-20.04.deb
+sudo apt install /opt/aleph-vm.ubuntu-20.04.deb
+```
+
+### Configuration
+
+Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
+
+You will want to insert your domain name in the form of:
+```
+ALEPH_VM_DOMAIN_NAME=vm.example.org
+```
+
+Ubuntu 20.04 by default uses [systemd-resolved](https://manpages.ubuntu.com/manpages/focal/man8/systemd-resolved.service.8.html)
+for DNS resolution. The following setting configures the VM Supervisor to use it instead of reading the default `/etc/resolv.conf`.
+```
+ALEPH_VM_DNS_RESOLUTION=resolvectl
+```
+
+> 🛈 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
+
+On some systems, the default network interface is not `eth0` and you will want to configure the default interface
+by adding:
+```
+ALEPH_VM_NETWORK_INTERFACE=enp0s1
+```
+(don't forget to replace `enp0s1` with the name of your default network interface).
+
+Finally, restart the service:
+```shell
+sudo systemctl restart aleph-vm-supervisor
+```
+
+## 3. Reverse Proxy
+
+We document how to use Caddy as a reverse proxy since it manages and renews HTTPS certificates automatically.
+
+Any other reverse-proxy (Nginx, HAProxy, Apache2, ...) should do the job as well, just make sure to renew the 
+HTTPS/TLS certificates on time.
+
+First, create a domain name that points to the server on IPv4 (A) and IPv6 (AAAA).
+
+This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CONFIGURE_CADDY.md).
+
+Again, run these commands as `root`:
+```shell
+sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https gnupg
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo apt-key add -
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list
+sudo apt update
+sudo apt install caddy
+```
+
+Then, after replacing the domain `vm.example.org` with your own, use configure Caddy:
+```shell
+sudo cat >/etc/caddy/Caddyfile < ![image](https://user-images.githubusercontent.com/404665/150202090-91a02536-4e04-4af2-967f-fe105d116e1f.png)
+
+If you face an issue, check the logs of the different services for errors:
+
+VM-Supervisor:
+```shell
+sudo journalctl -f -u aleph-vm-supervisor.service 
+```
+
+Caddy:
+```shell
+sudo journalctl -f -u caddy.service 
+```
+
+VM-Connector:
+```shell
+sudo docker logs -f vm-connector
+```
+
+### Common errors
+
+#### "Network interface eth0 does not exist"
+
+Did you update the configuration file `/etc/aleph-vm/vm-supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
+the default network interface of your server ?
+
+#### "Aleph Connector unavailable"
+
+Investigate the installation of the VM-Connector using Docker in step 2.
+
diff --git a/doc/INSTALL.md b/doc/INSTALL.md
new file mode 100644
index 000000000..3ede0680a
--- /dev/null
+++ b/doc/INSTALL.md
@@ -0,0 +1,4 @@
+# Installing Aleph-VM
+
+- [On Debian 11](./INSTALL-Debian-11.md)
+- [On Ubuntu 20.04](./INSTALL-Ubuntu-20.04.md)

From 2b373cebc02781aaba5cbae55d27d370f809fdf2 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 10 Feb 2022 16:53:40 +0100
Subject: [PATCH 258/990] Fix: forget_vm called twice when VM init times out

Solution: Allow execution to be already missing.
---
 vm_supervisor/pool.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 2a51a8143..00a09d348 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -47,7 +47,10 @@ async def get_running_vm(self, vm_hash: VmHash) -> Optional[VmExecution]:
             return None
 
     def forget_vm(self, vm_hash: VmHash) -> None:
-        self.executions.pop(vm_hash)
+        try:
+            del self.executions[vm_hash]
+        except KeyError:
+            pass
 
     async def stop(self):
         """Stop all VMs in the pool."""

From 83a673c5cef77914cc9f10c003ebb10a6839bbbb Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 16 Feb 2022 12:28:53 +0100
Subject: [PATCH 259/990] Doc: Install version 0.1.10 in the doc.

---
 doc/INSTALL-Debian-11.md    | 2 +-
 doc/INSTALL-Ubuntu-20.04.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index d1e13856d..7126a52c6 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.9/aleph-vm.debian-11.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.10/aleph-vm.debian-11.deb
 apt install /opt/aleph-vm.debian-11.deb
 ```
 
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
index f85dcadbb..575625cf1 100644
--- a/doc/INSTALL-Ubuntu-20.04.md
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.9/aleph-vm.ubuntu-20.04.deb
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.10/aleph-vm.ubuntu-20.04.deb
 sudo apt install /opt/aleph-vm.ubuntu-20.04.deb
 ```
 

From 0dd5849a7a29dc18151a583ed22ffd2b047cc8ab Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 22 Mar 2022 11:28:42 +0100
Subject: [PATCH 260/990] Fix: Undefined variable crashed `stop_all_vms()`

---
 vm_supervisor/supervisor.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index bcc71b5bb..478222372 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -11,6 +11,7 @@
 from aiohttp import web
 
 from .conf import settings
+from .run import pool
 from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task
 from .views import (
     run_code_from_path,

From e405cc23f2abc5f7882184ceda52c0948c381484 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 22 Mar 2022 11:29:48 +0100
Subject: [PATCH 261/990] Fix: Login URL displayed `https` on localhost

It therefore did not work for most developers.
---
 vm_supervisor/supervisor.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 478222372..446cba6b4 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -46,10 +46,13 @@ def run():
     """Run the VM Supervisor."""
     settings.check()
 
+    hostname = settings.DOMAIN_NAME
+    protocol = "http" if hostname == "localhost" else "https"
+
     # Require a random token to access /about APIs
     secret_token = token_urlsafe(nbytes=32)
     app["secret_token"] = secret_token
-    print(f"Login to /about pages /about/login?token={secret_token}")
+    print(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
 
     if settings.WATCH_FOR_MESSAGES:
         app.on_startup.append(start_watch_for_messages_task)

From d80fb4302d93f3aebb50c625dd836566d13eaa63 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 22 Mar 2022 11:30:49 +0100
Subject: [PATCH 262/990] Cleanup: Add type hints and better error message

---
 vm_supervisor/views.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index 3f54de5c6..a7d13d9c3 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -69,13 +69,13 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
     return await run_code_on_request(message_ref, path, request)
 
 
-def authenticate_request(request: web.Request):
+def authenticate_request(request: web.Request) -> web.Response:
     """Check that the token in the cookies matches the app's secret token."""
     if request.cookies.get("token") != request.app["secret_token"]:
-        raise web.HTTPUnauthorized(reason="Invalid token")
+        raise web.HTTPUnauthorized(reason="Invalid token", body="401 Invalid token")
 
 
-async def about_login(request: web.Request):
+async def about_login(request: web.Request) -> web.Response:
     token = request.query.get("token")
     if token == request.app["secret_token"]:
         response = web.HTTPFound("/about/config")
@@ -85,7 +85,7 @@ async def about_login(request: web.Request):
         return web.json_response({"success": False}, status=401)
 
 
-async def about_executions(request: web.Request):
+async def about_executions(request: web.Request) -> web.Response:
     authenticate_request(request)
     return web.json_response(
         [{key: value for key, value in pool.executions.items()}],
@@ -93,7 +93,7 @@ async def about_executions(request: web.Request):
     )
 
 
-async def about_config(request: web.Request):
+async def about_config(request: web.Request) -> web.Response:
     authenticate_request(request)
     return web.json_response(
         settings,

From 10214e90d66da93a42da628e6d999116b1113610 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 22 Mar 2022 11:31:16 +0100
Subject: [PATCH 263/990] Fix: `make clean` did not remove source copy

---
 packaging/Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packaging/Makefile b/packaging/Makefile
index 5ded65f57..4cef3fe3b 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -53,6 +53,7 @@ target-dir:
 clean:
 	rm -fr ./target/*
 	rm -fr ./build/*
+	rm -fr ./aleph-vm/opt/aleph-vm/
 
 all-podman-debian-11: version
 	cd .. && podman build -t localhost/aleph-vm-packaging:latest -f ./packaging/debian-11.dockerfile .

From be47d686fad1fe925382370dd837d1a97850c0e0 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 22 Mar 2022 11:32:16 +0100
Subject: [PATCH 264/990] Change: Store Aleph variable files in `/var/` instead
 of `/tmp` and `/var/tmp`.

---
 vm_supervisor/conf.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index f0018bcdc..7574a32cf 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -89,14 +89,14 @@ class Settings(BaseSettings):
 
     CONNECTOR_URL: Url = Url("http://localhost:4021")
 
-    CACHE_ROOT: FilePath = FilePath("/tmp/aleph/vm_supervisor")
+    CACHE_ROOT: FilePath = FilePath("/var/cache/aleph/vm")
     MESSAGE_CACHE: FilePath = FilePath(join(CACHE_ROOT, "message"))
     CODE_CACHE: FilePath = FilePath(join(CACHE_ROOT, "code"))
     RUNTIME_CACHE: FilePath = FilePath(join(CACHE_ROOT, "runtime"))
     DATA_CACHE: FilePath = FilePath(join(CACHE_ROOT, "data"))
 
     PERSISTENT_VOLUMES_DIR: FilePath = FilePath(
-        join("/var/tmp/aleph", "volumes", "persistent")
+        join("/var/lib/aleph/vm/volumes", "volumes", "persistent")
     )
 
     MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000  # 10 MB

From b01573ed6123c8bbe551272d0c1767a0ba58e2d9 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 21 Mar 2022 18:05:32 +0100
Subject: [PATCH 265/990] Problem: Users could not know the version of Aleph-VM
 for diagnostic

Solution: Add the version of the software in the HTTP response headers
---
 packaging/Makefile            |  1 +
 packaging/version_from_git.py |  4 +++-
 vm_supervisor/__init__.py     | 11 +++++++++++
 vm_supervisor/supervisor.py   | 17 +++++++++++++++--
 4 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 4cef3fe3b..83ef366c7 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -43,6 +43,7 @@ vmlinux:
 
 version:
 	python3 ./version_from_git.py --inplace deb aleph-vm/DEBIAN/control
+	python3 ./version_from_git.py --inplace __version__ ../vm_supervisor/__init__.py
 
 build-dir:
 	mkdir -p target
diff --git a/packaging/version_from_git.py b/packaging/version_from_git.py
index 0c60b0b43..f64fc263d 100755
--- a/packaging/version_from_git.py
+++ b/packaging/version_from_git.py
@@ -46,8 +46,10 @@ def get_git_version():
     updated_content = re.sub(r"(Version:)\w*(.*)", "\\1 {}".format(version), target_content)
 elif format_ == 'setup.py':
     updated_content = re.sub(r"(version)\w*=(.*)'", "\\1='{}'".format(version), target_content)
+elif format_ == '__version__':
+    updated_content = re.sub(r"(__version__)\w*(.*)", "\\1 = '{}'".format(version), target_content)
 else:
-    print("Format must be 'deb' or 'setup.py', not '{}'".format(format_))
+    print("Format must be 'deb', 'setup.py' or '__version__', not '{}'".format(format_))
 
 if '--inplace' in args:
     with open(target_file_path, 'w') as target_file:
diff --git a/vm_supervisor/__init__.py b/vm_supervisor/__init__.py
index 8996cc760..6692f08ad 100644
--- a/vm_supervisor/__init__.py
+++ b/vm_supervisor/__init__.py
@@ -1 +1,12 @@
+from subprocess import check_output
+
+
+def get_version_from_git() -> str:
+    return check_output(("git", "describe", "--tags")).strip().decode()
+
+
+# The version number is harcoded in the following line when packaging the software
+__version__ = get_version_from_git()
+
+
 from . import supervisor
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 446cba6b4..7735546c5 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -7,9 +7,11 @@
 """
 import logging
 from secrets import token_urlsafe
+from typing import Awaitable, Callable
 
 from aiohttp import web
 
+from . import __version__
 from .conf import settings
 from .run import pool
 from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task
@@ -21,11 +23,22 @@
     about_config,
     status_check_fastapi,
 )
-from .run import pool
 
 logger = logging.getLogger(__name__)
 
-app = web.Application()
+
+@web.middleware
+async def server_version_middleware(
+        request: web.Request,
+        handler: Callable[[web.Request], Awaitable[web.StreamResponse]]
+) -> web.StreamResponse:
+    """Add the version of Aleph-VM in the HTTP headers of the responses.
+    """
+    resp: web.StreamResponse = await handler(request)
+    resp.headers.update({'Server': f"aleph-vm/{__version__}"},)
+    return resp
+
+app = web.Application(middlewares=[server_version_middleware])
 
 app.add_routes(
     [

From e55c73a0f838f64a90b87f2117aaab0f09a49bf8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 24 Mar 2022 11:31:12 +0100
Subject: [PATCH 266/990] Problem: Developers had no easy way to know which
 Python dependencies to use

Solution: Extract Python dependencies from Debian and Ubuntu container images.
---
 packaging/Makefile                      | 26 +++++++++++++++++++++----
 packaging/aleph-vm/DEBIAN/postinst      |  9 ++++++---
 packaging/extract_requirements.sh       |  8 ++++++++
 packaging/requirements-debian-11.txt    | 17 ++++++++++++++++
 packaging/requirements-ubuntu-20.04.txt | 18 +++++++++++++++++
 5 files changed, 71 insertions(+), 7 deletions(-)
 create mode 100755 packaging/extract_requirements.sh
 create mode 100644 packaging/requirements-debian-11.txt
 create mode 100644 packaging/requirements-ubuntu-20.04.txt

diff --git a/packaging/Makefile b/packaging/Makefile
index 83ef366c7..ce04395d2 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -57,27 +57,45 @@ clean:
 	rm -fr ./aleph-vm/opt/aleph-vm/
 
 all-podman-debian-11: version
-	cd .. && podman build -t localhost/aleph-vm-packaging:latest -f ./packaging/debian-11.dockerfile .
+	cd .. && podman build -t localhost/aleph-vm-packaging-debian-11:latest -f ./packaging/debian-11.dockerfile .
 	mkdir -p ./target
 	podman run --rm -ti \
 		-w /opt/packaging \
 		-v ./target:/opt/packaging/target \
-		localhost/aleph-vm-packaging:latest \
+		localhost/aleph-vm-packaging-debian-11:latest \
 		make
 	file target/aleph-vm.deb
 	mv target/aleph-vm.deb target/aleph-vm.debian-11.deb
 
 all-podman-ubuntu-2004: version
-	cd .. && podman build -t localhost/aleph-vm-packaging:latest -f ./packaging/ubuntu-20.04.dockerfile .
+	cd .. && podman build -t localhost/aleph-vm-packaging-ubuntu-2004:latest -f ./packaging/ubuntu-20.04.dockerfile .
 	mkdir -p ./target
 	podman run --rm -ti \
 		-w /opt/packaging \
 		-v ./target:/opt/packaging/target \
-		localhost/aleph-vm-packaging:latest \
+		localhost/aleph-vm-packaging-ubuntu-2004:latest \
 		make
 	file target/aleph-vm.deb
 	mv target/aleph-vm.deb target/aleph-vm.ubuntu-20.04.deb
 
+# extract Python requirements from Debian 11 container
+requirements-debian-11: all-podman-debian-11
+	podman run --rm -ti \
+		-v ./target/aleph-vm.debian-11.deb:/opt/packaging/target/aleph-vm.deb:ro \
+		-v ./extract_requirements.sh:/opt/extract_requirements.sh:ro \
+		-v ./requirements-debian-11.txt:/mnt/requirements-debian-11.txt \
+		debian:bullseye \
+		bash -c "/opt/extract_requirements.sh /mnt/requirements-debian-11.txt"
+
+# extract Python requirements from Ubuntu 20.04 container
+requirements-ubuntu-2004: all-podman-ubuntu-2004
+	podman run --rm -ti \
+		-v ./target/aleph-vm.ubuntu-20.04.deb:/opt/packaging/target/aleph-vm.deb:ro \
+		-v ./extract_requirements.sh:/opt/extract_requirements.sh:ro \
+		-v ./requirements-ubuntu-20.04.txt:/mnt/requirements-ubuntu-20.04.txt \
+		ubuntu:focal \
+		bash -c "/opt/extract_requirements.sh /mnt/requirements-ubuntu-20.04.txt"
+
 # run on host in order to sign with GPG
 repository-bullseye:
 	cd ./repositories/bullseye && reprepro -Vb . includedeb bullseye ../../target/aleph-vm.debian-11.deb && cd ..
diff --git a/packaging/aleph-vm/DEBIAN/postinst b/packaging/aleph-vm/DEBIAN/postinst
index 4bf426224..cfc978868 100755
--- a/packaging/aleph-vm/DEBIAN/postinst
+++ b/packaging/aleph-vm/DEBIAN/postinst
@@ -7,6 +7,9 @@ fi
 
 mkdir -p /srv/jailer
 
-systemctl daemon-reload
-systemctl enable aleph-vm-supervisor.service
-systemctl restart aleph-vm-supervisor.service
+# Systemd is absent from containers
+if ! [[ -v container ]]; then
+  systemctl daemon-reload
+  systemctl enable aleph-vm-supervisor.service
+  systemctl restart aleph-vm-supervisor.service
+fi
diff --git a/packaging/extract_requirements.sh b/packaging/extract_requirements.sh
new file mode 100755
index 000000000..aad72b7de
--- /dev/null
+++ b/packaging/extract_requirements.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+set -euf -o pipefail
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt update
+apt --yes install /opt/packaging/target/aleph-vm.deb
+pip freeze > $1
diff --git a/packaging/requirements-debian-11.txt b/packaging/requirements-debian-11.txt
new file mode 100644
index 000000000..ffa559d8d
--- /dev/null
+++ b/packaging/requirements-debian-11.txt
@@ -0,0 +1,17 @@
+aiodns==2.0.0
+aiohttp==3.7.4
+aioredis==1.3.1
+async-timeout==3.0.1
+attrs==20.3.0
+chardet==4.0.0
+hiredis==1.0.1
+idna==2.10
+msgpack==1.0.0
+multidict==5.1.0
+psutil==5.8.0
+pycares==3.1.1
+redis==3.5.3
+setproctitle==1.2.1
+SQLAlchemy==1.3.22
+typing-extensions==3.7.4.3
+yarl==1.6.3
diff --git a/packaging/requirements-ubuntu-20.04.txt b/packaging/requirements-ubuntu-20.04.txt
new file mode 100644
index 000000000..9aa01b5a3
--- /dev/null
+++ b/packaging/requirements-ubuntu-20.04.txt
@@ -0,0 +1,18 @@
+aiodns==2.0.0
+aiohttp==3.6.2
+aioredis==1.3.1
+async-timeout==3.0.1
+attrs==19.3.0
+chardet==3.0.4
+dbus-python==1.2.16
+hiredis==1.0.0
+idna==2.8
+msgpack==0.6.2
+multidict==4.7.3
+psutil==5.5.1
+pycares==3.1.1
+PyGObject==3.36.0
+redis==3.3.11
+setproctitle==1.1.10
+SQLAlchemy==1.3.12
+yarl==1.4.2

From 80ee7b79a4886d7fdc82fd2bb66fa24128694ca2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 28 Mar 2022 10:55:46 +0000
Subject: [PATCH 267/990] Build(deps): Bump psutil from 5.5.1 to 5.6.6 in
 /packaging

Bumps [psutil](https://github.com/giampaolo/psutil) from 5.5.1 to 5.6.6.
- [Release notes](https://github.com/giampaolo/psutil/releases)
- [Changelog](https://github.com/giampaolo/psutil/blob/master/HISTORY.rst)
- [Commits](https://github.com/giampaolo/psutil/compare/release-5.5.1...release-5.6.6)

---
updated-dependencies:
- dependency-name: psutil
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
---
 packaging/requirements-debian-11.txt    | 2 +-
 packaging/requirements-ubuntu-20.04.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packaging/requirements-debian-11.txt b/packaging/requirements-debian-11.txt
index ffa559d8d..e6774d4f8 100644
--- a/packaging/requirements-debian-11.txt
+++ b/packaging/requirements-debian-11.txt
@@ -8,7 +8,7 @@ hiredis==1.0.1
 idna==2.10
 msgpack==1.0.0
 multidict==5.1.0
-psutil==5.8.0
+psutil==5.6.6
 pycares==3.1.1
 redis==3.5.3
 setproctitle==1.2.1
diff --git a/packaging/requirements-ubuntu-20.04.txt b/packaging/requirements-ubuntu-20.04.txt
index 9aa01b5a3..89201658e 100644
--- a/packaging/requirements-ubuntu-20.04.txt
+++ b/packaging/requirements-ubuntu-20.04.txt
@@ -9,7 +9,7 @@ hiredis==1.0.0
 idna==2.8
 msgpack==0.6.2
 multidict==4.7.3
-psutil==5.5.1
+psutil==5.6.6
 pycares==3.1.1
 PyGObject==3.36.0
 redis==3.3.11

From eb03e3fb1c21814788632e3427bdcc7f67341a4a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 28 Mar 2022 10:55:04 +0000
Subject: [PATCH 268/990] Build(deps): Bump aiohttp from 3.6.2 to 3.7.4 in
 /packaging

Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.6.2 to 3.7.4.
- [Release notes](https://github.com/aio-libs/aiohttp/releases)
- [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst)
- [Commits](https://github.com/aio-libs/aiohttp/compare/v3.6.2...v3.7.4)

---
updated-dependencies:
- dependency-name: aiohttp
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
---
 packaging/requirements-ubuntu-20.04.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/requirements-ubuntu-20.04.txt b/packaging/requirements-ubuntu-20.04.txt
index 89201658e..82f134c03 100644
--- a/packaging/requirements-ubuntu-20.04.txt
+++ b/packaging/requirements-ubuntu-20.04.txt
@@ -1,5 +1,5 @@
 aiodns==2.0.0
-aiohttp==3.6.2
+aiohttp==3.7.4
 aioredis==1.3.1
 async-timeout==3.0.1
 attrs==19.3.0

From ba9ea3f2f1680ecfb05024927531f2694acdd6b6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 24 Mar 2022 11:54:06 +0100
Subject: [PATCH 269/990] Cleanup: Remove unnecessary type annotations

---
 vm_supervisor/conf.py | 46 +++++++++++++++++++++----------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 7574a32cf..ab3696bdb 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -58,7 +58,7 @@ def systemd_resolved_dns_servers(interface):
 
 
 class Settings(BaseSettings):
-    SUPERVISOR_HOST: str = "127.0.0.1"
+    SUPERVISOR_HOST = "127.0.0.1"
     SUPERVISOR_PORT: int = 4020
 
     # Public domain name
@@ -70,32 +70,32 @@ class Settings(BaseSettings):
     START_ID_INDEX: int = 4
     PREALLOC_VM_COUNT: int = 0
     REUSE_TIMEOUT: float = 60 * 60.0
-    WATCH_FOR_MESSAGES: bool = True
-    WATCH_FOR_UPDATES: bool = True
-    NETWORK_INTERFACE: str = "eth0"
+    WATCH_FOR_MESSAGES = True
+    WATCH_FOR_UPDATES = True
+    NETWORK_INTERFACE = "eth0"
     DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf
     DNS_NAMESERVERS: Optional[List[str]] = None
 
-    API_SERVER: str = "https://api2.aleph.im"
-    USE_JAILER: bool = True
+    API_SERVER = "https://api2.aleph.im"
+    USE_JAILER = True
     # System logs make boot ~2x slower
-    PRINT_SYSTEM_LOGS: bool = False
+    PRINT_SYSTEM_LOGS = False
     # Networking does not work inside Docker/Podman
-    ALLOW_VM_NETWORKING: bool = True
-    FIRECRACKER_PATH: str = "/opt/firecracker/firecracker"
-    JAILER_PATH: str = "/opt/firecracker/jailer"
-    LINUX_PATH: str = "/opt/firecracker/vmlinux.bin"
-    INIT_TIMEOUT: float = 20
+    ALLOW_VM_NETWORKING = True
+    FIRECRACKER_PATH = "/opt/firecracker/firecracker"
+    JAILER_PATH = "/opt/firecracker/jailer"
+    LINUX_PATH = "/opt/firecracker/vmlinux.bin"
+    INIT_TIMEOUT: float = 20.
 
-    CONNECTOR_URL: Url = Url("http://localhost:4021")
+    CONNECTOR_URL = Url("http://localhost:4021")
 
-    CACHE_ROOT: FilePath = FilePath("/var/cache/aleph/vm")
-    MESSAGE_CACHE: FilePath = FilePath(join(CACHE_ROOT, "message"))
-    CODE_CACHE: FilePath = FilePath(join(CACHE_ROOT, "code"))
-    RUNTIME_CACHE: FilePath = FilePath(join(CACHE_ROOT, "runtime"))
-    DATA_CACHE: FilePath = FilePath(join(CACHE_ROOT, "data"))
+    CACHE_ROOT = FilePath("/var/cache/aleph/vm")
+    MESSAGE_CACHE = FilePath(join(CACHE_ROOT, "message"))
+    CODE_CACHE = FilePath(join(CACHE_ROOT, "code"))
+    RUNTIME_CACHE = FilePath(join(CACHE_ROOT, "runtime"))
+    DATA_CACHE = FilePath(join(CACHE_ROOT, "data"))
 
-    PERSISTENT_VOLUMES_DIR: FilePath = FilePath(
+    PERSISTENT_VOLUMES_DIR = FilePath(
         join("/var/lib/aleph/vm/volumes", "volumes", "persistent")
     )
 
@@ -103,24 +103,24 @@ class Settings(BaseSettings):
     MAX_DATA_ARCHIVE_SIZE = 10_000_000  # 10 MB
 
     FAKE_DATA_PROGRAM: Optional[FilePath] = None
-    BENCHMARK_FAKE_DATA_PROGRAM: FilePath = FilePath(
+    BENCHMARK_FAKE_DATA_PROGRAM = FilePath(
         abspath(join(__file__, "../../examples/example_fastapi"))
     )
 
-    FAKE_DATA_MESSAGE: FilePath = FilePath(
+    FAKE_DATA_MESSAGE = FilePath(
         abspath(join(__file__, "../../examples/message_from_aleph.json"))
     )
     FAKE_DATA_DATA: Optional[FilePath] = FilePath(
         abspath(join(__file__, "../../examples/data/"))
     )
-    FAKE_DATA_RUNTIME: FilePath = FilePath(
+    FAKE_DATA_RUNTIME = FilePath(
         abspath(join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs"))
     )
     FAKE_DATA_VOLUME: Optional[FilePath] = FilePath(
         abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs"))
     )
 
-    CHECK_FASTAPI_VM_ID: str = (
+    CHECK_FASTAPI_VM_ID = (
         "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
     )
 

From 2bbc59fa2cea408233726246a7db7db051b607a7 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 22 Mar 2022 11:27:43 +0100
Subject: [PATCH 270/990] Feature: Record VM execution times and log details on
 files.

The resources used by a VM during its execution are recorded in an SQLite database.

The detailed information about VM executions can optionally be stored in JSON files on disk.
---
 packaging/aleph-vm/DEBIAN/control |  2 +-
 vm_supervisor/conf.py             |  5 ++
 vm_supervisor/metrics.py          | 99 +++++++++++++++++++++++++++++++
 vm_supervisor/models.py           | 37 +++++++++++-
 vm_supervisor/supervisor.py       |  3 +-
 vm_supervisor/utils.py            | 11 ++--
 vm_supervisor/views.py            |  9 +++
 7 files changed, 159 insertions(+), 7 deletions(-)
 create mode 100644 vm_supervisor/metrics.py

diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index 5dbc339e8..59ccd5a63 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap
 Section: aleph-im
 Priority: Extra
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index ab3696bdb..f340a5aea 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -95,6 +95,11 @@ class Settings(BaseSettings):
     RUNTIME_CACHE = FilePath(join(CACHE_ROOT, "runtime"))
     DATA_CACHE = FilePath(join(CACHE_ROOT, "data"))
 
+    EXECUTION_ROOT: FilePath = FilePath("/var/lib/aleph/vm")
+    EXECUTION_DATABASE: FilePath = FilePath(join(EXECUTION_ROOT, "executions.sqlite3"))
+    EXECUTION_LOG_ENABLED: bool = False
+    EXECUTION_LOG_DIRECTORY: FilePath = FilePath(join(EXECUTION_ROOT, "executions"))
+
     PERSISTENT_VOLUMES_DIR = FilePath(
         join("/var/lib/aleph/vm/volumes", "volumes", "persistent")
     )
diff --git a/vm_supervisor/metrics.py b/vm_supervisor/metrics.py
new file mode 100644
index 000000000..91dd905f5
--- /dev/null
+++ b/vm_supervisor/metrics.py
@@ -0,0 +1,99 @@
+import logging
+import os
+from os.path import join
+from typing import Optional, Iterable
+from uuid import UUID
+
+from sqlalchemy import Column, Integer, String, Float, DateTime
+from sqlalchemy import create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+
+from .conf import settings
+
+logger = logging.getLogger(__name__)
+
+
+Base = declarative_base()
+session_maker: Optional[sessionmaker] = None
+
+
+def get_database_sessionmaker() -> sessionmaker:
+    global session_maker
+    if session_maker:
+        return session_maker
+
+    # engine = create_engine('sqlite:///:memory:', echo=True)
+    engine = create_engine(f"sqlite:///{settings.EXECUTION_DATABASE}", echo=True)
+    Base.metadata.create_all(engine)
+    session_maker = sessionmaker(bind=engine)
+    return session_maker
+
+
+class ExecutionRecord(Base):
+    __tablename__ = "records"
+
+    uuid = Column(String, primary_key=True)
+    vm_hash = Column(String)
+
+    time_defined = Column(DateTime)
+    time_prepared = Column(DateTime)
+    time_started = Column(DateTime)
+    time_stopping = Column(DateTime)
+
+    cpu_time_user = Column(Float)
+    cpu_time_system = Column(Float)
+
+    io_read_count = Column(Integer)
+    io_write_count = Column(Integer)
+    io_read_bytes = Column(Integer)
+    io_write_bytes = Column(Integer)
+
+    vcpus = Column(Integer)
+    memory = Column(Integer)
+    network_tap = Column(String, nullable=True)
+
+    def __repr__(self):
+        return f""
+
+    def to_dict(self):
+        return {
+            "uuid": self.uuid,
+            "vm_hash": self.vm_hash,
+            "time_defined": self.time_defined,
+            "time_prepared": self.time_prepared,
+            "time_started": self.time_started,
+            "time_stopping": self.time_stopping,
+            "cpu_time_user": self.cpu_time_user,
+            "cpu_time_system": self.cpu_time_system,
+            "io_read_count": self.io_read_count,
+            "io_write_count": self.io_write_count,
+            "io_read_bytes": self.io_read_bytes,
+            "io_write_bytes": self.io_write_bytes,
+            "vcpus": self.vcpus,
+            "memory": self.memory,
+            "network_tap": self.network_tap,
+        }
+
+
+async def save_execution_data(execution_uuid: UUID, execution_data: str):
+    """Save the execution data in a file on disk"""
+    os.makedirs(settings.EXECUTION_LOG_DIRECTORY, exist_ok=True)
+    filepath = join(settings.EXECUTION_LOG_DIRECTORY, f"{execution_uuid}.json")
+    with open(filepath, "w") as fd:
+        fd.write(execution_data)
+
+
+async def save_record(record: ExecutionRecord):
+    """Record the resource usage in database"""
+    sessionmaker = get_database_sessionmaker()
+    session = sessionmaker()
+    session.add(record)
+    session.commit()
+
+
+async def get_execution_records() -> Iterable[ExecutionRecord]:
+    """Get the execution records from the database."""
+    sessionmaker = get_database_sessionmaker()
+    session = sessionmaker()
+    return session.query(ExecutionRecord).all()
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 9b9966edf..e95062979 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -1,15 +1,20 @@
 import asyncio
 import logging
 import sys
+import uuid
 from asyncio import Task
 from dataclasses import dataclass
 from datetime import datetime
 from typing import NewType, Optional, Dict
 
 from aleph_message.models import ProgramContent
+
+from .metrics import save_record, save_execution_data, ExecutionRecord
 from .pubsub import PubSub
+from .utils import dumps_for_json
 from .vm import AlephFirecrackerVM
 from .vm.firecracker_microvm import AlephFirecrackerResources
+from .conf import settings
 
 logger = logging.getLogger(__name__)
 
@@ -36,7 +41,7 @@ class VmExecution:
 
     Implementation agnostic (Firecracker, maybe WASM in the future, ...).
     """
-
+    uuid: uuid.UUID  # Unique identifier of this execution
     vm_hash: VmHash
     original: ProgramContent
     program: ProgramContent
@@ -61,6 +66,7 @@ def becomes_ready(self):
     def __init__(
         self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent
     ):
+        self.uuid = uuid.uuid1()  # uuid1() includes the hardware address and timestamp
         self.vm_hash = vm_hash
         self.program = program
         self.original = original
@@ -75,6 +81,9 @@ def to_dict(self) -> Dict:
             **self.__dict__,
         }
 
+    def to_json(self, indent: Optional[int] = None) -> str:
+        return dumps_for_json(self.to_dict(), indent=indent)
+
     async def prepare(self):
         """Download VM required files"""
         self.times.preparing_at = datetime.now()
@@ -143,6 +152,7 @@ async def stop(self):
             return
         await self.all_runs_complete()
         self.times.stopping_at = datetime.now()
+        await self.record_usage()
         await self.vm.teardown()
         self.times.stopped_at = datetime.now()
         self.cancel_expiration()
@@ -174,6 +184,31 @@ async def all_runs_complete(self):
             logger.debug("Stop: waiting for runs to complete...")
             await self.runs_done_event.wait()
 
+    async def record_usage(self):
+        if settings.EXECUTION_LOG_ENABLED:
+            await save_execution_data(
+                execution_uuid=self.uuid,
+                execution_data=self.to_json()
+            )
+        pid_info = self.vm.to_dict()
+        await save_record(ExecutionRecord(
+            uuid=str(self.uuid),
+            vm_hash=self.vm_hash,
+            time_defined=self.times.defined_at,
+            time_prepared=self.times.prepared_at,
+            time_started=self.times.started_at,
+            time_stopping=self.times.stopping_at,
+            cpu_time_user=pid_info["process"]["cpu_times"].user,
+            cpu_time_system=pid_info["process"]["cpu_times"].system,
+            io_read_count=pid_info["process"]["io_counters"][0],
+            io_write_count=pid_info["process"]["io_counters"][1],
+            io_read_bytes=pid_info["process"]["io_counters"][2],
+            io_write_bytes=pid_info["process"]["io_counters"][3],
+            vcpus=self.vm.hardware_resources.vcpus,
+            memory=self.vm.hardware_resources.memory,
+            network_tap=self.vm.fvm.network_tap,
+        ))
+
     async def run_code(self, scope: dict = None) -> bytes:
         if not self.vm:
             raise ValueError("The VM has not been created yet")
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 7735546c5..49364e85e 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -21,7 +21,7 @@
     about_login,
     about_executions,
     about_config,
-    status_check_fastapi,
+    status_check_fastapi, about_execution_records,
 )
 
 logger = logging.getLogger(__name__)
@@ -44,6 +44,7 @@ async def server_version_middleware(
     [
         web.get("/about/login", about_login),
         web.get("/about/executions", about_executions),
+        web.get("/about/executions/records", about_execution_records),
         web.get("/about/config", about_config),
         web.get("/status/check/fastapi", status_check_fastapi),
         web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index e925dbeb8..3af8170f1 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -1,6 +1,7 @@
 import json
 from base64 import b32decode, b16encode
-from typing import Any
+from dataclasses import is_dataclass, asdict as dataclass_as_dict
+from typing import Any, Optional
 
 import aiodns
 
@@ -20,13 +21,15 @@ async def get_ref_from_dns(domain):
 
 
 def to_json(o: Any):
-    if hasattr(o, "to_dict"):  # dataclasses
+    if hasattr(o, "to_dict"):  # default method
         return o.to_dict()
     elif hasattr(o, "dict"):  # Pydantic
         return o.dict()
+    elif is_dataclass(o):
+        return dataclass_as_dict(o)
     else:
         return str(o)
 
 
-def dumps_for_json(o: Any):
-    return json.dumps(o, default=to_json)
+def dumps_for_json(o: Any, indent: Optional[int]=None):
+    return json.dumps(o, default=to_json, indent=indent)
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index a7d13d9c3..e11f8d9b3 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -11,6 +11,7 @@
 
 from . import status
 from .conf import settings
+from .metrics import get_execution_records
 from .models import VmHash
 from .run import run_code_on_request, pool
 from .utils import b32_to_b16, get_ref_from_dns, dumps_for_json
@@ -101,6 +102,14 @@ async def about_config(request: web.Request) -> web.Response:
     )
 
 
+async def about_execution_records(request: web.Request):
+    records = await get_execution_records()
+    return web.json_response(
+        records,
+        dumps=dumps_for_json,
+    )
+
+
 async def index(request: web.Request):
     assert request.method == "GET"
     path = os.path.join(os.path.dirname(__file__), 'templates/index.html')

From 227ecd3266ea26b28ab8e1ab714cc5e80f11eb0e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 24 Mar 2022 15:30:32 +0100
Subject: [PATCH 271/990] WIP: In response to PR comments

---
 vm_supervisor/conf.py       |  8 ++---
 vm_supervisor/metrics.py    | 66 +++++++++++++++----------------------
 vm_supervisor/supervisor.py |  4 +++
 3 files changed, 35 insertions(+), 43 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index f340a5aea..594e99a0e 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -95,10 +95,10 @@ class Settings(BaseSettings):
     RUNTIME_CACHE = FilePath(join(CACHE_ROOT, "runtime"))
     DATA_CACHE = FilePath(join(CACHE_ROOT, "data"))
 
-    EXECUTION_ROOT: FilePath = FilePath("/var/lib/aleph/vm")
-    EXECUTION_DATABASE: FilePath = FilePath(join(EXECUTION_ROOT, "executions.sqlite3"))
-    EXECUTION_LOG_ENABLED: bool = False
-    EXECUTION_LOG_DIRECTORY: FilePath = FilePath(join(EXECUTION_ROOT, "executions"))
+    EXECUTION_ROOT = FilePath("/var/lib/aleph/vm")
+    EXECUTION_DATABASE = FilePath(join(EXECUTION_ROOT, "executions.sqlite3"))
+    EXECUTION_LOG_ENABLED = False
+    EXECUTION_LOG_DIRECTORY = FilePath(join(EXECUTION_ROOT, "executions"))
 
     PERSISTENT_VOLUMES_DIR = FilePath(
         join("/var/lib/aleph/vm/volumes", "volumes", "persistent")
diff --git a/vm_supervisor/metrics.py b/vm_supervisor/metrics.py
index 91dd905f5..479080763 100644
--- a/vm_supervisor/metrics.py
+++ b/vm_supervisor/metrics.py
@@ -1,42 +1,42 @@
 import logging
 import os
 from os.path import join
-from typing import Optional, Iterable
+from typing import Iterable
 from uuid import UUID
 
 from sqlalchemy import Column, Integer, String, Float, DateTime
 from sqlalchemy import create_engine
+from sqlalchemy.engine import Engine
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 
 from .conf import settings
 
-logger = logging.getLogger(__name__)
+Session: sessionmaker
 
+logger = logging.getLogger(__name__)
 
 Base = declarative_base()
-session_maker: Optional[sessionmaker] = None
-
 
-def get_database_sessionmaker() -> sessionmaker:
-    global session_maker
-    if session_maker:
-        return session_maker
 
-    # engine = create_engine('sqlite:///:memory:', echo=True)
+def setup_engine():
+    global Session
     engine = create_engine(f"sqlite:///{settings.EXECUTION_DATABASE}", echo=True)
+    Session = sessionmaker(bind=engine)
+    return engine
+
+
+def create_tables(engine: Engine):
     Base.metadata.create_all(engine)
-    session_maker = sessionmaker(bind=engine)
-    return session_maker
 
 
 class ExecutionRecord(Base):
     __tablename__ = "records"
 
     uuid = Column(String, primary_key=True)
-    vm_hash = Column(String)
+    vm_hash = Column(String, nullable=False)
 
-    time_defined = Column(DateTime)
+    time_defined = Column(DateTime, nullable=False)
     time_prepared = Column(DateTime)
     time_started = Column(DateTime)
     time_stopping = Column(DateTime)
@@ -49,31 +49,15 @@ class ExecutionRecord(Base):
     io_read_bytes = Column(Integer)
     io_write_bytes = Column(Integer)
 
-    vcpus = Column(Integer)
-    memory = Column(Integer)
+    vcpus = Column(Integer, nullable=False)
+    memory = Column(Integer, nullable=False)
     network_tap = Column(String, nullable=True)
 
     def __repr__(self):
         return f""
 
     def to_dict(self):
-        return {
-            "uuid": self.uuid,
-            "vm_hash": self.vm_hash,
-            "time_defined": self.time_defined,
-            "time_prepared": self.time_prepared,
-            "time_started": self.time_started,
-            "time_stopping": self.time_stopping,
-            "cpu_time_user": self.cpu_time_user,
-            "cpu_time_system": self.cpu_time_system,
-            "io_read_count": self.io_read_count,
-            "io_write_count": self.io_write_count,
-            "io_read_bytes": self.io_read_bytes,
-            "io_write_bytes": self.io_write_bytes,
-            "vcpus": self.vcpus,
-            "memory": self.memory,
-            "network_tap": self.network_tap,
-        }
+        return {c.name: getattr(self, c.name) for c in self.__table__.c}
 
 
 async def save_execution_data(execution_uuid: UUID, execution_data: str):
@@ -86,14 +70,18 @@ async def save_execution_data(execution_uuid: UUID, execution_data: str):
 
 async def save_record(record: ExecutionRecord):
     """Record the resource usage in database"""
-    sessionmaker = get_database_sessionmaker()
-    session = sessionmaker()
-    session.add(record)
-    session.commit()
+    session = Session()
+    try:
+        session.add(record)
+        session.commit()
+    finally:
+        session.close()
 
 
 async def get_execution_records() -> Iterable[ExecutionRecord]:
     """Get the execution records from the database."""
-    sessionmaker = get_database_sessionmaker()
-    session = sessionmaker()
-    return session.query(ExecutionRecord).all()
+    session = Session()
+    try:
+        return session.query(ExecutionRecord).all()
+    finally:
+        session.close()
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 49364e85e..9cedfb739 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -12,6 +12,7 @@
 from aiohttp import web
 
 from . import __version__
+from . import metrics
 from .conf import settings
 from .run import pool
 from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task
@@ -68,6 +69,9 @@ def run():
     app["secret_token"] = secret_token
     print(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
 
+    engine = metrics.setup_engine()
+    metrics.create_tables(engine)
+
     if settings.WATCH_FOR_MESSAGES:
         app.on_startup.append(start_watch_for_messages_task)
         app.on_cleanup.append(stop_watch_for_messages_task)

From b9d93a524de4fe041cb38c37a6a79f7933c5d0cd Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 10 Mar 2022 22:50:51 +0100
Subject: [PATCH 272/990] Fix: Querying index with HTTP HEAD raised 500 due to
 AssertionError

Solution: Only allow HTTP method GET to access the index page.
---
 vm_supervisor/views.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index e11f8d9b3..50d3d99f1 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -42,7 +42,9 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
     we expect the hash to be encoded in base32 instead of hexadecimal. Padding is added
     automatically.
     """
-    if request.host.split(':')[0] == settings.DOMAIN_NAME:
+    if request.host.split(':')[0] == settings.DOMAIN_NAME \
+            and request.method == "GET" \
+            and request.path == "/":
         # Serve the index page
         return await index(request=request)
 

From 34c179ad65ded80900d342d6e8c5561c8d1c70bc Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 28 Mar 2022 19:46:37 +0200
Subject: [PATCH 273/990] Dependency: Upgrade Firecracker to v1.0.0

---
 firecracker/config.py   | 2 +-
 packaging/Makefile      | 2 +-
 vm_supervisor/README.md | 7 +++----
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/firecracker/config.py b/firecracker/config.py
index 4c906e8b6..01acb4498 100644
--- a/firecracker/config.py
+++ b/firecracker/config.py
@@ -32,7 +32,7 @@ class Drive(BaseModel):
 class MachineConfig(BaseModel):
     vcpu_count: PositiveInt = 1
     mem_size_mib: PositiveInt = 128
-    ht_enabled: bool = False
+    smt: bool = False
 
 
 class Vsock(BaseModel):
diff --git a/packaging/Makefile b/packaging/Makefile
index ce04395d2..4b6b7e750 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -29,7 +29,7 @@ debian-package-resources: firecracker-bins vmlinux
 firecracker-bins: target-dir build-dir
 	mkdir -p ./build/firecracker-release
 	# Download latest release
-	curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v0.25.2/firecracker-v0.25.2-x86_64.tgz | tar -xz --directory ./build/firecracker-release
+	curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.0.0/firecracker-v1.0.0-x86_64.tgz | tar -xz --directory ./build/firecracker-release
 	# Copy binaries:
 	cp ./build/firecracker-release/release-v*/firecracker-v* ./target/firecracker
 	cp ./build/firecracker-release/release-v*/jailer-v* ./target/jailer
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index 90c291165..3d505b65c 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -65,12 +65,11 @@ useradd jailman
 from the [Firecracker project releases](https://github.com/firecracker-microvm/firecracker/releases):
 ```shell
 mkdir /opt/firecracker
-chown $(whoami) /opt/firecracker
-curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v0.24.2/firecracker-v0.24.2-x86_64.tgz | tar -xz --directory /opt/firecracker
+curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.0.0/firecracker-v1.0.0-x86_64.tgz | tar -xz --directory /opt/firecracker
 
 # Link binaries on version-agnostic paths:
-ln /opt/firecracker/firecracker-v* /opt/firecracker/firecracker
-ln /opt/firecracker/jailer-v* /opt/firecracker/jailer
+ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker
+ln /opt/firecracker/release-*/jailer-v* /opt/firecracker/jailer
 ```
 
 ### 2.d. Clone this repository on the host machine and enter it.

From 29c18b3186b74c75bf04067f2e17ce2bcf89dd72 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 28 Mar 2022 18:55:58 +0200
Subject: [PATCH 274/990] Fix: Service was not stopped during upgrade

---
 packaging/aleph-vm/DEBIAN/preinst | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 packaging/aleph-vm/DEBIAN/preinst

diff --git a/packaging/aleph-vm/DEBIAN/preinst b/packaging/aleph-vm/DEBIAN/preinst
new file mode 100644
index 000000000..5dd10fac4
--- /dev/null
+++ b/packaging/aleph-vm/DEBIAN/preinst
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -uf -o pipefail
+
+# Systemd is absent from containers
+if ! [[ -v container ]]; then
+  # Stop the service during an upgrade.
+  # The service does not exist during a new install and will fail, this is okay
+  systemctl stop aleph-vm-supervisorz.service
+fi
+
+set -e

From cd1e108c1d2afd3275d0ef259b0c2f42292681a4 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 28 Mar 2022 19:12:18 +0200
Subject: [PATCH 275/990] Change: Move jailer files from /srv/jailer to
 /var/lib/aleph/vm/jailer

---
 docker/vm_supervisor-dev.dockerfile | 2 +-
 firecracker/microvm.py              | 4 ++--
 packaging/aleph-vm/DEBIAN/postinst  | 3 ++-
 packaging/aleph-vm/DEBIAN/postrm    | 1 +
 vm_supervisor/README.md             | 2 +-
 5 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index ab8f5210c..31932a976 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -21,7 +21,7 @@ RUN ln /opt/firecracker/jailer-v* /opt/firecracker/jailer
 
 RUN pip3 install typing-extensions 'aleph-message>=0.1.18'
 
-RUN mkdir /srv/jailer
+RUN mkdir /var/lib/aleph/vm/jailer
 
 ENV PYTHONPATH /mnt
 
diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 6f3e8c646..11be26f4a 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -17,7 +17,7 @@
 logger = logging.getLogger(__name__)
 
 VSOCK_PATH = "/tmp/v.sock"
-
+JAILER_BASE_DIRECTORY = "/var/lib/aleph/vm/jailer"
 
 class MicroVMFailedInit(Exception):
     pass
@@ -74,7 +74,7 @@ class MicroVM:
     @property
     def namespace_path(self):
         firecracker_bin_name = os.path.basename(self.firecracker_bin_path)
-        return f"/srv/jailer/{firecracker_bin_name}/{self.vm_id}"
+        return f"{JAILER_BASE_DIRECTORY}/{firecracker_bin_name}/{self.vm_id}"
 
     @property
     def jailer_path(self):
diff --git a/packaging/aleph-vm/DEBIAN/postinst b/packaging/aleph-vm/DEBIAN/postinst
index cfc978868..1286c189d 100755
--- a/packaging/aleph-vm/DEBIAN/postinst
+++ b/packaging/aleph-vm/DEBIAN/postinst
@@ -5,7 +5,8 @@ if ! id -u jailman > /dev/null 2>&1; then
   useradd jailman
 fi
 
-mkdir -p /srv/jailer
+rm -r /srv/jailer  # Upgrade from < 0.1.11
+mkdir -p /var/lib/aleph/vm/jailer
 
 # Systemd is absent from containers
 if ! [[ -v container ]]; then
diff --git a/packaging/aleph-vm/DEBIAN/postrm b/packaging/aleph-vm/DEBIAN/postrm
index 0f62a0480..293e02e38 100755
--- a/packaging/aleph-vm/DEBIAN/postrm
+++ b/packaging/aleph-vm/DEBIAN/postrm
@@ -2,5 +2,6 @@
 set -euf -o pipefail
 
 rm -r /srv/jailer
+rm -r /var/lib/aleph/vm/jailer
 
 systemctl daemon-reload
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index 3d505b65c..379e6d711 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -93,7 +93,7 @@ pip3 install --update aleph-message
 ### 2.f. Create the jailer working directory:
 
 ```shell
-mkdir /srv/jailer
+mkdir -p /var/lib/aleph/vm/jailer
 ```
 
 ### 2.g. Download a Linux kernel

From 84236b1842efdc2f5b963aa475b1d91b0dd9fc30 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 28 Mar 2022 19:12:37 +0200
Subject: [PATCH 276/990] Fix: Manual install missed some packages

---
 vm_supervisor/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index 379e6d711..d72ee7e07 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -56,7 +56,7 @@ when running the VM Supervisor.
 
 ```shell
 apt update
-apt install -y git python3 python3-aiohttp python3-msgpack python3-aiodns redis python3-aioredis \
+apt install -y git python3 python3-aiohttp python3-msgpack python3-aiodns python3-sqlalchemy python3-setproctitle redis python3-aioredis \
  python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap
 useradd jailman
 ```
@@ -87,7 +87,7 @@ is used to parse and validate Aleph messages.
 ```shell
 apt install -y --no-install-recommends --no-install-suggests python3-pip
 pip3 install pydantic[dotenv]
-pip3 install --update aleph-message
+pip3 install aleph-message
 ```
 
 ### 2.f. Create the jailer working directory:

From a68e0c903a91e468aea9928ed3ab5c8da9c335c9 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 28 Mar 2022 19:13:18 +0200
Subject: [PATCH 277/990] Doc: Mention reboot since many installations ship
 outdated kernel

---
 doc/INSTALL-Debian-11.md          | 2 ++
 doc/INSTALL-Ubuntu-20.04.md       | 2 ++
 packaging/aleph-vm/DEBIAN/preinst | 0
 3 files changed, 4 insertions(+)
 mode change 100644 => 100755 packaging/aleph-vm/DEBIAN/preinst

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index 7126a52c6..9f79f02d9 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -41,6 +41,8 @@ wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.10/aleph
 apt install /opt/aleph-vm.debian-11.deb
 ```
 
+Reboot if required (new kernel, ...).
+
 ### Configuration
 
 Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
index 575625cf1..675774043 100644
--- a/doc/INSTALL-Ubuntu-20.04.md
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -41,6 +41,8 @@ sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.10/
 sudo apt install /opt/aleph-vm.ubuntu-20.04.deb
 ```
 
+Reboot if required (new kernel, ...).
+
 ### Configuration
 
 Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
diff --git a/packaging/aleph-vm/DEBIAN/preinst b/packaging/aleph-vm/DEBIAN/preinst
old mode 100644
new mode 100755

From bd438d1c94ff4dbd023c7bdfe197e4a932f28d10 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 29 Mar 2022 14:19:04 +0200
Subject: [PATCH 278/990] Cleanup: Use `black` formatter

---
 runtimes/aleph-alpine-3.13-python/init1.py | 144 ++++++++++++---------
 vm_connector/main.py                       |  10 +-
 vm_supervisor/__main__.py                  |   9 +-
 vm_supervisor/conf.py                      |  19 ++-
 vm_supervisor/messages.py                  |   8 +-
 vm_supervisor/models.py                    |  40 +++---
 vm_supervisor/pool.py                      |   7 +-
 vm_supervisor/run.py                       |   8 +-
 vm_supervisor/supervisor.py                |  20 ++-
 vm_supervisor/utils.py                     |   2 +-
 vm_supervisor/views.py                     |  21 +--
 vm_supervisor/vm/firecracker_microvm.py    |   6 +-
 12 files changed, 170 insertions(+), 124 deletions(-)

diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py
index 384e2845d..7f5b6652c 100644
--- a/runtimes/aleph-alpine-3.13-python/init1.py
+++ b/runtimes/aleph-alpine-3.13-python/init1.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3 -OO
 
 import logging
+
 logging.basicConfig(
     level=logging.DEBUG,
     format="%(relativeCreated)4f |V %(levelname)s | %(message)s",
@@ -29,7 +30,7 @@
 
 logger.debug("Imports finished")
 
-ASGIApplication = NewType('AsgiApplication', Any)
+ASGIApplication = NewType("AsgiApplication", Any)
 
 
 class Encoding(str, Enum):
@@ -105,8 +106,9 @@ def setup_variables(variables: Optional[Dict[str, str]]):
         os.environ[key] = value
 
 
-def setup_network(ip: Optional[str], route: Optional[str],
-                  dns_servers: Optional[List[str]] = None):
+def setup_network(
+    ip: Optional[str], route: Optional[str], dns_servers: Optional[List[str]] = None
+):
     """Setup the system with info from the host."""
     dns_servers = dns_servers or []
     if not os.path.exists("/sys/class/net/eth0"):
@@ -157,7 +159,9 @@ def setup_volumes(volumes: List[Volume]):
     system("mount")
 
 
-def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplication:
+def setup_code_asgi(
+    code: bytes, encoding: Encoding, entrypoint: str
+) -> ASGIApplication:
     # Allow importing packages from /opt/packages
     sys.path.append("/opt/packages")
 
@@ -167,7 +171,7 @@ def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApp
         module_name, app_name = entrypoint.split(":", 1)
         logger.debug("import module")
         module = __import__(module_name)
-        for level in module_name.split('.')[1:]:
+        for level in module_name.split(".")[1:]:
             module = getattr(module, level)
         app: ASGIApplication = getattr(module, app_name)
     elif encoding == Encoding.zip:
@@ -180,7 +184,7 @@ def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApp
         module_name, app_name = entrypoint.split(":", 1)
         logger.debug("import module")
         module = __import__(module_name)
-        for level in module_name.split('.')[1:]:
+        for level in module_name.split(".")[1:]:
             module = getattr(module, level)
         app: ASGIApplication = getattr(module, app_name)
     elif encoding == Encoding.plain:
@@ -193,7 +197,9 @@ def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApp
     return app
 
 
-def setup_code_executable(code: bytes, encoding: Encoding, entrypoint: str) -> subprocess.Popen:
+def setup_code_executable(
+    code: bytes, encoding: Encoding, entrypoint: str
+) -> subprocess.Popen:
     logger.debug("Extracting code")
     if encoding == Encoding.squashfs:
         path = f"/opt/code/{entrypoint}"
@@ -223,32 +229,38 @@ def setup_code_executable(code: bytes, encoding: Encoding, entrypoint: str) -> s
     return process
 
 
-def setup_code(code: bytes, encoding: Encoding, entrypoint: str, interface: Interface
-               ) -> Union[ASGIApplication, subprocess.Popen]:
+def setup_code(
+    code: bytes, encoding: Encoding, entrypoint: str, interface: Interface
+) -> Union[ASGIApplication, subprocess.Popen]:
 
     if interface == Interface.asgi:
         return setup_code_asgi(code=code, encoding=encoding, entrypoint=entrypoint)
     elif interface == Interface.executable:
-        return setup_code_executable(code=code, encoding=encoding, entrypoint=entrypoint)
+        return setup_code_executable(
+            code=code, encoding=encoding, entrypoint=entrypoint
+        )
     else:
         raise ValueError("Invalid interface. This should never happen.")
 
 
-async def run_python_code_http(application: ASGIApplication, scope: dict
-                               ) -> Tuple[Dict, Dict, str, Optional[bytes]]:
+async def run_python_code_http(
+    application: ASGIApplication, scope: dict
+) -> Tuple[Dict, Dict, str, Optional[bytes]]:
 
     logger.debug("Running code")
     with StringIO() as buf, redirect_stdout(buf):
         # Execute in the same process, saves ~20ms than a subprocess
 
         # The body should not be part of the ASGI scope itself
-        body: bytes = scope.pop('body')
+        body: bytes = scope.pop("body")
 
         async def receive():
-            type_ = 'http.request' if scope['type'] in ('http', 'websocket') else 'aleph.message'
-            return {'type': type_,
-                    'body': body,
-                    'more_body': False}
+            type_ = (
+                "http.request"
+                if scope["type"] in ("http", "websocket")
+                else "aleph.message"
+            )
+            return {"type": type_, "body": body, "more_body": False}
 
         send_queue: asyncio.Queue = asyncio.Queue()
 
@@ -261,7 +273,7 @@ async def send(dico):
 
         logger.debug("Waiting for headers")
         headers: Dict
-        if scope['type'] == 'http':
+        if scope["type"] == "http":
             headers = await send_queue.get()
         else:
             headers = {}
@@ -278,12 +290,12 @@ async def send(dico):
 
     logger.debug("Getting output data")
     output_data: bytes
-    if os.path.isdir('/data') and os.listdir('/data'):
-        make_archive("/opt/output", 'zip', "/data")
+    if os.path.isdir("/data") and os.listdir("/data"):
+        make_archive("/opt/output", "zip", "/data")
         with open("/opt/output.zip", "rb") as output_zipfile:
             output_data = output_zipfile.read()
     else:
-        output_data = b''
+        output_data = b""
 
     logger.debug("Returning result")
     return headers, body, output, output_data
@@ -291,21 +303,19 @@ async def send(dico):
 
 async def make_request(session, scope):
     async with session.request(
-                scope["method"],
-                url="http://localhost:8080{}".format(scope["path"]),
-                params=scope["query_string"],
-                headers=[(a.decode('utf-8'), b.decode('utf-8'))
-                         for a, b in scope['headers']],
-                data=scope.get("body", None)
-            ) as resp:
+        scope["method"],
+        url="http://localhost:8080{}".format(scope["path"]),
+        params=scope["query_string"],
+        headers=[(a.decode("utf-8"), b.decode("utf-8")) for a, b in scope["headers"]],
+        data=scope.get("body", None),
+    ) as resp:
         headers = {
-            'headers': [(a.encode('utf-8'), b.encode('utf-8'))
-                        for a, b in resp.headers.items()],
-            'status': resp.status
-        }
-        body = {
-            'body': await resp.content.read()
+            "headers": [
+                (a.encode("utf-8"), b.encode("utf-8")) for a, b in resp.headers.items()
+            ],
+            "status": resp.status,
         }
+        body = {"body": await resp.content.read()}
     return headers, body
 
 
@@ -325,7 +335,7 @@ async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[by
             except aiohttp.ClientConnectorError:
                 if tries > 20:
                     raise
-                await asyncio.sleep(.05)
+                await asyncio.sleep(0.05)
 
     output = ""  # Process stdout is not captured per request
     output_data = None
@@ -334,7 +344,9 @@ async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[by
 
 
 async def process_instruction(
-        instruction: bytes, interface: Interface, application: Union[ASGIApplication, subprocess.Popen]
+    instruction: bytes,
+    interface: Interface,
+    application: Union[ASGIApplication, subprocess.Popen],
 ) -> AsyncIterable[bytes]:
 
     if instruction == b"halt":
@@ -348,6 +360,7 @@ async def process_instruction(
         else:
             # Close the cached session in aleph_client:
             from aleph_client.asynchronous import get_fallback_session
+
             session: aiohttp.ClientSession = get_fallback_session()
             await session.close()
             logger.debug("Aiohttp cached session closed")
@@ -358,7 +371,9 @@ async def process_instruction(
         # Execute shell commands in the form `!ls /`
         msg = instruction[1:].decode()
         try:
-            process_output = subprocess.check_output(msg, stderr=subprocess.STDOUT, shell=True)
+            process_output = subprocess.check_output(
+                msg, stderr=subprocess.STDOUT, shell=True
+            )
             yield process_output
         except subprocess.CalledProcessError as error:
             yield str(error).encode() + b"\n" + error.output
@@ -376,11 +391,13 @@ async def process_instruction(
             output_data: Optional[bytes]
 
             if interface == Interface.asgi:
-                headers, body, output, output_data = \
-                    await run_python_code_http(application=application, scope=payload.scope)
+                headers, body, output, output_data = await run_python_code_http(
+                    application=application, scope=payload.scope
+                )
             elif interface == Interface.executable:
-                headers, body, output, output_data = \
-                    await run_executable_http(scope=payload.scope)
+                headers, body, output, output_data = await run_executable_http(
+                    scope=payload.scope
+                )
             else:
                 raise ValueError("Unknown interface. This should never happen")
 
@@ -392,11 +409,13 @@ async def process_instruction(
             }
             yield msgpack.dumps(result, use_bin_type=True)
         except Exception as error:
-            yield msgpack.dumps({
-                "error": str(error),
-                "traceback": str(traceback.format_exc()),
-                "output": output
-            })
+            yield msgpack.dumps(
+                {
+                    "error": str(error),
+                    "traceback": str(traceback.format_exc()),
+                    "output": output,
+                }
+            )
 
 
 def receive_data_length(client) -> int:
@@ -413,8 +432,7 @@ def receive_data_length(client) -> int:
 
 def load_configuration(data: bytes) -> ConfigurationPayload:
     msg_ = msgpack.loads(data, raw=False)
-    msg_['volumes'] = [Volume(**volume_dict)
-                       for volume_dict in msg_.get('volumes')]
+    msg_["volumes"] = [Volume(**volume_dict) for volume_dict in msg_.get("volumes")]
     return ConfigurationPayload(**msg_)
 
 
@@ -452,20 +470,26 @@ async def main():
 
     try:
         app: Union[ASGIApplication, subprocess.Popen] = setup_code(
-            config.code, config.encoding, config.entrypoint, config.interface)
+            config.code, config.encoding, config.entrypoint, config.interface
+        )
         client.send(msgpack.dumps({"success": True}))
     except Exception as error:
-        client.send(msgpack.dumps({
-            "success": False,
-            "error": str(error),
-            "traceback": str(traceback.format_exc()),
-        }))
+        client.send(
+            msgpack.dumps(
+                {
+                    "success": False,
+                    "error": str(error),
+                    "traceback": str(traceback.format_exc()),
+                }
+            )
+        )
         logger.exception("Program could not be started")
         raise
 
     class ServerReference:
         "Reference used to close the server from within `handle_instruction"
         server: asyncio.AbstractServer
+
     server_reference = ServerReference()
 
     async def handle_instruction(reader, writer):
@@ -477,8 +501,9 @@ async def handle_instruction(reader, writer):
             logger.debug(f"<<<\n\n{data_to_print}\n\n>>>")
 
         try:
-            async for result in process_instruction(instruction=data, interface=config.interface,
-                                                    application=app):
+            async for result in process_instruction(
+                instruction=data, interface=config.interface, application=app
+            ):
                 writer.write(result)
                 await writer.drain()
 
@@ -497,7 +522,7 @@ async def handle_instruction(reader, writer):
     server_reference.server = server
 
     addr = server.sockets[0].getsockname()
-    print(f'Serving on {addr}')
+    print(f"Serving on {addr}")
 
     try:
         async with server:
@@ -511,7 +536,8 @@ async def handle_instruction(reader, writer):
         umount_volumes(config.volumes)
         logger.debug("User volumes unmounted")
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     logging.basicConfig(level=logging.DEBUG)
     asyncio.run(main())
 
@@ -524,6 +550,6 @@ async def handle_instruction(reader, writer):
     # Send reboot syscall, see man page
     # https://man7.org/linux/man-pages/man2/reboot.2.html
     libc = ctypes.CDLL(None)
-    libc.syscall(169, 0xfee1dead, 672274793, 0x4321fedc, None)
+    libc.syscall(169, 0xFEE1DEAD, 672274793, 0x4321FEDC, None)
     # The exit should not happen due to system halt.
     sys.exit(0)
diff --git a/vm_connector/main.py b/vm_connector/main.py
index 675904925..a00d5646a 100644
--- a/vm_connector/main.py
+++ b/vm_connector/main.py
@@ -67,12 +67,12 @@ async def download_message(
     ref: str, use_latest: Optional[bool] = True
 ) -> Union[Dict, Response]:
     """
-        Fetch on Aleph and return a VM function message, after checking its validity.
-        Used by the VM Supervisor run the code.
+    Fetch on Aleph and return a VM function message, after checking its validity.
+    Used by the VM Supervisor run the code.
 
-        :param ref: item_hash of the code file
-        :param use_latest: should the last amend to the code be used
-        :return: a file containing the code file
+    :param ref: item_hash of the code file
+    :param use_latest: should the last amend to the code be used
+    :return: a file containing the code file
     """
 
     msg = await get_message(hash_=ref)
diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index 361fd2ce2..45bb3f21a 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -218,17 +218,18 @@ def main():
             sentry_sdk.init(
                 dsn=settings.SENTRY_DSN,
                 server_name=settings.DOMAIN_NAME,
-
                 # Set traces_sample_rate to 1.0 to capture 100%
                 # of transactions for performance monitoring.
                 # We recommend adjusting this value in production.
-                traces_sample_rate=1.0
+                traces_sample_rate=1.0,
             )
         else:
             logger.debug("Sentry SDK found with no DNS configured.")
     else:
-        logger.debug("Sentry SDK not found. \n"
-                     "Use `pip install sentry-sdk` and configure SENTRY_DSN if you'd like to monitor errors.")
+        logger.debug(
+            "Sentry SDK not found. \n"
+            "Use `pip install sentry-sdk` and configure SENTRY_DSN if you'd like to monitor errors."
+        )
 
     settings.setup()
     if args.print_settings:
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 594e99a0e..2e16329a7 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -85,7 +85,7 @@ class Settings(BaseSettings):
     FIRECRACKER_PATH = "/opt/firecracker/firecracker"
     JAILER_PATH = "/opt/firecracker/jailer"
     LINUX_PATH = "/opt/firecracker/vmlinux.bin"
-    INIT_TIMEOUT: float = 20.
+    INIT_TIMEOUT: float = 20.0
 
     CONNECTOR_URL = Url("http://localhost:4021")
 
@@ -132,8 +132,10 @@ class Settings(BaseSettings):
     SENTRY_DSN: Optional[str] = None
 
     # Fields
-    SENSITIVE_FIELDS: List[str] = Field(default=["SENTRY_DSN"],
-                                        description="Sensitive fields, redacted from `--print-settings`.")
+    SENSITIVE_FIELDS: List[str] = Field(
+        default=["SENTRY_DSN"],
+        description="Sensitive fields, redacted from `--print-settings`.",
+    )
 
     def update(self, **kwargs):
         for key, value in kwargs.items():
@@ -162,8 +164,12 @@ def check(self):
             ), "Local fake program directory is missing"
             assert isfile(self.FAKE_DATA_MESSAGE), "Local fake message is missing"
             assert isdir(self.FAKE_DATA_DATA), "Local fake data directory is missing"
-            assert isfile(self.FAKE_DATA_RUNTIME), "Local runtime .squashfs build is missing"
-            assert isfile(self.FAKE_DATA_VOLUME), "Local data volume .squashfs is missing"
+            assert isfile(
+                self.FAKE_DATA_RUNTIME
+            ), "Local runtime .squashfs build is missing"
+            assert isfile(
+                self.FAKE_DATA_VOLUME
+            ), "Local data volume .squashfs is missing"
 
     def setup(self):
         os.makedirs(self.MESSAGE_CACHE, exist_ok=True)
@@ -192,8 +198,7 @@ def display(self) -> str:
                 annotations[attr] = getattr(self, attr)
 
         return "\n".join(
-            f"{annotation:<27} = {value}"
-            for annotation, value in annotations.items()
+            f"{annotation:<27} = {value}" for annotation, value in annotations.items()
         )
 
     class Config:
diff --git a/vm_supervisor/messages.py b/vm_supervisor/messages.py
index 8abaa186f..b408685d5 100644
--- a/vm_supervisor/messages.py
+++ b/vm_supervisor/messages.py
@@ -18,8 +18,7 @@ async def try_get_message(ref: str) -> ProgramMessage:
         raise HTTPServiceUnavailable(reason="Aleph Connector unavailable")
     except ClientResponseError as error:
         if error.status == 404:
-            raise HTTPNotFound(reason="Hash not found",
-                               body=f"Hash not found: {ref}")
+            raise HTTPNotFound(reason="Hash not found", body=f"Hash not found: {ref}")
         else:
             raise
 
@@ -31,8 +30,9 @@ async def get_latest_ref(item_hash: str) -> str:
         raise HTTPServiceUnavailable(reason="Aleph Connector unavailable")
     except ClientResponseError as error:
         if error.status == 404:
-            raise HTTPNotFound(reason="Hash not found",
-                               body=f"Hash not found: {item_hash}")
+            raise HTTPNotFound(
+                reason="Hash not found", body=f"Hash not found: {item_hash}"
+            )
         else:
             raise
 
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index e95062979..5e7a15bc8 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -41,6 +41,7 @@ class VmExecution:
 
     Implementation agnostic (Firecracker, maybe WASM in the future, ...).
     """
+
     uuid: uuid.UUID  # Unique identifier of this execution
     vm_hash: VmHash
     original: ProgramContent
@@ -187,27 +188,28 @@ async def all_runs_complete(self):
     async def record_usage(self):
         if settings.EXECUTION_LOG_ENABLED:
             await save_execution_data(
-                execution_uuid=self.uuid,
-                execution_data=self.to_json()
+                execution_uuid=self.uuid, execution_data=self.to_json()
             )
         pid_info = self.vm.to_dict()
-        await save_record(ExecutionRecord(
-            uuid=str(self.uuid),
-            vm_hash=self.vm_hash,
-            time_defined=self.times.defined_at,
-            time_prepared=self.times.prepared_at,
-            time_started=self.times.started_at,
-            time_stopping=self.times.stopping_at,
-            cpu_time_user=pid_info["process"]["cpu_times"].user,
-            cpu_time_system=pid_info["process"]["cpu_times"].system,
-            io_read_count=pid_info["process"]["io_counters"][0],
-            io_write_count=pid_info["process"]["io_counters"][1],
-            io_read_bytes=pid_info["process"]["io_counters"][2],
-            io_write_bytes=pid_info["process"]["io_counters"][3],
-            vcpus=self.vm.hardware_resources.vcpus,
-            memory=self.vm.hardware_resources.memory,
-            network_tap=self.vm.fvm.network_tap,
-        ))
+        await save_record(
+            ExecutionRecord(
+                uuid=str(self.uuid),
+                vm_hash=self.vm_hash,
+                time_defined=self.times.defined_at,
+                time_prepared=self.times.prepared_at,
+                time_started=self.times.started_at,
+                time_stopping=self.times.stopping_at,
+                cpu_time_user=pid_info["process"]["cpu_times"].user,
+                cpu_time_system=pid_info["process"]["cpu_times"].system,
+                io_read_count=pid_info["process"]["io_counters"][0],
+                io_write_count=pid_info["process"]["io_counters"][1],
+                io_read_bytes=pid_info["process"]["io_counters"][2],
+                io_write_bytes=pid_info["process"]["io_counters"][3],
+                vcpus=self.vm.hardware_resources.vcpus,
+                memory=self.vm.hardware_resources.memory,
+                network_tap=self.vm.fvm.network_tap,
+            )
+        )
 
     async def run_code(self, scope: dict = None) -> bytes:
         if not self.vm:
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 00a09d348..dc2f07c09 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -56,7 +56,6 @@ async def stop(self):
         """Stop all VMs in the pool."""
 
         # Stop executions in parallel:
-        await asyncio.gather(*(
-            execution.stop()
-            for vm_hash, execution in self.executions.items()
-        ))
+        await asyncio.gather(
+            *(execution.stop() for vm_hash, execution in self.executions.items())
+        )
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 624b84f9a..b881df744 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -89,8 +89,12 @@ async def run_code_on_request(
         await execution.becomes_ready()
         result_raw: bytes = await execution.run_code(scope=scope)
     except asyncio.TimeoutError:
-        logger.warning(f"VM{execution.vm.vm_id} did not respond within `resource.seconds`")
-        return web.HTTPGatewayTimeout(body="Program did not respond within `resource.seconds`")
+        logger.warning(
+            f"VM{execution.vm.vm_id} did not respond within `resource.seconds`"
+        )
+        return web.HTTPGatewayTimeout(
+            body="Program did not respond within `resource.seconds`"
+        )
     except UnpackValueError as error:
         logger.exception(error)
         return web.Response(status=502, reason="Invalid response from VM")
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 9cedfb739..d3fb9bcb8 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -22,7 +22,8 @@
     about_login,
     about_executions,
     about_config,
-    status_check_fastapi, about_execution_records,
+    status_check_fastapi,
+    about_execution_records,
 )
 
 logger = logging.getLogger(__name__)
@@ -30,15 +31,17 @@
 
 @web.middleware
 async def server_version_middleware(
-        request: web.Request,
-        handler: Callable[[web.Request], Awaitable[web.StreamResponse]]
+    request: web.Request,
+    handler: Callable[[web.Request], Awaitable[web.StreamResponse]],
 ) -> web.StreamResponse:
-    """Add the version of Aleph-VM in the HTTP headers of the responses.
-    """
+    """Add the version of Aleph-VM in the HTTP headers of the responses."""
     resp: web.StreamResponse = await handler(request)
-    resp.headers.update({'Server': f"aleph-vm/{__version__}"},)
+    resp.headers.update(
+        {"Server": f"aleph-vm/{__version__}"},
+    )
     return resp
 
+
 app = web.Application(middlewares=[server_version_middleware])
 
 app.add_routes(
@@ -53,6 +56,7 @@ async def server_version_middleware(
     ]
 )
 
+
 async def stop_all_vms(app: web.Application):
     await pool.stop()
 
@@ -67,7 +71,9 @@ def run():
     # Require a random token to access /about APIs
     secret_token = token_urlsafe(nbytes=32)
     app["secret_token"] = secret_token
-    print(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
+    print(
+        f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}"
+    )
 
     engine = metrics.setup_engine()
     metrics.create_tables(engine)
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index 3af8170f1..c128b8369 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -31,5 +31,5 @@ def to_json(o: Any):
         return str(o)
 
 
-def dumps_for_json(o: Any, indent: Optional[int]=None):
+def dumps_for_json(o: Any, indent: Optional[int] = None):
     return json.dumps(o, default=to_json, indent=indent)
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index 50d3d99f1..2de25d018 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -42,9 +42,11 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
     we expect the hash to be encoded in base32 instead of hexadecimal. Padding is added
     automatically.
     """
-    if request.host.split(':')[0] == settings.DOMAIN_NAME \
-            and request.method == "GET" \
-            and request.path == "/":
+    if (
+        request.host.split(":")[0] == settings.DOMAIN_NAME
+        and request.method == "GET"
+        and request.path == "/"
+    ):
         # Serve the index page
         return await index(request=request)
 
@@ -114,18 +116,17 @@ async def about_execution_records(request: web.Request):
 
 async def index(request: web.Request):
     assert request.method == "GET"
-    path = os.path.join(os.path.dirname(__file__), 'templates/index.html')
-    with open(path, 'r') as template:
+    path = os.path.join(os.path.dirname(__file__), "templates/index.html")
+    with open(path, "r") as template:
         body = template.read()
         s = Template(body)
         body = s.substitute(
-            public_url=f'https://{settings.DOMAIN_NAME}/',
-            multiaddr_dns4=f'/dns4/{settings.DOMAIN_NAME}/tcp/443/https',
-            multiaddr_dns6=f'/dns6/{settings.DOMAIN_NAME}/tcp/443/https',
+            public_url=f"https://{settings.DOMAIN_NAME}/",
+            multiaddr_dns4=f"/dns4/{settings.DOMAIN_NAME}/tcp/443/https",
+            multiaddr_dns6=f"/dns6/{settings.DOMAIN_NAME}/tcp/443/https",
             check_fastapi_vm_id=settings.CHECK_FASTAPI_VM_ID,
         )
-    return web.Response(content_type="text/html",
-                        body=body)
+    return web.Response(content_type="text/html", body=body)
 
 
 async def status_check_fastapi(request: web.Request):
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 04f3907a7..285041780 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -449,8 +449,10 @@ async def communicate(reader, writer, scope):
 
         reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
         try:
-            return await asyncio.wait_for(communicate(reader, writer, scope),
-                                          timeout=self.hardware_resources.seconds)
+            return await asyncio.wait_for(
+                communicate(reader, writer, scope),
+                timeout=self.hardware_resources.seconds,
+            )
         finally:
             logger.debug("Cleaning VM socket resources")
             writer.close()

From b2e113e70e2a345c554927c6bdafe342bd48985a Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 29 Mar 2022 15:38:05 +0200
Subject: [PATCH 279/990] Fix: Jailer process did not receive new jail chroot
 base

---
 firecracker/microvm.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 11be26f4a..7a5ebd855 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -221,6 +221,8 @@ async def start_jailed_firecracker(
                     uid,
                     "--gid",
                     gid,
+                    "--chroot-base-dir",
+                    JAILER_BASE_DIRECTORY,
                     "--",
                     "--config-file",
                     "/tmp/" + os.path.basename(config_file.name),
@@ -238,6 +240,8 @@ async def start_jailed_firecracker(
             uid,
             "--gid",
             gid,
+            "--chroot-base-dir",
+            JAILER_BASE_DIRECTORY,
             "--",
             "--config-file",
             "/tmp/" + os.path.basename(config_file.name),

From 6535c7a71cb74fb579f9ca97677cda94624f6583 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 29 Mar 2022 15:38:35 +0200
Subject: [PATCH 280/990] Fix: Path contained duplicated "volumes" element

---
 vm_supervisor/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 2e16329a7..21a6779ab 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -101,7 +101,7 @@ class Settings(BaseSettings):
     EXECUTION_LOG_DIRECTORY = FilePath(join(EXECUTION_ROOT, "executions"))
 
     PERSISTENT_VOLUMES_DIR = FilePath(
-        join("/var/lib/aleph/vm/volumes", "volumes", "persistent")
+        join("/var/lib/aleph/vm", "volumes", "persistent")
     )
 
     MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000  # 10 MB

From 1073fbf04979d4a6d282574828a80840763ebb1f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 31 Mar 2022 11:19:56 +0200
Subject: [PATCH 281/990] Fix: Package postrm/postinst failed if directory was
 missing

---
 packaging/aleph-vm/DEBIAN/postinst | 3 ++-
 packaging/aleph-vm/DEBIAN/postrm   | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/packaging/aleph-vm/DEBIAN/postinst b/packaging/aleph-vm/DEBIAN/postinst
index 1286c189d..f7d9f642a 100755
--- a/packaging/aleph-vm/DEBIAN/postinst
+++ b/packaging/aleph-vm/DEBIAN/postinst
@@ -5,7 +5,8 @@ if ! id -u jailman > /dev/null 2>&1; then
   useradd jailman
 fi
 
-rm -r /srv/jailer  # Upgrade from < 0.1.11
+rm -fr /srv/jailer  # Upgrade from < 0.1.11
+rm -fr /tmp/aleph   # Upgrade from < 0.1.11
 mkdir -p /var/lib/aleph/vm/jailer
 
 # Systemd is absent from containers
diff --git a/packaging/aleph-vm/DEBIAN/postrm b/packaging/aleph-vm/DEBIAN/postrm
index 293e02e38..5da106c35 100755
--- a/packaging/aleph-vm/DEBIAN/postrm
+++ b/packaging/aleph-vm/DEBIAN/postrm
@@ -1,7 +1,8 @@
 #!/bin/bash
 set -euf -o pipefail
 
-rm -r /srv/jailer
+rm -fr /srv/jailer  # Upgrade from < 0.1.11
+rm -fr /tmp/aleph/  # Upgrade from < 0.1.11
 rm -r /var/lib/aleph/vm/jailer
 
 systemctl daemon-reload

From 82e7f5d2b74ce19fcb8bb660ac0b96b4b63479f7 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 27 Apr 2022 11:46:32 +0200
Subject: [PATCH 282/990] Fix: Cargo.toml file was missing from Rust example

---
 examples/example_http_rust/Cargo.toml | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 examples/example_http_rust/Cargo.toml

diff --git a/examples/example_http_rust/Cargo.toml b/examples/example_http_rust/Cargo.toml
new file mode 100644
index 000000000..69cc888f8
--- /dev/null
+++ b/examples/example_http_rust/Cargo.toml
@@ -0,0 +1,8 @@
+[package]
+name = "example_http_rust"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

From c6de09e47e95a92ec799d363b0f482bb475b8b6f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 26 Apr 2022 19:45:09 +0200
Subject: [PATCH 283/990] Fix: Developer Dockerfile was obsolete and broken

---
 docker/vm_supervisor-dev.dockerfile | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 31932a976..d891dad8d 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -3,25 +3,25 @@
 FROM debian:bullseye
 
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
-    sudo acl curl systemd-container  \
-    python3 python3-aiohttp python3-msgpack python3-pip python3-aiodns python3-aioredis \
-    squashfs-tools python3-psutil \
+    sudo acl curl squashfs-tools git \
+    python3 python3-aiohttp python3-msgpack python3-pip python3-aiodns python3-aioredis  \
+    python3-psutil python3-setproctitle python3-sqlalchemy \
     && rm -rf /var/lib/apt/lists/*
 
 RUN useradd jailman
 
 RUN mkdir /opt/firecracker
 RUN chown $(whoami) /opt/firecracker
-RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v0.24.2/firecracker-v0.24.2-x86_64.tgz | tar -xz --directory /opt/firecracker
+RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.0.0/firecracker-v1.0.0-x86_64.tgz | tar -xz --directory /opt/firecracker
 RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/vmlinux.bin
 
 # Link binaries on version-agnostic paths:
-RUN ln /opt/firecracker/firecracker-v* /opt/firecracker/firecracker
-RUN ln /opt/firecracker/jailer-v* /opt/firecracker/jailer
+RUN ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker
+RUN ln /opt/firecracker/release-*/jailer-v* /opt/firecracker/jailer
 
 RUN pip3 install typing-extensions 'aleph-message>=0.1.18'
 
-RUN mkdir /var/lib/aleph/vm/jailer
+RUN mkdir -p /var/lib/aleph/vm/jailer
 
 ENV PYTHONPATH /mnt
 
@@ -47,4 +47,4 @@ COPY ./runtimes /opt/aleph-vm/runtimes
 
 WORKDIR /opt/aleph-vm
 
-CMD "bash"
+CMD "bash"
\ No newline at end of file

From 9163c13ffd4c56c528dca03a19f75aeaab9455d6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 26 Apr 2022 19:48:34 +0200
Subject: [PATCH 284/990] Fix: Database access during benchmarks errored

Solution: Initiate the database in the benchmark procedure.
---
 vm_supervisor/__main__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index 45bb3f21a..218d00633 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -14,7 +14,7 @@
     sentry_sdk = None
 
 from vm_supervisor.pubsub import PubSub
-from . import supervisor
+from . import supervisor, metrics
 from .conf import settings
 from .models import VmHash
 from .run import run_code_on_request, run_code_on_event
@@ -115,6 +115,9 @@ async def benchmark(runs: int):
     """Measure performance by immediately running the supervisor
     with fake requests.
     """
+    engine = metrics.setup_engine()
+    metrics.create_tables(engine)
+
     ref = VmHash("fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-hash")
     settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM
 

From 639e515fc23855d99ad945bc952b407d04f20b6d Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 26 Apr 2022 19:44:17 +0200
Subject: [PATCH 285/990] Feature: Developer could not run custom command in
 container

---
 docker/run_vm_supervisor.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/run_vm_supervisor.sh b/docker/run_vm_supervisor.sh
index 22ca76db4..7b076783c 100755
--- a/docker/run_vm_supervisor.sh
+++ b/docker/run_vm_supervisor.sh
@@ -17,4 +17,4 @@ $DOCKER_COMMAND run -ti --rm \
   -v "$(pwd)/firecracker:/opt/aleph-vm/firecracker:ro" \
   --device /dev/kvm \
   -p 4020:4020 \
-  alephim/vm-supervisor-dev
+  alephim/vm-supervisor-dev $@

From 04fb5f8ab992749330e1da9d306d7a4b4c000fc0 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 26 Apr 2022 19:30:55 +0200
Subject: [PATCH 286/990] Fix: Getting version from git crashed when git was
 missing or on shallow clone

---
 vm_supervisor/__init__.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/vm_supervisor/__init__.py b/vm_supervisor/__init__.py
index 6692f08ad..efd0503dc 100644
--- a/vm_supervisor/__init__.py
+++ b/vm_supervisor/__init__.py
@@ -1,8 +1,18 @@
-from subprocess import check_output
+import logging
+from subprocess import check_output, CalledProcessError
+
+logger = logging.getLogger(__name__)
 
 
 def get_version_from_git() -> str:
-    return check_output(("git", "describe", "--tags")).strip().decode()
+    try:
+        return check_output(("git", "describe", "--tags")).strip().decode()
+    except FileNotFoundError:
+        logger.warning("git not found")
+        return "unknown-version"
+    except CalledProcessError:
+        logger.warning("git description not available")
+        return "unavailable-version"
 
 
 # The version number is harcoded in the following line when packaging the software

From ffa11be8e259f55714e37d34a3e4e0e7bc75d3c5 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Thu, 28 Apr 2022 17:33:56 +0200
Subject: [PATCH 287/990] [Doc] Fix typos

---
 doc/INSTALL-Debian-11.md    |  3 +--
 doc/INSTALL-Ubuntu-20.04.md |  3 +--
 vm_supervisor/README.md     | 11 +++--------
 3 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index 9f79f02d9..3f7a62ca7 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -146,10 +146,9 @@ docker logs -f vm-connector
 
 #### "Network interface eth0 does not exist"
 
-Did you update the configuration file `/etc/aleph-vm/vm-supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
+Did you update the configuration file `/etc/aleph-vm/supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
 the default network interface of your server ?
 
 #### "Aleph Connector unavailable"
 
 Investigate the installation of the VM-Connector using Docker in step 2.
-
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
index 675774043..df061471c 100644
--- a/doc/INSTALL-Ubuntu-20.04.md
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -145,10 +145,9 @@ sudo docker logs -f vm-connector
 
 #### "Network interface eth0 does not exist"
 
-Did you update the configuration file `/etc/aleph-vm/vm-supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
+Did you update the configuration file `/etc/aleph-vm/supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
 the default network interface of your server ?
 
 #### "Aleph Connector unavailable"
 
 Investigate the installation of the VM-Connector using Docker in step 2.
-
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index d72ee7e07..72b0362f7 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -47,7 +47,7 @@ While not supported at the moment, it is possible to run the VM Supervisor insid
 container. 
 
 This will be less secure since the `Jailer` tool used to secure Firecracker MicroVMs
-will not run inside containers. Pass the command-lien argument `--no-jailer` to disable the Jailer
+will not run inside containers. Pass the command-line argument `--no-jailer` to disable the Jailer
 when running the VM Supervisor.
 
 ## 2. Installation
@@ -126,7 +126,7 @@ http://localhost:4020/
 
 The VM Supervisor can be configured using command-line arguments or using environment variables.
 
-List the available command-lien arguments using:
+List the available command-line arguments using:
 ```shell
 python3 -m vm_supervisor --help
 ```
@@ -144,10 +144,6 @@ ALEPH_VM_DNS_RESOLUTION=resolvectl
 ALEPH_VM_NETWORK_INTERFACE=enp7s0
 ```
 
-
-
-```
-
 ## 6. Production security concerns
 
 See advanced security related concerns here:
@@ -162,8 +158,7 @@ A runtime consist in the root filesystem used by a VM.
 Runtimes contain a customized init that allows the VM Supervisor to run
 functions within the MicroVM.
 
-Official Aleph runtimes are built using scripts located in 
-in [`../runtimes`](../runtimes), and are distributed on the Aleph network.
+Official Aleph runtimes are built using scripts located in [`../runtimes`](../runtimes), and are distributed on the Aleph network.
 
 To build the default runtime locally:
 

From 356e374ffcb3299fe329d69d846701b4834b293e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 26 Apr 2022 19:27:11 +0200
Subject: [PATCH 288/990] Fix: Users could not tell if redacted fields were
 empty

---
 vm_supervisor/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 21a6779ab..f758312f2 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -192,7 +192,7 @@ def display(self) -> str:
         annotations = self.__annotations__.copy()
 
         for attr in annotations.keys():
-            if attr in self.SENSITIVE_FIELDS:
+            if getattr(self, attr) and attr in self.SENSITIVE_FIELDS:
                 annotations[attr] = ""
             else:
                 annotations[attr] = getattr(self, attr)

From 36bc57c1ae435ad22a0b049f8da79494376e37e6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Sun, 1 May 2022 21:16:07 +0200
Subject: [PATCH 289/990] Fix: Incomplete file download could lead to
 inconsistent state (#177)

---
 vm_supervisor/storage.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 5f6efeef8..0e3aeb07b 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -10,6 +10,7 @@
 import logging
 import os
 import re
+import sys
 from os.path import isfile, join, dirname
 from shutil import make_archive
 
@@ -34,21 +35,32 @@ async def download_file(url: str, local_path: FilePath) -> None:
     if isfile(local_path):
         logger.debug(f"File already exists: {local_path}")
     else:
-        logger.debug(f"Downloading {url} -> {local_path}")
+        tmp_path = f"{local_path}.part"
+        logger.debug(f"Downloading {url} -> {tmp_path}")
         async with aiohttp.ClientSession() as session:
             resp = await session.get(url)
             resp.raise_for_status()
             try:
-                with open(local_path, "wb") as cache_file:
+                with open(tmp_path, "wb") as cache_file:
+                    counter = 0
                     while True:
                         chunk = await resp.content.read(65536)
                         if not chunk:
                             break
                         cache_file.write(chunk)
-                logger.debug("Download complete")
+                        counter += 1
+                        if not (counter % 20):
+                            sys.stdout.write(".")
+                            sys.stdout.flush()
+
+                os.rename(tmp_path, local_path)
+                logger.debug(f"Download complete, moved {tmp_path} -> {local_path}")
             except Exception:
                 # Ensure no partial file is left
-                os.remove(local_path)
+                try:
+                    os.remove(tmp_path)
+                except FileNotFoundError:
+                    pass
                 raise
 
 

From d00ed764e6e0b7958a38cf763fcc7af7ccba1755 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Sun, 1 May 2022 21:17:10 +0200
Subject: [PATCH 290/990] Fix: Process metrics crashed if process was already
 stopped (#169)

---
 vm_supervisor/vm/firecracker_microvm.py | 30 ++++++++++++++-----------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 285041780..105d680a6 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -228,19 +228,23 @@ def __init__(
 
     def to_dict(self):
         if self.fvm.proc and psutil:
-            p = psutil.Process(self.fvm.proc.pid)
-            pid_info = {
-                "status": p.status(),
-                "create_time": p.create_time(),
-                "cpu_times": p.cpu_times(),
-                "cpu_percent": p.cpu_percent(),
-                "memory_info": p.memory_info(),
-                "io_counters": p.io_counters(),
-                "open_files": p.open_files(),
-                "connections": p.connections(),
-                "num_threads": p.num_threads(),
-                "num_ctx_switches": p.num_ctx_switches(),
-            }
+            try:
+                p = psutil.Process(self.fvm.proc.pid)
+                pid_info = {
+                    "status": p.status(),
+                    "create_time": p.create_time(),
+                    "cpu_times": p.cpu_times(),
+                    "cpu_percent": p.cpu_percent(),
+                    "memory_info": p.memory_info(),
+                    "io_counters": p.io_counters(),
+                    "open_files": p.open_files(),
+                    "connections": p.connections(),
+                    "num_threads": p.num_threads(),
+                    "num_ctx_switches": p.num_ctx_switches(),
+                }
+            except psutil.NoSuchProcess:
+                logger.warning("Cannot read process metrics (process not found)", exc_info=True)
+                pid_info = None
         else:
             pid_info = None
 

From 7dd5a8726cd53c3f3f975f6aaef878ff2fae0ab9 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 3 May 2022 14:27:50 +0200
Subject: [PATCH 291/990] Fix: Mime type of all files was not
 application/(g)zip.

Solution: Use the mime-type from the message if possible, else use the generic "application/octet-stream".
---
 vm_connector/main.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/vm_connector/main.py b/vm_connector/main.py
index a00d5646a..cb9ada218 100644
--- a/vm_connector/main.py
+++ b/vm_connector/main.py
@@ -99,12 +99,15 @@ async def download_code(
     if not msg:
         return Response(status_code=404, content="Hash not found")
 
+    media_type = msg["content"].get("mime_type", default="application/octet-stream")
+
     data_hash = msg["content"]["item_hash"]
     if msg["content"]["item_type"] == "ipfs":
         url = f"{settings.IPFS_SERVER}/{data_hash}"
     else:
         url = f"{settings.API_SERVER}/api/v0/storage/raw/{data_hash}"
-    return StreamingResponse(stream_url_chunks(url), media_type="application/zip")
+
+    return StreamingResponse(stream_url_chunks(url), media_type=media_type)
 
 
 @app.get("/download/data/{ref}")
@@ -125,9 +128,15 @@ async def download_data(
     if not msg:
         return Response(status_code=404, content="Hash not found")
 
+    media_type = msg["content"].get("mime_type", default="application/octet-stream")
+
     data_hash = msg["content"]["item_hash"]
-    url = f"{settings.IPFS_SERVER}/{data_hash}"
-    return StreamingResponse(stream_url_chunks(url), media_type="application/gzip")
+    if msg["content"]["item_type"] == "ipfs":
+        url = f"{settings.IPFS_SERVER}/{data_hash}"
+    else:
+        url = f"{settings.API_SERVER}/api/v0/storage/raw/{data_hash}"
+
+    return StreamingResponse(stream_url_chunks(url), media_type=media_type)
 
 
 @app.get("/download/runtime/{ref}")

From c40cc6b94cb46100e937d1290af402d0db2d4801 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 3 May 2022 14:33:05 +0200
Subject: [PATCH 292/990] Fix: Remove code duplication

---
 vm_connector/main.py | 25 ++-----------------------
 1 file changed, 2 insertions(+), 23 deletions(-)

diff --git a/vm_connector/main.py b/vm_connector/main.py
index cb9ada218..81f2e23f7 100644
--- a/vm_connector/main.py
+++ b/vm_connector/main.py
@@ -94,20 +94,7 @@ async def download_code(
     :param use_latest: should the last amend to the code be used
     :return: a file containing the code file
     """
-
-    msg = await get_message(hash_=ref)
-    if not msg:
-        return Response(status_code=404, content="Hash not found")
-
-    media_type = msg["content"].get("mime_type", default="application/octet-stream")
-
-    data_hash = msg["content"]["item_hash"]
-    if msg["content"]["item_type"] == "ipfs":
-        url = f"{settings.IPFS_SERVER}/{data_hash}"
-    else:
-        url = f"{settings.API_SERVER}/api/v0/storage/raw/{data_hash}"
-
-    return StreamingResponse(stream_url_chunks(url), media_type=media_type)
+    return await download_data(ref=ref, use_latest=use_latest)
 
 
 @app.get("/download/data/{ref}")
@@ -151,15 +138,7 @@ async def download_runtime(
     :param use_latest: should the last amend to the runtime be used
     :return: a file containing the runtime
     """
-
-    # Download message
-    msg = await get_message(hash_=ref)
-    if not msg:
-        return Response(status_code=404, content="Hash not found")
-
-    data_hash = msg["content"]["item_hash"]
-    url = f"{settings.IPFS_SERVER}/{data_hash}"
-    return StreamingResponse(stream_url_chunks(url), media_type="application/ext4")
+    return await download_data(ref=ref, use_latest=use_latest)
 
 
 @app.get("/compute/latest_amend/{item_hash}")

From 3196b4fc0edd2410bad87d380173fcae96dc9a03 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 3 May 2022 14:35:07 +0200
Subject: [PATCH 293/990] Fix: Argument `use_latest` was never used.

---
 vm_connector/main.py | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/vm_connector/main.py b/vm_connector/main.py
index 81f2e23f7..44d4f6d3f 100644
--- a/vm_connector/main.py
+++ b/vm_connector/main.py
@@ -63,9 +63,7 @@ async def stream_url_chunks(url):
 
 
 @app.get("/download/message/{ref}")
-async def download_message(
-    ref: str, use_latest: Optional[bool] = True
-) -> Union[Dict, Response]:
+async def download_message(ref: str) -> Union[Dict, Response]:
     """
     Fetch on Aleph and return a VM function message, after checking its validity.
     Used by the VM Supervisor run the code.
@@ -83,9 +81,7 @@ async def download_message(
 
 
 @app.get("/download/code/{ref}")
-async def download_code(
-    ref: str, use_latest: Optional[bool] = True
-) -> Union[StreamingResponse, Response]:
+async def download_code(ref: str) -> Union[StreamingResponse, Response]:
     """
     Fetch on Aleph and return a VM code file, after checking its validity.
     Used by the VM Supervisor to download function source code.
@@ -94,13 +90,11 @@ async def download_code(
     :param use_latest: should the last amend to the code be used
     :return: a file containing the code file
     """
-    return await download_data(ref=ref, use_latest=use_latest)
+    return await download_data(ref=ref)
 
 
 @app.get("/download/data/{ref}")
-async def download_data(
-    ref: str, use_latest: Optional[bool] = True
-) -> Union[StreamingResponse, Response]:
+async def download_data(ref: str) -> Union[StreamingResponse, Response]:
     """
     Fetch on Aleph and return a VM data file, after checking its validity.
     Used by the VM Supervisor to download state data.
@@ -127,9 +121,7 @@ async def download_data(
 
 
 @app.get("/download/runtime/{ref}")
-async def download_runtime(
-    ref: str, use_latest: Optional[bool] = True
-) -> Union[StreamingResponse, Response]:
+async def download_runtime(ref: str) -> Union[StreamingResponse, Response]:
     """
     Fetch on Aleph and return a VM runtime, after checking its validity.
     Used by the VM Supervisor to download a runtime.
@@ -138,7 +130,7 @@ async def download_runtime(
     :param use_latest: should the last amend to the runtime be used
     :return: a file containing the runtime
     """
-    return await download_data(ref=ref, use_latest=use_latest)
+    return await download_data(ref=ref)
 
 
 @app.get("/compute/latest_amend/{item_hash}")

From 9adf4f132b0969cd2060182068798502d6d705da Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 3 May 2022 15:10:55 +0200
Subject: [PATCH 294/990] Feature: User could not easily access the current
 version of the software.

Solution: Display the version info on the index page, and whether it is up-to-date.

Fetch information about the latest version from the browser, so no request to GitHub is done by the server.
---
 vm_supervisor/templates/index.html | 37 ++++++++++++++++++++++++++++++
 vm_supervisor/views.py             |  3 ++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/templates/index.html b/vm_supervisor/templates/index.html
index 920e0459b..a6483454c 100644
--- a/vm_supervisor/templates/index.html
+++ b/vm_supervisor/templates/index.html
@@ -107,6 +107,22 @@ 
    Diagnostic
    
         Diagnostics checks |
         Open diagnostic VM
     
    
+
+
    
+    
    Version
    
+    
    
+        Running version $version.
+    
    
+    
    
+        
+        
+        
+        
    
+    
    
 
 
    
 
@@ -146,6 +162,27 @@ 
    Tools
    
     }
     fetchMoviesJSON();
 
+    async function fetchLatestRelease() {
+        const response = await fetch('https://api.github.com/repos/aleph-im/aleph-vm/releases/latest');
+        if (response.ok) {
+            const response_data = await response.json();
+            const latest_version = response_data['tag_name'];
+            document.getElementById("status_check_button").style.display = "none";
+            if (latest_version === "$version") {
+                document.getElementById("status_latest_version").style.display = "";
+            }
+            else {
+                const comment = document.getElementById("status_outdated_version");
+                comment.style.display = "";
+                comment.getElementsByTagName('a')[0].textContent = latest_version;
+            }
+        }
+        else {
+            document.getElementById("status_error_version").style.display = "";
+            document.getElementById("status_error_version_reason").innerText = response.status + " " + response.statusText;
+        }
+        return response.status;
+    }
 
 
 
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index 2de25d018..8a98030a4 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -9,7 +9,7 @@
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPNotFound
 
-from . import status
+from . import status, get_version_from_git
 from .conf import settings
 from .metrics import get_execution_records
 from .models import VmHash
@@ -125,6 +125,7 @@ async def index(request: web.Request):
             multiaddr_dns4=f"/dns4/{settings.DOMAIN_NAME}/tcp/443/https",
             multiaddr_dns6=f"/dns6/{settings.DOMAIN_NAME}/tcp/443/https",
             check_fastapi_vm_id=settings.CHECK_FASTAPI_VM_ID,
+            version=get_version_from_git(),
         )
     return web.Response(content_type="text/html", body=body)
 

From ed39a38e6e23dba7b0d16fcf91800ae34b5ed48f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 11 May 2022 15:36:08 +0200
Subject: [PATCH 295/990] Fix: Response body requires bytes, was given str

---
 vm_supervisor/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index 8a98030a4..41c7406cc 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -77,7 +77,7 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
 def authenticate_request(request: web.Request) -> web.Response:
     """Check that the token in the cookies matches the app's secret token."""
     if request.cookies.get("token") != request.app["secret_token"]:
-        raise web.HTTPUnauthorized(reason="Invalid token", body="401 Invalid token")
+        raise web.HTTPUnauthorized(reason="Invalid token", text="401 Invalid token")
 
 
 async def about_login(request: web.Request) -> web.Response:

From 471ab9b0a298c8c08388f75feb3368d8ea9875d6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Sun, 8 May 2022 11:30:39 +0200
Subject: [PATCH 296/990] Fix: AttributeError crashed VM teardown

During VM teardown, the field AlephFirecrackerVM().guest_api_process._popen
could be None, and the following traceback be raised:

```python
  File "/root/aleph-vm/vm_supervisor/models.py", line 116, in create
    await vm.teardown()
  File "/root/aleph-vm/vm_supervisor/vm/firecracker_microvm.py", line 429, in teardown
    await self.stop_guest_api()
  File "/root/aleph-vm/vm_supervisor/vm/firecracker_microvm.py", line 424, in stop_guest_api
    self.guest_api_process.terminate()
  File "/usr/lib/python3.8/multiprocessing/process.py", line 133, in terminate
    self._popen.terminate()
AttributeError: 'NoneType' object has no attribute 'terminate'
```
---
 vm_supervisor/vm/firecracker_microvm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 105d680a6..ba9e90c7e 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -420,7 +420,7 @@ async def start_guest_api(self):
         logger.debug(f"started guest API for {self.vm_id}")
 
     async def stop_guest_api(self):
-        if self.guest_api_process:
+        if self.guest_api_process and self.guest_api_process._popen:
             self.guest_api_process.terminate()
 
     async def teardown(self):

From e8fd32b2dfe61f576ec10bfaf58b12e6adf76001 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 11 May 2022 15:33:05 +0200
Subject: [PATCH 297/990] Feature: Load balancer could not distinguish outdated
 supervisors

The load balancer could not check if a supervisor is running the latest release using the status of a GET response.

This is useful for the healthcheck of the Traefik Proxy to ensure that backend CRNs are running the latest version.
---
 packaging/aleph-vm/DEBIAN/control |  2 +-
 vm_supervisor/supervisor.py       |  3 ++-
 vm_supervisor/views.py            | 26 +++++++++++++++++++++++++-
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index 59ccd5a63..0b213d9d9 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging
 Section: aleph-im
 Priority: Extra
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index d3fb9bcb8..49a0708c5 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -23,7 +23,7 @@
     about_executions,
     about_config,
     status_check_fastapi,
-    about_execution_records,
+    about_execution_records, status_check_version,
 )
 
 logger = logging.getLogger(__name__)
@@ -51,6 +51,7 @@ async def server_version_middleware(
         web.get("/about/executions/records", about_execution_records),
         web.get("/about/config", about_config),
         web.get("/status/check/fastapi", status_check_fastapi),
+        web.get("/status/check/version", status_check_version),
         web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
         web.route("*", "/{suffix:.*}", run_code_from_hostname),
     ]
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index 41c7406cc..161133f30 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -2,7 +2,8 @@
 import logging
 import os.path
 from string import Template
-from typing import Awaitable
+from typing import Awaitable, Optional
+from packaging.version import Version, InvalidVersion
 
 import aiodns
 import aiohttp
@@ -141,3 +142,26 @@ async def status_check_fastapi(request: web.Request):
             "persistent_storage": await status.check_persistent_storage(session),
         }
         return web.json_response(result, status=200 if all(result.values()) else 503)
+
+
+async def status_check_version(request: web.Request):
+    """Check if the software is running a version equal or newer than the given one"""
+    reference_str: Optional[str] = request.query.get("reference")
+    if not reference_str:
+        raise web.HTTPBadRequest(text="Query field '?reference=` must be specified")
+    try:
+        reference = Version(reference_str)
+    except InvalidVersion as error:
+        raise web.HTTPBadRequest(text=error.args[0])
+
+    try:
+        current = Version(get_version_from_git())
+    except InvalidVersion as error:
+        raise web.HTTPServiceUnavailable(text=error.args[0])
+
+    if current >= reference:
+        return web.Response(
+            status=200, text=f"Up-to-date: version {current} >= {reference}"
+        )
+    else:
+        return web.HTTPForbidden(text=f"Outdated: version {current} < {reference}")

From cbabe9478d78d2892109a7f3b8eb3835a1a42de1 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 11 May 2022 16:07:57 +0200
Subject: [PATCH 298/990] Internal: Enhance typing

---
 vm_connector/main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/vm_connector/main.py b/vm_connector/main.py
index 44d4f6d3f..7467ecde9 100644
--- a/vm_connector/main.py
+++ b/vm_connector/main.py
@@ -6,6 +6,7 @@
 from aleph_client.asynchronous import create_post
 from aleph_client.chains.common import get_fallback_private_key
 from aleph_client.chains.ethereum import ETHAccount
+from aleph_client.types import StorageEnum
 from fastapi import FastAPI, Request, HTTPException
 from fastapi.responses import StreamingResponse, Response
 from pydantic import BaseModel
@@ -177,7 +178,7 @@ async def publish_data(body: PostBody):
         ref=None,
         channel=message["channel"],
         inline=True,
-        storage_engine="storage",
+        storage_engine=StorageEnum.storage,
     )
     return {"status": "success"}
 

From 76afc09a9fc56bcef01c52afde846ae64200b731 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 11 May 2022 16:02:00 +0200
Subject: [PATCH 299/990] Change: Use the load balancer instead of api2

This should make requests more reliable with high availability.
---
 examples/example_fastapi/main.py | 2 +-
 guest_api/__main__.py            | 2 +-
 vm_connector/README.md           | 2 +-
 vm_connector/conf.py             | 2 +-
 vm_supervisor/conf.py            | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index 9464837e1..e50bfd1c2 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -151,7 +151,7 @@ async def receive_post(data: Data):
 async def aleph_event(event):
     print("aleph_event", event)
     async with aiohttp.ClientSession(connector=aiohttp.TCPConnector()) as session:
-        async with session.get("https://api2.aleph.im/api/v0/info/public.json") as resp:
+        async with session.get("https://official.aleph.cloud/api/v0/info/public.json") as resp:
             print('RESP', resp)
             resp.raise_for_status()
     return {
diff --git a/guest_api/__main__.py b/guest_api/__main__.py
index fec80dc4d..b2a2f2f62 100644
--- a/guest_api/__main__.py
+++ b/guest_api/__main__.py
@@ -10,7 +10,7 @@
 
 logger = logging.getLogger(__name__)
 
-ALEPH_API_SERVER = "https://api2.aleph.im"
+ALEPH_API_SERVER = "https://official.aleph.cloud"
 ALEPH_VM_CONNECTOR = "http://localhost:4021"
 CACHE_EXPIRES_AFTER = 7 * 24 * 3600  # Seconds
 REDIS_ADDRESS = "redis://localhost"
diff --git a/vm_connector/README.md b/vm_connector/README.md
index acfaf6237..ad86e1a2c 100644
--- a/vm_connector/README.md
+++ b/vm_connector/README.md
@@ -38,6 +38,6 @@ docker run -d -p 4021:4021/tcp --restart=always --name vm-connector alephim/vm-c
 The VM Supervisor can be configured using environment variables:
 
 `API_SERVER` should point to your Aleph Node. 
-Defaults to https://api2.aleph.im
+Defaults to https://official.aleph.cloud
 
 `IPFS_SERVER` should point to your IPFS Gateway, defaults to https://ipfs.aleph.im/ipfs
diff --git a/vm_connector/conf.py b/vm_connector/conf.py
index 893b90a04..7c7558bf2 100644
--- a/vm_connector/conf.py
+++ b/vm_connector/conf.py
@@ -9,7 +9,7 @@
 
 
 class ConnectorSettings(BaseSettings):
-    API_SERVER: Url = "https://api2.aleph.im"
+    API_SERVER: Url = "https://official.aleph.cloud"
     IPFS_SERVER: Url = "https://ipfs.aleph.im/ipfs"
     OFFLINE_TEST_MODE: bool = False
 
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index f758312f2..496f50bfd 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -76,7 +76,7 @@ class Settings(BaseSettings):
     DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf
     DNS_NAMESERVERS: Optional[List[str]] = None
 
-    API_SERVER = "https://api2.aleph.im"
+    API_SERVER = "https://official.aleph.cloud"
     USE_JAILER = True
     # System logs make boot ~2x slower
     PRINT_SYSTEM_LOGS = False

From f937b413217be64f43477587d675411016874e4f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 11 May 2022 17:06:32 +0200
Subject: [PATCH 300/990] Feature: Allow to run CI in a DigitalOcean Droplet

These support nested virtualization, which GitHub Actions does not.
---
 .github/scripts/extract_droplet_ipv4.py       | 11 +++++
 .../workflows/test-integration-fakedata.yml   | 49 -------------------
 .github/workflows/test-on-droplet.yml         | 47 ++++++++++++++++++
 3 files changed, 58 insertions(+), 49 deletions(-)
 create mode 100755 .github/scripts/extract_droplet_ipv4.py
 delete mode 100644 .github/workflows/test-integration-fakedata.yml
 create mode 100644 .github/workflows/test-on-droplet.yml

diff --git a/.github/scripts/extract_droplet_ipv4.py b/.github/scripts/extract_droplet_ipv4.py
new file mode 100755
index 000000000..891058072
--- /dev/null
+++ b/.github/scripts/extract_droplet_ipv4.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+"""
+Extract the IP address of a DigitalOcean Droplet
+from the JSON returned by `doctl compute droplet get $name --output json
+"""
+
+import json
+import sys
+
+droplet_info = json.load(sys.stdin)
+print(droplet_info[0]["networks"]["v4"][0]["ip_address"])
diff --git a/.github/workflows/test-integration-fakedata.yml b/.github/workflows/test-integration-fakedata.yml
deleted file mode 100644
index de7fd7bfe..000000000
--- a/.github/workflows/test-integration-fakedata.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-name: Run VM Supervisor
-on: [push]
-jobs:
-  Run-VM-Supervisor-Fake-Data:
-    runs-on: self-hosted
-    timeout-minutes: 10
-    env:
-      ALEPH_VM_FAKE_DATA: true
-      ALEPH_VM_LINUX_PATH: /opt/vmlinux.bin
-      ALEPH_VM_INIT_TIMEOUT: 20
-
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v2
-
-      - name: Upgrade aleph-message
-        run: pip3 install --upgrade aleph-message
-
-      - name: Build the example squashfs
-        run: |
-          cd examples/volumes
-          bash build_squashfs.sh
-
-      - name: Update the rootfs
-        run: |
-          cd runtimes/aleph-debian-11-python/
-          cp -pr /var/tmp/rootfs-debian ./rootfs
-          bash update_inits.sh
-#          bash ./create_disk_image.sh
-
-      - name: Build VM Connector
-        run: |
-          docker build -t aleph-connector -f docker/vm_connector.dockerfile .
-
-      - name: Run the VM Connector
-        run: |
-          docker stop aleph-connector || true
-          docker run -d --rm -p 8000:8000/tcp \
-            -v $(pwd)/kernels:/opt/kernels:ro \
-            -v $(pwd)/vm_connector:/opt/vm_connector:ro \
-            --name aleph-connector \
-            aleph-connector $@
-
-      - name: Run the main entrypoint
-        run: python3 -m vm_supervisor -p -vv --profile --print-settings --system-logs --benchmark=1
-
-      - name: Stop the VM Connector
-        run: |
-          docker stop aleph-connector
diff --git a/.github/workflows/test-on-droplet.yml b/.github/workflows/test-on-droplet.yml
new file mode 100644
index 000000000..6d0b285a2
--- /dev/null
+++ b/.github/workflows/test-on-droplet.yml
@@ -0,0 +1,47 @@
+name: "Run tests on DigitalOcean Droplet"
+on:
+  push
+
+jobs:
+  build_deb:
+    name: "Run in DigitalOcean Droplet"
+    runs-on: ubuntu-latest
+    concurrency: droplet-aleph-vm
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install doctl
+        uses: digitalocean/action-doctl@v2
+        with:
+          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+
+      - name: Setup SSH private key
+        run: |
+          mkdir ~/.ssh
+          echo $DIGITALOCEAN_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519
+          chmod 0700 ~/.ssh
+          chmod 0600 ~/.ssh/id_ed25519
+        env:
+          DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }}
+
+      - name: Create the droplet
+        run: |
+          doctl compute droplet create --image debian-11-x64 \
+          --size s-1vcpu-1gb --region ams3 aleph-vm-ci \
+          --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5
+
+      - name: Wait for the system to setup and boot
+        run: sleep 60
+
+      - run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          echo $DROPLET_IPV4
+          echo ${DROPLET_IPV4}
+          ssh -o "StrictHostKeyChecking no" root@${DROPLET_IPV4} ls /etc 
+
+      - name: Cleanup
+        if: always()
+        run: |
+          doctl compute droplet delete -f aleph-vm-ci

From 3ebfbb0f27f558ad74fd8b02077dbfccb6b7c6d2 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 11 May 2022 18:56:22 +0200
Subject: [PATCH 301/990] Tests: Install from package in a Droplet

---
 .github/workflows/test-on-droplet.yml | 29 +++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test-on-droplet.yml b/.github/workflows/test-on-droplet.yml
index 6d0b285a2..87ac29c06 100644
--- a/.github/workflows/test-on-droplet.yml
+++ b/.github/workflows/test-on-droplet.yml
@@ -11,6 +11,9 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+        with:
+          # Fetch the whole history for all tags and branches (required for aleph.__version__)
+          fetch-depth: 0
 
       - name: Install doctl
         uses: digitalocean/action-doctl@v2
@@ -26,20 +29,34 @@ jobs:
         env:
           DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }}
 
-      - name: Create the droplet
+      - name: Create the Droplet
         run: |
           doctl compute droplet create --image debian-11-x64 \
           --size s-1vcpu-1gb --region ams3 aleph-vm-ci \
           --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5
 
+      - name: Build Debian Package
+        run: |
+          cd packaging && make all-podman-debian-11 && cd ..
+          ls packaging/target
+
       - name: Wait for the system to setup and boot
-        run: sleep 60
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
 
-      - run: |
+      - name: Install Aleph-VM on the Droplet
+        run: |
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          echo $DROPLET_IPV4
-          echo ${DROPLET_IPV4}
-          ssh -o "StrictHostKeyChecking no" root@${DROPLET_IPV4} ls /etc 
+          ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
+          
+          ssh root@${DROPLET_IPV4} "apt-get update"
+          ssh root@${DROPLET_IPV4} "apt-get upgrade -y"
+          ssh root@${DROPLET_IPV4} "apt-get install -y docker.io apparmor-profiles"
+          ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
+          
+          scp packaging/target/aleph-vm.debian-11.deb root@${DROPLET_IPV4}:/opt
+          ssh root@${DROPLET_IPV4} "apt install -y /opt/aleph-vm.debian-11.deb"
 
       - name: Cleanup
         if: always()

From 3f74a36477cc4bce94ceb94ddb8e985532809856 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 18 May 2022 17:23:48 +0200
Subject: [PATCH 302/990] Tests: Test fastapi check in the Actions Droplet

---
 .github/workflows/test-on-droplet.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/workflows/test-on-droplet.yml b/.github/workflows/test-on-droplet.yml
index 87ac29c06..04039fe1b 100644
--- a/.github/workflows/test-on-droplet.yml
+++ b/.github/workflows/test-on-droplet.yml
@@ -57,6 +57,15 @@ jobs:
           
           scp packaging/target/aleph-vm.debian-11.deb root@${DROPLET_IPV4}:/opt
           ssh root@${DROPLET_IPV4} "apt install -y /opt/aleph-vm.debian-11.deb"
+          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
+          ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
+
+      - name: Test Aleph-VM on the Droplet
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          
+          sleep 3
+          curl --retry 5 "http://${DROPLET_IPV4}:4020/status/check/fastapi"
 
       - name: Cleanup
         if: always()

From a4499577e74e705242ed176e6087898710dde08b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 19 May 2022 16:54:16 +0200
Subject: [PATCH 303/990] Fix: Software version could not be fetched from apt

---
 vm_supervisor/__init__.py | 22 ++++++++++++++++++----
 vm_supervisor/views.py    |  6 +++---
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/vm_supervisor/__init__.py b/vm_supervisor/__init__.py
index efd0503dc..88495ea73 100644
--- a/vm_supervisor/__init__.py
+++ b/vm_supervisor/__init__.py
@@ -1,22 +1,36 @@
 import logging
 from subprocess import check_output, CalledProcessError
+from typing import Optional
 
 logger = logging.getLogger(__name__)
 
 
-def get_version_from_git() -> str:
+def get_version_from_git() -> Optional[str]:
     try:
         return check_output(("git", "describe", "--tags")).strip().decode()
     except FileNotFoundError:
         logger.warning("git not found")
-        return "unknown-version"
+        return None
     except CalledProcessError:
         logger.warning("git description not available")
-        return "unavailable-version"
+        return None
+
+
+def get_version_from_apt() -> Optional[str]:
+    try:
+        import apt
+        return apt.Cache().get('aleph-vm').installed.version
+    except ImportError:
+        logger.warning("apt version not available")
+        return None
+
+
+def get_version() -> Optional[str]:
+    return get_version_from_git() or get_version_from_apt()
 
 
 # The version number is harcoded in the following line when packaging the software
-__version__ = get_version_from_git()
+__version__ = get_version() or "version-unavailable"
 
 
 from . import supervisor
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index 161133f30..701dc455e 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -10,7 +10,7 @@
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPNotFound
 
-from . import status, get_version_from_git
+from . import status, get_version
 from .conf import settings
 from .metrics import get_execution_records
 from .models import VmHash
@@ -126,7 +126,7 @@ async def index(request: web.Request):
             multiaddr_dns4=f"/dns4/{settings.DOMAIN_NAME}/tcp/443/https",
             multiaddr_dns6=f"/dns6/{settings.DOMAIN_NAME}/tcp/443/https",
             check_fastapi_vm_id=settings.CHECK_FASTAPI_VM_ID,
-            version=get_version_from_git(),
+            version=get_version(),
         )
     return web.Response(content_type="text/html", body=body)
 
@@ -155,7 +155,7 @@ async def status_check_version(request: web.Request):
         raise web.HTTPBadRequest(text=error.args[0])
 
     try:
-        current = Version(get_version_from_git())
+        current = Version(get_version())
     except InvalidVersion as error:
         raise web.HTTPServiceUnavailable(text=error.args[0])
 

From 084156b68a30bc27880a595fc2c8b937ca0957d3 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 19 May 2022 17:43:06 +0200
Subject: [PATCH 304/990] Fix: Software version in apt is hardcoded during
 build

---
 vm_supervisor/views.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index 701dc455e..ec4cd962a 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -10,7 +10,7 @@
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPNotFound
 
-from . import status, get_version
+from . import status, __version__
 from .conf import settings
 from .metrics import get_execution_records
 from .models import VmHash
@@ -126,7 +126,7 @@ async def index(request: web.Request):
             multiaddr_dns4=f"/dns4/{settings.DOMAIN_NAME}/tcp/443/https",
             multiaddr_dns6=f"/dns6/{settings.DOMAIN_NAME}/tcp/443/https",
             check_fastapi_vm_id=settings.CHECK_FASTAPI_VM_ID,
-            version=get_version(),
+            version=__version__,
         )
     return web.Response(content_type="text/html", body=body)
 
@@ -155,7 +155,7 @@ async def status_check_version(request: web.Request):
         raise web.HTTPBadRequest(text=error.args[0])
 
     try:
-        current = Version(get_version())
+        current = Version(__version__)
     except InvalidVersion as error:
         raise web.HTTPServiceUnavailable(text=error.args[0])
 

From 0664812548d82a59177ea789f5e45eb0d7c387e6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 19 May 2022 18:05:02 +0200
Subject: [PATCH 305/990] Doc: Use v 0.2.1 instead of 0.1.10

---
 doc/INSTALL-Debian-11.md    | 2 +-
 doc/INSTALL-Ubuntu-20.04.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index 3f7a62ca7..234a6a6ca 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.10/aleph-vm.debian-11.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.1/aleph-vm.debian-11.deb
 apt install /opt/aleph-vm.debian-11.deb
 ```
 
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
index df061471c..173e5b1d5 100644
--- a/doc/INSTALL-Ubuntu-20.04.md
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.1.10/aleph-vm.ubuntu-20.04.deb
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.1/aleph-vm.ubuntu-20.04.deb
 sudo apt install /opt/aleph-vm.ubuntu-20.04.deb
 ```
 

From 60c86413222c18f939ce72c3280d61c96c354ad8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 23 May 2022 15:33:49 +0200
Subject: [PATCH 306/990] Fix: Lib `packaging` was not added to Dockerfile

---
 docker/vm_supervisor-dev.dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index d891dad8d..52230f5c6 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -5,7 +5,7 @@ FROM debian:bullseye
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     sudo acl curl squashfs-tools git \
     python3 python3-aiohttp python3-msgpack python3-pip python3-aiodns python3-aioredis  \
-    python3-psutil python3-setproctitle python3-sqlalchemy \
+    python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging \
     && rm -rf /var/lib/apt/lists/*
 
 RUN useradd jailman

From 178c800c3099bde3a6d804e1834ed09d25b50fa1 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 31 May 2022 16:13:41 +0200
Subject: [PATCH 307/990] Internal: Require aleph-message>=0.1.19

---
 docker/vm_supervisor-dev.dockerfile | 2 +-
 examples/volumes/Dockerfile         | 2 +-
 packaging/Makefile                  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 52230f5c6..5503869a4 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -19,7 +19,7 @@ RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://github.com/aleph-im/aleph
 RUN ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker
 RUN ln /opt/firecracker/release-*/jailer-v* /opt/firecracker/jailer
 
-RUN pip3 install typing-extensions 'aleph-message>=0.1.18'
+RUN pip3 install typing-extensions 'aleph-message>=0.1.19'
 
 RUN mkdir -p /var/lib/aleph/vm/jailer
 
diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile
index af522dda5..d8d72e780 100644
--- a/examples/volumes/Dockerfile
+++ b/examples/volumes/Dockerfile
@@ -6,6 +6,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 RUN python3 -m venv /opt/venv
-RUN /opt/venv/bin/pip install 'aleph-message>=0.1.18'
+RUN /opt/venv/bin/pip install 'aleph-message>=0.1.19'
 
 CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs
diff --git a/packaging/Makefile b/packaging/Makefile
index 4b6b7e750..261ffd098 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -16,7 +16,7 @@ debian-package-code:
 	cp ../examples/message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message>=0.1.18'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message>=0.1.19'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux

From 7378869ccc6b9a23643c28c9c6d12c355fbf10d6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 30 May 2022 17:28:11 +0200
Subject: [PATCH 308/990] Fix: Concurrent requests could result in a different
 counter value

Co-authored-by: aliel 
---
 vm_supervisor/status.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py
index c2d9a0c08..32a378c90 100644
--- a/vm_supervisor/status.py
+++ b/vm_supervisor/status.py
@@ -82,7 +82,8 @@ async def check_persistent_storage(session: ClientSession) -> bool:
         counter = result["counter"]
         result_2: Dict = await get_json_from_vm(session, "/state/increment")
         counter_2 = result_2["counter"]
-        assert counter_2 == counter + 1
+        # Use >= to handle potential concurrency
+        assert counter_2 >= counter + 1
         return True
     except ClientResponseError:
         return False

From dbd86f917af13995760f59cabcf19c977779afea Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 23 Jun 2022 11:51:34 +0200
Subject: [PATCH 309/990] Chore: Update firecracker to 1.1.0

---
 packaging/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 261ffd098..66c789282 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -29,7 +29,7 @@ debian-package-resources: firecracker-bins vmlinux
 firecracker-bins: target-dir build-dir
 	mkdir -p ./build/firecracker-release
 	# Download latest release
-	curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.0.0/firecracker-v1.0.0-x86_64.tgz | tar -xz --directory ./build/firecracker-release
+	curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.1.0/firecracker-v1.1.0-x86_64.tgz | tar -xz --directory ./build/firecracker-release
 	# Copy binaries:
 	cp ./build/firecracker-release/release-v*/firecracker-v* ./target/firecracker
 	cp ./build/firecracker-release/release-v*/jailer-v* ./target/jailer

From 22d6400f4f59b46a7d579c5d898735e537d5e87b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 22 Jun 2022 19:56:20 +0200
Subject: [PATCH 310/990] Internal: Raise exception VmInitNotConnected when VM
 init cannot be reached

This exception can be used to diagnose when the init system may have crashed and should be investigated.
---
 vm_supervisor/vm/firecracker_microvm.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index ba9e90c7e..5efa3b0cc 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -198,6 +198,10 @@ class VmSetupError(Exception):
     pass
 
 
+class VmInitNotConnected(Exception):
+    pass
+
+
 class AlephFirecrackerVM:
     vm_id: int
     vm_hash: str
@@ -451,7 +455,10 @@ async def communicate(reader, writer, scope):
 
             return response
 
-        reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
+        try:
+            reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
+        except ConnectionRefusedError:
+            raise VmInitNotConnected("MicroVM may have crashed")
         try:
             return await asyncio.wait_for(
                 communicate(reader, writer, scope),

From 36b04a53e1a6a0c6c840b7e775ae4492526d65de Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 22 Jun 2022 19:54:14 +0200
Subject: [PATCH 311/990] Fix: Settings with implicit annotations were not
 displayed

`self.__annotations__` did not include attributes with implicit type annotation (the type of the default value), and those were therefore not displayed when using `--print-settings`
---
 vm_supervisor/conf.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 496f50bfd..219de32e8 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -4,7 +4,7 @@
 from enum import Enum
 from os.path import isfile, join, exists, abspath, isdir
 from subprocess import check_output
-from typing import NewType, Optional, List
+from typing import NewType, Optional, List, Dict, Any
 
 from firecracker.models import FilePath
 from pydantic import BaseSettings, Field
@@ -189,16 +189,20 @@ def setup(self):
                 assert "This should never happen"
 
     def display(self) -> str:
-        annotations = self.__annotations__.copy()
+        attributes: Dict[str, Any] = {}
+
+        for attr in self.__dict__.keys():
+            if attr != attr.upper():
+                # Settings are expected to be ALL_UPPERCASE, other attributes snake_case or CamelCase
+                continue
 
-        for attr in annotations.keys():
             if getattr(self, attr) and attr in self.SENSITIVE_FIELDS:
-                annotations[attr] = ""
+                attributes[attr] = ""
             else:
-                annotations[attr] = getattr(self, attr)
+                attributes[attr] = getattr(self, attr)
 
         return "\n".join(
-            f"{annotation:<27} = {value}" for annotation, value in annotations.items()
+            f"{attribute:<27} = {value}" for attribute, value in attributes.items()
         )
 
     class Config:

From 080028006ca9daa19a58b8895636e20cc80a50b1 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 22 Jun 2022 18:33:04 +0200
Subject: [PATCH 312/990] Fix: Recording VM usage crashed if process had
 stopped beforehand

---
 vm_supervisor/metrics.py                | 14 +++---
 vm_supervisor/models.py                 | 60 +++++++++++++++++--------
 vm_supervisor/vm/firecracker_microvm.py |  2 +-
 3 files changed, 50 insertions(+), 26 deletions(-)

diff --git a/vm_supervisor/metrics.py b/vm_supervisor/metrics.py
index 479080763..db9dbab4a 100644
--- a/vm_supervisor/metrics.py
+++ b/vm_supervisor/metrics.py
@@ -1,7 +1,7 @@
 import logging
 import os
 from os.path import join
-from typing import Iterable
+from typing import Iterable, Optional
 from uuid import UUID
 
 from sqlalchemy import Column, Integer, String, Float, DateTime
@@ -41,13 +41,13 @@ class ExecutionRecord(Base):
     time_started = Column(DateTime)
     time_stopping = Column(DateTime)
 
-    cpu_time_user = Column(Float)
-    cpu_time_system = Column(Float)
+    cpu_time_user: Optional[float] = Column(Float, nullable=True)
+    cpu_time_system: Optional[float] = Column(Float, nullable=True)
 
-    io_read_count = Column(Integer)
-    io_write_count = Column(Integer)
-    io_read_bytes = Column(Integer)
-    io_write_bytes = Column(Integer)
+    io_read_count: Optional[int] = Column(Integer, nullable=True)
+    io_write_count: Optional[int] = Column(Integer, nullable=True)
+    io_read_bytes: Optional[int] = Column(Integer, nullable=True)
+    io_write_bytes: Optional[int] = Column(Integer, nullable=True)
 
     vcpus = Column(Integer, nullable=False)
     memory = Column(Integer, nullable=False)
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 5e7a15bc8..e66fa1341 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -191,25 +191,49 @@ async def record_usage(self):
                 execution_uuid=self.uuid, execution_data=self.to_json()
             )
         pid_info = self.vm.to_dict()
-        await save_record(
-            ExecutionRecord(
-                uuid=str(self.uuid),
-                vm_hash=self.vm_hash,
-                time_defined=self.times.defined_at,
-                time_prepared=self.times.prepared_at,
-                time_started=self.times.started_at,
-                time_stopping=self.times.stopping_at,
-                cpu_time_user=pid_info["process"]["cpu_times"].user,
-                cpu_time_system=pid_info["process"]["cpu_times"].system,
-                io_read_count=pid_info["process"]["io_counters"][0],
-                io_write_count=pid_info["process"]["io_counters"][1],
-                io_read_bytes=pid_info["process"]["io_counters"][2],
-                io_write_bytes=pid_info["process"]["io_counters"][3],
-                vcpus=self.vm.hardware_resources.vcpus,
-                memory=self.vm.hardware_resources.memory,
-                network_tap=self.vm.fvm.network_tap,
+        # Handle cases when the process cannot be accessed
+        if pid_info and pid_info.get("process"):
+            await save_record(
+                ExecutionRecord(
+                    uuid=str(self.uuid),
+                    vm_hash=self.vm_hash,
+                    time_defined=self.times.defined_at,
+                    time_prepared=self.times.prepared_at,
+                    time_started=self.times.started_at,
+                    time_stopping=self.times.stopping_at,
+                    cpu_time_user=pid_info["process"]["cpu_times"].user,
+                    cpu_time_system=pid_info["process"]["cpu_times"].system,
+                    io_read_count=pid_info["process"]["io_counters"][0],
+                    io_write_count=pid_info["process"]["io_counters"][1],
+                    io_read_bytes=pid_info["process"]["io_counters"][2],
+                    io_write_bytes=pid_info["process"]["io_counters"][3],
+                    vcpus=self.vm.hardware_resources.vcpus,
+                    memory=self.vm.hardware_resources.memory,
+                    network_tap=self.vm.fvm.network_tap,
+                )
+            )
+        else:
+            # The process cannot be accessed. It has probably already exited
+            # and its metrics are not available anymore.
+            await save_record(
+                ExecutionRecord(
+                    uuid=str(self.uuid),
+                    vm_hash=self.vm_hash,
+                    time_defined=self.times.defined_at,
+                    time_prepared=self.times.prepared_at,
+                    time_started=self.times.started_at,
+                    time_stopping=self.times.stopping_at,
+                    cpu_time_user=None,
+                    cpu_time_system=None,
+                    io_read_count=None,
+                    io_write_count=None,
+                    io_read_bytes=None,
+                    io_write_bytes=None,
+                    vcpus=self.vm.hardware_resources.vcpus,
+                    memory=self.vm.hardware_resources.memory,
+                    network_tap=self.vm.fvm.network_tap,
+                )
             )
-        )
 
     async def run_code(self, scope: dict = None) -> bytes:
         if not self.vm:
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 5efa3b0cc..dc7c59d14 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -247,7 +247,7 @@ def to_dict(self):
                     "num_ctx_switches": p.num_ctx_switches(),
                 }
             except psutil.NoSuchProcess:
-                logger.warning("Cannot read process metrics (process not found)", exc_info=True)
+                logger.warning("Cannot read process metrics (process not found)")
                 pid_info = None
         else:
             pid_info = None

From 277e5f74674841e33bd2b90a9e5f6378addfb7d4 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 22 Jun 2022 19:34:42 +0200
Subject: [PATCH 313/990] Refactoring: Move creation of VM execution in a
 separate function

---
 vm_supervisor/run.py | 58 ++++++++++++++++++++++++--------------------
 1 file changed, 32 insertions(+), 26 deletions(-)

diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index b881df744..b048d0f7e 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -44,6 +44,37 @@ async def build_event_scope(event) -> Dict[str, Any]:
     }
 
 
+async def create_vm_execution(vm_hash: VmHash) -> VmExecution:
+    message, original_message = await load_updated_message(vm_hash)
+    pool.message_cache[vm_hash] = message
+
+    try:
+        execution = await pool.create_a_vm(
+            vm_hash=vm_hash,
+            program=message.content,
+            original=original_message.content,
+        )
+    except ResourceDownloadError as error:
+        logger.exception(error)
+        pool.forget_vm(vm_hash=vm_hash)
+        raise HTTPBadRequest(reason="Code, runtime or data not available")
+    except FileTooLargeError as error:
+        raise HTTPInternalServerError(reason=error.args[0])
+    except VmSetupError as error:
+        logger.exception(error)
+        pool.forget_vm(vm_hash=vm_hash)
+        raise HTTPInternalServerError(reason="Error during program initialisation")
+    except MicroVMFailedInit as error:
+        logger.exception(error)
+        pool.forget_vm(vm_hash=vm_hash)
+        raise HTTPInternalServerError(reason="Error during runtime initialisation")
+
+    if not execution.vm:
+        raise ValueError("The VM has not been created")
+
+    return execution
+
+
 async def run_code_on_request(
     vm_hash: VmHash, path: str, request: web.Request
 ) -> web.Response:
@@ -54,32 +85,7 @@ async def run_code_on_request(
     execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
-        message, original_message = await load_updated_message(vm_hash)
-        pool.message_cache[vm_hash] = message
-
-        try:
-            execution = await pool.create_a_vm(
-                vm_hash=vm_hash,
-                program=message.content,
-                original=original_message.content,
-            )
-        except ResourceDownloadError as error:
-            logger.exception(error)
-            pool.forget_vm(vm_hash=vm_hash)
-            raise HTTPBadRequest(reason="Code, runtime or data not available")
-        except FileTooLargeError as error:
-            raise HTTPInternalServerError(reason=error.args[0])
-        except VmSetupError as error:
-            logger.exception(error)
-            pool.forget_vm(vm_hash=vm_hash)
-            raise HTTPInternalServerError(reason="Error during program initialisation")
-        except MicroVMFailedInit as error:
-            logger.exception(error)
-            pool.forget_vm(vm_hash=vm_hash)
-            raise HTTPInternalServerError(reason="Error during runtime initialisation")
-
-    if not execution.vm:
-        raise ValueError("The VM has not been created")
+        execution = await create_vm_execution(vm_hash=vm_hash)
 
     logger.debug(f"Using vm={execution.vm.vm_id}")
 

From 5128e0856aa105c4b6cbfc85f0a71234bab9ea9f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 22 Jun 2022 19:35:45 +0200
Subject: [PATCH 314/990] Fix: A VM would not restart if the init process
 crashes

---
 examples/example_fastapi/main.py | 15 +++++++++++++++
 vm_supervisor/run.py             | 12 ++++++++++++
 vm_supervisor/status.py          | 26 ++++++++++++++++++++++++++
 vm_supervisor/views.py           |  2 ++
 4 files changed, 55 insertions(+)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index e50bfd1c2..652e9853a 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import sys
 from datetime import datetime
 from os import listdir
 from typing import Dict
@@ -142,6 +143,20 @@ async def receive_post(data: Data):
     return str(data)
 
 
+class CustomError(Exception):
+    pass
+
+
+@app.get("/raise")
+def raise_error():
+    raise CustomError("Whoops")
+
+
+@app.get("/crash")
+def crash():
+    sys.exit(1)
+
+
 filters = [{
     # "sender": "0xB31B787AdA86c6067701d4C0A250c89C7f1f29A5",
     "channel": "TEST"
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index b048d0f7e..285cf0d14 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -94,6 +94,18 @@ async def run_code_on_request(
     try:
         await execution.becomes_ready()
         result_raw: bytes = await execution.run_code(scope=scope)
+
+        if result_raw == b'':
+            # Missing result from the init process of the virtual machine, not even an error message.
+            # It may have completely crashed.
+
+            # Stop the virtual machine due to failing init.
+            # It will be restarted on a future request.
+            await execution.stop()
+
+            return web.Response(status=502, reason="No response from VM",
+                                text="VM did not respond and was shut down")
+
     except asyncio.TimeoutError:
         logger.warning(
             f"VM{execution.vm.vm_id} did not respond within `resource.seconds`"
diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py
index 32a378c90..a1625b93d 100644
--- a/vm_supervisor/status.py
+++ b/vm_supervisor/status.py
@@ -87,3 +87,29 @@ async def check_persistent_storage(session: ClientSession) -> bool:
         return True
     except ClientResponseError:
         return False
+
+
+async def check_error_raised(session: ClientSession) -> bool:
+    try:
+        async with session.get(f"{CHECK_VM_URL}/raise") as resp:
+            text = await resp.text()
+            return (resp.status == 500 and "Traceback" in text)
+    except ClientResponseError:
+        return False
+
+
+async def check_crash_and_restart(session: ClientSession) -> bool:
+
+    # Crash the VM init.
+    async with session.get(f"{CHECK_VM_URL}/crash") as resp:
+        if resp.status != 502:
+            return False
+
+    # Try loading the index page. A new execution should be created.
+    try:
+        result: Dict = await get_json_from_vm(session, "/")
+        assert result["Example"] == "example_fastapi"
+        return True
+
+    except ClientResponseError:
+        return False
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index ec4cd962a..1c40427df 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -140,6 +140,8 @@ async def status_check_fastapi(request: web.Request):
             "internet": await status.check_internet(session),
             "cache": await status.check_cache(session),
             "persistent_storage": await status.check_persistent_storage(session),
+            "error_handling": await status.check_error_raised(session),
+            "crash_handling": await status.check_crash_and_restart(session),
         }
         return web.json_response(result, status=200 if all(result.values()) else 503)
 

From 73bcb015e0b2f1165db7cd44895d28e4dc7cbf8b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 23 Jun 2022 10:44:50 +0200
Subject: [PATCH 315/990] Chore: Update kernel to 5.10

Firecracker supports Linux kernels 4.20 and 5.10.

This updates the Linux kernel used to 5.10.

The configuration is based on the Firecracker config [1]
on which `make menuconfig` is run to disable the following
fields: CONFIG_INPUT_KEYBOARD CONFIG_INPUT_MISC CONFIG_INPUT_FF_MEMLESS
and CONFIG_SERIO.

[1] https://github.com/firecracker-microvm/firecracker/blob/0fa080b137fd29e5bcd95073473b0a57c3868d86/resources/guest_configs/microvm-kernel-x86_64-5.10.config
---
 .gitignore                                |    3 +
 kernels/build-kernel.sh                   |   18 +
 kernels/microvm-kernel-x86_64-5.10.config | 2932 +++++++++++++++++++++
 packaging/Makefile                        |    2 +-
 4 files changed, 2954 insertions(+), 1 deletion(-)
 create mode 100644 kernels/build-kernel.sh
 create mode 100644 kernels/microvm-kernel-x86_64-5.10.config

diff --git a/.gitignore b/.gitignore
index 49fefe952..cb2fd28d8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,3 +21,6 @@ node_modules
 /packaging/repositories/*/db/
 /packaging/repositories/*/dists/
 /packaging/repositories/*/pool/
+/kernels/linux-*/
+/kernels/linux-*.tar
+/kernels/linux-*.tar.sign
diff --git a/kernels/build-kernel.sh b/kernels/build-kernel.sh
new file mode 100644
index 000000000..99c85c18e
--- /dev/null
+++ b/kernels/build-kernel.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -euf -o pipefail
+
+curl -OL "https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.124.tar.xz"
+curl -OL "https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.124.tar.sign"
+unxz linux-5.10.124.tar.xz
+
+gpg --locate-keys torvalds@kernel.org gregkh@kernel.org
+gpg --verify linux-5.10.124.tar.sign linux-5.10.124.tar
+
+tar -xvf linux-5.10.124.tar
+
+cp microvm-kernel-x86_64-5.10.config linux-5.10.124/.config
+
+cd linux-5.10.124/
+make menuconfig
+
+make -j32 vmlinux
\ No newline at end of file
diff --git a/kernels/microvm-kernel-x86_64-5.10.config b/kernels/microvm-kernel-x86_64-5.10.config
new file mode 100644
index 000000000..411e36c27
--- /dev/null
+++ b/kernels/microvm-kernel-x86_64-5.10.config
@@ -0,0 +1,2932 @@
+#
+# Automatically generated file; DO NOT EDIT.
+# Linux/x86 5.10.124 Kernel Configuration
+#
+CONFIG_CC_VERSION_TEXT="gcc (GCC) 12.1.0"
+CONFIG_CC_IS_GCC=y
+CONFIG_GCC_VERSION=120100
+CONFIG_LD_VERSION=238000000
+CONFIG_CLANG_VERSION=0
+CONFIG_LLD_VERSION=0
+CONFIG_CC_CAN_LINK=y
+CONFIG_CC_CAN_LINK_STATIC=y
+CONFIG_CC_HAS_ASM_GOTO=y
+CONFIG_CC_HAS_ASM_GOTO_OUTPUT=y
+CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT=y
+CONFIG_CC_HAS_ASM_INLINE=y
+CONFIG_IRQ_WORK=y
+CONFIG_BUILDTIME_TABLE_SORT=y
+CONFIG_THREAD_INFO_IN_TASK=y
+
+#
+# General setup
+#
+CONFIG_INIT_ENV_ARG_LIMIT=32
+# CONFIG_COMPILE_TEST is not set
+CONFIG_LOCALVERSION=""
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_BUILD_SALT=""
+CONFIG_HAVE_KERNEL_GZIP=y
+CONFIG_HAVE_KERNEL_BZIP2=y
+CONFIG_HAVE_KERNEL_LZMA=y
+CONFIG_HAVE_KERNEL_XZ=y
+CONFIG_HAVE_KERNEL_LZO=y
+CONFIG_HAVE_KERNEL_LZ4=y
+CONFIG_HAVE_KERNEL_ZSTD=y
+CONFIG_KERNEL_GZIP=y
+# CONFIG_KERNEL_BZIP2 is not set
+# CONFIG_KERNEL_LZMA is not set
+# CONFIG_KERNEL_XZ is not set
+# CONFIG_KERNEL_LZO is not set
+# CONFIG_KERNEL_LZ4 is not set
+# CONFIG_KERNEL_ZSTD is not set
+CONFIG_DEFAULT_INIT=""
+CONFIG_DEFAULT_HOSTNAME="(none)"
+CONFIG_SWAP=y
+CONFIG_SYSVIPC=y
+CONFIG_SYSVIPC_SYSCTL=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_POSIX_MQUEUE_SYSCTL=y
+# CONFIG_WATCH_QUEUE is not set
+CONFIG_CROSS_MEMORY_ATTACH=y
+# CONFIG_USELIB is not set
+CONFIG_AUDIT=y
+CONFIG_HAVE_ARCH_AUDITSYSCALL=y
+CONFIG_AUDITSYSCALL=y
+
+#
+# IRQ subsystem
+#
+CONFIG_GENERIC_IRQ_PROBE=y
+CONFIG_GENERIC_IRQ_SHOW=y
+CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK=y
+CONFIG_GENERIC_PENDING_IRQ=y
+CONFIG_GENERIC_IRQ_MIGRATION=y
+CONFIG_HARDIRQS_SW_RESEND=y
+CONFIG_IRQ_DOMAIN=y
+CONFIG_IRQ_DOMAIN_HIERARCHY=y
+CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR=y
+CONFIG_GENERIC_IRQ_RESERVATION_MODE=y
+CONFIG_IRQ_FORCED_THREADING=y
+CONFIG_SPARSE_IRQ=y
+# CONFIG_GENERIC_IRQ_DEBUGFS is not set
+# end of IRQ subsystem
+
+CONFIG_CLOCKSOURCE_WATCHDOG=y
+CONFIG_ARCH_CLOCKSOURCE_INIT=y
+CONFIG_CLOCKSOURCE_VALIDATE_LAST_CYCLE=y
+CONFIG_GENERIC_TIME_VSYSCALL=y
+CONFIG_GENERIC_CLOCKEVENTS=y
+CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y
+CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST=y
+CONFIG_GENERIC_CMOS_UPDATE=y
+CONFIG_HAVE_POSIX_CPU_TIMERS_TASK_WORK=y
+CONFIG_POSIX_CPU_TIMERS_TASK_WORK=y
+
+#
+# Timers subsystem
+#
+CONFIG_TICK_ONESHOT=y
+CONFIG_NO_HZ_COMMON=y
+# CONFIG_HZ_PERIODIC is not set
+CONFIG_NO_HZ_IDLE=y
+# CONFIG_NO_HZ_FULL is not set
+CONFIG_NO_HZ=y
+CONFIG_HIGH_RES_TIMERS=y
+# end of Timers subsystem
+
+CONFIG_PREEMPT_NONE=y
+# CONFIG_PREEMPT_VOLUNTARY is not set
+# CONFIG_PREEMPT is not set
+
+#
+# CPU/Task time and stats accounting
+#
+CONFIG_TICK_CPU_ACCOUNTING=y
+# CONFIG_VIRT_CPU_ACCOUNTING_GEN is not set
+# CONFIG_IRQ_TIME_ACCOUNTING is not set
+CONFIG_HAVE_SCHED_AVG_IRQ=y
+CONFIG_BSD_PROCESS_ACCT=y
+CONFIG_BSD_PROCESS_ACCT_V3=y
+CONFIG_TASKSTATS=y
+CONFIG_TASK_DELAY_ACCT=y
+CONFIG_TASK_XACCT=y
+CONFIG_TASK_IO_ACCOUNTING=y
+# CONFIG_PSI is not set
+# end of CPU/Task time and stats accounting
+
+CONFIG_CPU_ISOLATION=y
+
+#
+# RCU Subsystem
+#
+CONFIG_TREE_RCU=y
+# CONFIG_RCU_EXPERT is not set
+CONFIG_SRCU=y
+CONFIG_TREE_SRCU=y
+CONFIG_TASKS_RCU_GENERIC=y
+CONFIG_TASKS_TRACE_RCU=y
+CONFIG_RCU_STALL_COMMON=y
+CONFIG_RCU_NEED_SEGCBLIST=y
+# end of RCU Subsystem
+
+CONFIG_BUILD_BIN2C=y
+# CONFIG_IKCONFIG is not set
+# CONFIG_IKHEADERS is not set
+CONFIG_LOG_BUF_SHIFT=21
+CONFIG_LOG_CPU_MAX_BUF_SHIFT=12
+CONFIG_PRINTK_SAFE_LOG_BUF_SHIFT=13
+CONFIG_HAVE_UNSTABLE_SCHED_CLOCK=y
+
+#
+# Scheduler features
+#
+# CONFIG_UCLAMP_TASK is not set
+# end of Scheduler features
+
+CONFIG_ARCH_SUPPORTS_NUMA_BALANCING=y
+CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH=y
+CONFIG_CC_HAS_INT128=y
+CONFIG_ARCH_SUPPORTS_INT128=y
+CONFIG_NUMA_BALANCING=y
+# CONFIG_NUMA_BALANCING_DEFAULT_ENABLED is not set
+CONFIG_CGROUPS=y
+CONFIG_PAGE_COUNTER=y
+CONFIG_MEMCG=y
+CONFIG_MEMCG_SWAP=y
+CONFIG_MEMCG_KMEM=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_WRITEBACK=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_FAIR_GROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_RT_GROUP_SCHED=y
+CONFIG_CGROUP_PIDS=y
+# CONFIG_CGROUP_RDMA is not set
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CGROUP_HUGETLB=y
+CONFIG_CPUSETS=y
+CONFIG_PROC_PID_CPUSET=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+# CONFIG_CGROUP_DEBUG is not set
+CONFIG_SOCK_CGROUP_DATA=y
+CONFIG_NAMESPACES=y
+CONFIG_UTS_NS=y
+CONFIG_TIME_NS=y
+CONFIG_IPC_NS=y
+CONFIG_USER_NS=y
+CONFIG_PID_NS=y
+CONFIG_NET_NS=y
+# CONFIG_CHECKPOINT_RESTORE is not set
+CONFIG_SCHED_AUTOGROUP=y
+# CONFIG_SYSFS_DEPRECATED is not set
+CONFIG_RELAY=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_INITRAMFS_SOURCE=""
+CONFIG_RD_GZIP=y
+CONFIG_RD_BZIP2=y
+CONFIG_RD_LZMA=y
+CONFIG_RD_XZ=y
+CONFIG_RD_LZO=y
+CONFIG_RD_LZ4=y
+CONFIG_RD_ZSTD=y
+# CONFIG_BOOT_CONFIG is not set
+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE=y
+# CONFIG_CC_OPTIMIZE_FOR_SIZE is not set
+CONFIG_LD_ORPHAN_WARN=y
+CONFIG_SYSCTL=y
+CONFIG_HAVE_UID16=y
+CONFIG_SYSCTL_EXCEPTION_TRACE=y
+CONFIG_HAVE_PCSPKR_PLATFORM=y
+CONFIG_BPF=y
+# CONFIG_EXPERT is not set
+CONFIG_UID16=y
+CONFIG_MULTIUSER=y
+CONFIG_SGETMASK_SYSCALL=y
+CONFIG_SYSFS_SYSCALL=y
+CONFIG_FHANDLE=y
+CONFIG_POSIX_TIMERS=y
+CONFIG_PRINTK=y
+CONFIG_PRINTK_NMI=y
+CONFIG_BUG=y
+CONFIG_ELF_CORE=y
+CONFIG_PCSPKR_PLATFORM=y
+CONFIG_BASE_FULL=y
+CONFIG_FUTEX=y
+CONFIG_FUTEX_PI=y
+CONFIG_EPOLL=y
+CONFIG_SIGNALFD=y
+CONFIG_TIMERFD=y
+CONFIG_EVENTFD=y
+CONFIG_SHMEM=y
+CONFIG_AIO=y
+CONFIG_IO_URING=y
+CONFIG_ADVISE_SYSCALLS=y
+CONFIG_HAVE_ARCH_USERFAULTFD_WP=y
+CONFIG_MEMBARRIER=y
+CONFIG_KALLSYMS=y
+# CONFIG_KALLSYMS_ALL is not set
+CONFIG_KALLSYMS_ABSOLUTE_PERCPU=y
+CONFIG_KALLSYMS_BASE_RELATIVE=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
+CONFIG_BPF_JIT_ALWAYS_ON=y
+CONFIG_BPF_JIT_DEFAULT_ON=y
+# CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set
+# CONFIG_BPF_PRELOAD is not set
+CONFIG_USERFAULTFD=y
+CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y
+CONFIG_RSEQ=y
+# CONFIG_EMBEDDED is not set
+CONFIG_HAVE_PERF_EVENTS=y
+
+#
+# Kernel Performance Events And Counters
+#
+CONFIG_PERF_EVENTS=y
+# CONFIG_DEBUG_PERF_USE_VMALLOC is not set
+# end of Kernel Performance Events And Counters
+
+CONFIG_VM_EVENT_COUNTERS=y
+CONFIG_SLUB_DEBUG=y
+# CONFIG_COMPAT_BRK is not set
+# CONFIG_SLAB is not set
+CONFIG_SLUB=y
+CONFIG_SLAB_MERGE_DEFAULT=y
+# CONFIG_SLAB_FREELIST_RANDOM is not set
+CONFIG_SLAB_FREELIST_HARDENED=y
+# CONFIG_SHUFFLE_PAGE_ALLOCATOR is not set
+CONFIG_SLUB_CPU_PARTIAL=y
+CONFIG_SYSTEM_DATA_VERIFICATION=y
+CONFIG_PROFILING=y
+# end of General setup
+
+CONFIG_64BIT=y
+CONFIG_X86_64=y
+CONFIG_X86=y
+CONFIG_INSTRUCTION_DECODER=y
+CONFIG_OUTPUT_FORMAT="elf64-x86-64"
+CONFIG_LOCKDEP_SUPPORT=y
+CONFIG_STACKTRACE_SUPPORT=y
+CONFIG_MMU=y
+CONFIG_ARCH_MMAP_RND_BITS_MIN=28
+CONFIG_ARCH_MMAP_RND_BITS_MAX=32
+CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN=8
+CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX=16
+CONFIG_GENERIC_ISA_DMA=y
+CONFIG_GENERIC_BUG=y
+CONFIG_GENERIC_BUG_RELATIVE_POINTERS=y
+CONFIG_ARCH_MAY_HAVE_PC_FDC=y
+CONFIG_GENERIC_CALIBRATE_DELAY=y
+CONFIG_ARCH_HAS_CPU_RELAX=y
+CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
+CONFIG_ARCH_HAS_FILTER_PGPROT=y
+CONFIG_HAVE_SETUP_PER_CPU_AREA=y
+CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK=y
+CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK=y
+CONFIG_ARCH_HIBERNATION_POSSIBLE=y
+CONFIG_ARCH_SUSPEND_POSSIBLE=y
+CONFIG_ARCH_WANT_GENERAL_HUGETLB=y
+CONFIG_ZONE_DMA32=y
+CONFIG_AUDIT_ARCH=y
+CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
+CONFIG_X86_64_SMP=y
+CONFIG_ARCH_SUPPORTS_UPROBES=y
+CONFIG_FIX_EARLYCON_MEM=y
+CONFIG_DYNAMIC_PHYSICAL_MASK=y
+CONFIG_PGTABLE_LEVELS=4
+CONFIG_CC_HAS_SANE_STACKPROTECTOR=y
+
+#
+# Processor type and features
+#
+CONFIG_ZONE_DMA=y
+CONFIG_SMP=y
+CONFIG_X86_FEATURE_NAMES=y
+CONFIG_X86_X2APIC=y
+CONFIG_X86_MPPARSE=y
+# CONFIG_GOLDFISH is not set
+CONFIG_RETPOLINE=y
+# CONFIG_X86_CPU_RESCTRL is not set
+# CONFIG_X86_EXTENDED_PLATFORM is not set
+# CONFIG_X86_AMD_PLATFORM_DEVICE is not set
+CONFIG_SCHED_OMIT_FRAME_POINTER=y
+CONFIG_HYPERVISOR_GUEST=y
+CONFIG_PARAVIRT=y
+# CONFIG_PARAVIRT_DEBUG is not set
+CONFIG_PARAVIRT_SPINLOCKS=y
+CONFIG_X86_HV_CALLBACK_VECTOR=y
+# CONFIG_XEN is not set
+CONFIG_KVM_GUEST=y
+CONFIG_ARCH_CPUIDLE_HALTPOLL=y
+# CONFIG_PVH is not set
+CONFIG_PARAVIRT_TIME_ACCOUNTING=y
+CONFIG_PARAVIRT_CLOCK=y
+# CONFIG_ACRN_GUEST is not set
+# CONFIG_MK8 is not set
+# CONFIG_MPSC is not set
+# CONFIG_MCORE2 is not set
+# CONFIG_MATOM is not set
+CONFIG_GENERIC_CPU=y
+CONFIG_X86_INTERNODE_CACHE_SHIFT=6
+CONFIG_X86_L1_CACHE_SHIFT=6
+CONFIG_X86_TSC=y
+CONFIG_X86_CMPXCHG64=y
+CONFIG_X86_CMOV=y
+CONFIG_X86_MINIMUM_CPU_FAMILY=64
+CONFIG_X86_DEBUGCTLMSR=y
+CONFIG_IA32_FEAT_CTL=y
+CONFIG_X86_VMX_FEATURE_NAMES=y
+CONFIG_CPU_SUP_INTEL=y
+CONFIG_CPU_SUP_AMD=y
+CONFIG_CPU_SUP_HYGON=y
+CONFIG_CPU_SUP_CENTAUR=y
+CONFIG_CPU_SUP_ZHAOXIN=y
+CONFIG_HPET_TIMER=y
+CONFIG_DMI=y
+# CONFIG_MAXSMP is not set
+CONFIG_NR_CPUS_RANGE_BEGIN=2
+CONFIG_NR_CPUS_RANGE_END=512
+CONFIG_NR_CPUS_DEFAULT=64
+CONFIG_NR_CPUS=128
+CONFIG_SCHED_SMT=y
+CONFIG_SCHED_MC=y
+CONFIG_SCHED_MC_PRIO=y
+CONFIG_X86_LOCAL_APIC=y
+CONFIG_X86_IO_APIC=y
+CONFIG_X86_REROUTE_FOR_BROKEN_BOOT_IRQS=y
+# CONFIG_X86_MCE is not set
+
+#
+# Performance monitoring
+#
+# CONFIG_PERF_EVENTS_AMD_POWER is not set
+# end of Performance monitoring
+
+CONFIG_X86_16BIT=y
+CONFIG_X86_ESPFIX64=y
+CONFIG_X86_VSYSCALL_EMULATION=y
+CONFIG_X86_IOPL_IOPERM=y
+# CONFIG_MICROCODE is not set
+CONFIG_X86_MSR=y
+CONFIG_X86_CPUID=y
+# CONFIG_X86_5LEVEL is not set
+CONFIG_X86_DIRECT_GBPAGES=y
+# CONFIG_X86_CPA_STATISTICS is not set
+CONFIG_AMD_MEM_ENCRYPT=y
+# CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT is not set
+CONFIG_NUMA=y
+# CONFIG_NUMA_EMU is not set
+CONFIG_NODES_SHIFT=10
+CONFIG_ARCH_SPARSEMEM_ENABLE=y
+CONFIG_ARCH_SPARSEMEM_DEFAULT=y
+CONFIG_ARCH_SELECT_MEMORY_MODEL=y
+CONFIG_ARCH_MEMORY_PROBE=y
+CONFIG_ARCH_PROC_KCORE_TEXT=y
+CONFIG_ILLEGAL_POINTER_VALUE=0xdead000000000000
+# CONFIG_X86_PMEM_LEGACY is not set
+CONFIG_X86_CHECK_BIOS_CORRUPTION=y
+CONFIG_X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK=y
+CONFIG_X86_RESERVE_LOW=64
+CONFIG_MTRR=y
+CONFIG_MTRR_SANITIZER=y
+CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT=0
+CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT=1
+CONFIG_X86_PAT=y
+CONFIG_ARCH_USES_PG_UNCACHED=y
+CONFIG_ARCH_RANDOM=y
+CONFIG_X86_SMAP=y
+CONFIG_X86_UMIP=y
+CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS=y
+CONFIG_X86_INTEL_TSX_MODE_OFF=y
+# CONFIG_X86_INTEL_TSX_MODE_ON is not set
+# CONFIG_X86_INTEL_TSX_MODE_AUTO is not set
+# CONFIG_EFI is not set
+# CONFIG_HZ_100 is not set
+CONFIG_HZ_250=y
+# CONFIG_HZ_300 is not set
+# CONFIG_HZ_1000 is not set
+CONFIG_HZ=250
+CONFIG_SCHED_HRTICK=y
+# CONFIG_KEXEC is not set
+CONFIG_KEXEC_FILE=y
+CONFIG_ARCH_HAS_KEXEC_PURGATORY=y
+# CONFIG_KEXEC_SIG is not set
+# CONFIG_CRASH_DUMP is not set
+CONFIG_PHYSICAL_START=0x1000000
+CONFIG_RELOCATABLE=y
+# CONFIG_RANDOMIZE_BASE is not set
+CONFIG_PHYSICAL_ALIGN=0x1000000
+CONFIG_HOTPLUG_CPU=y
+# CONFIG_BOOTPARAM_HOTPLUG_CPU0 is not set
+# CONFIG_DEBUG_HOTPLUG_CPU0 is not set
+# CONFIG_COMPAT_VDSO is not set
+CONFIG_LEGACY_VSYSCALL_EMULATE=y
+# CONFIG_LEGACY_VSYSCALL_XONLY is not set
+# CONFIG_LEGACY_VSYSCALL_NONE is not set
+# CONFIG_CMDLINE_BOOL is not set
+CONFIG_MODIFY_LDT_SYSCALL=y
+CONFIG_HAVE_LIVEPATCH=y
+# end of Processor type and features
+
+CONFIG_ARCH_HAS_ADD_PAGES=y
+CONFIG_ARCH_ENABLE_MEMORY_HOTPLUG=y
+CONFIG_ARCH_ENABLE_MEMORY_HOTREMOVE=y
+CONFIG_USE_PERCPU_NUMA_NODE_ID=y
+CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK=y
+CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION=y
+CONFIG_ARCH_ENABLE_THP_MIGRATION=y
+
+#
+# Power management and ACPI options
+#
+CONFIG_ARCH_HIBERNATION_HEADER=y
+# CONFIG_SUSPEND is not set
+CONFIG_HIBERNATE_CALLBACKS=y
+CONFIG_HIBERNATION=y
+CONFIG_HIBERNATION_SNAPSHOT_DEV=y
+CONFIG_PM_STD_PARTITION=""
+CONFIG_PM_SLEEP=y
+CONFIG_PM_SLEEP_SMP=y
+# CONFIG_PM_AUTOSLEEP is not set
+# CONFIG_PM_WAKELOCKS is not set
+CONFIG_PM=y
+# CONFIG_PM_DEBUG is not set
+# CONFIG_WQ_POWER_EFFICIENT_DEFAULT is not set
+# CONFIG_ENERGY_MODEL is not set
+CONFIG_ARCH_SUPPORTS_ACPI=y
+CONFIG_ACPI=y
+CONFIG_ACPI_LEGACY_TABLES_LOOKUP=y
+CONFIG_ARCH_MIGHT_HAVE_ACPI_PDC=y
+CONFIG_ACPI_SYSTEM_POWER_STATES_SUPPORT=y
+# CONFIG_ACPI_DEBUGGER is not set
+CONFIG_ACPI_SPCR_TABLE=y
+CONFIG_ACPI_LPIT=y
+CONFIG_ACPI_SLEEP=y
+CONFIG_ACPI_REV_OVERRIDE_POSSIBLE=y
+# CONFIG_ACPI_EC_DEBUGFS is not set
+CONFIG_ACPI_AC=y
+CONFIG_ACPI_BATTERY=y
+CONFIG_ACPI_BUTTON=y
+CONFIG_ACPI_FAN=y
+# CONFIG_ACPI_TAD is not set
+# CONFIG_ACPI_DOCK is not set
+CONFIG_ACPI_CPU_FREQ_PSS=y
+CONFIG_ACPI_PROCESSOR_CSTATE=y
+CONFIG_ACPI_PROCESSOR_IDLE=y
+CONFIG_ACPI_CPPC_LIB=y
+CONFIG_ACPI_PROCESSOR=y
+CONFIG_ACPI_HOTPLUG_CPU=y
+# CONFIG_ACPI_PROCESSOR_AGGREGATOR is not set
+CONFIG_ACPI_THERMAL=y
+CONFIG_ARCH_HAS_ACPI_TABLE_UPGRADE=y
+CONFIG_ACPI_TABLE_UPGRADE=y
+# CONFIG_ACPI_DEBUG is not set
+CONFIG_ACPI_CONTAINER=y
+# CONFIG_ACPI_HOTPLUG_MEMORY is not set
+# CONFIG_ACPI_SBS is not set
+# CONFIG_ACPI_HED is not set
+# CONFIG_ACPI_CUSTOM_METHOD is not set
+# CONFIG_ACPI_NFIT is not set
+# CONFIG_ACPI_NUMA is not set
+CONFIG_HAVE_ACPI_APEI=y
+CONFIG_HAVE_ACPI_APEI_NMI=y
+# CONFIG_ACPI_APEI is not set
+# CONFIG_ACPI_DPTF is not set
+# CONFIG_ACPI_CONFIGFS is not set
+# CONFIG_PMIC_OPREGION is not set
+CONFIG_X86_PM_TIMER=y
+# CONFIG_SFI is not set
+
+#
+# CPU Frequency scaling
+#
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_GOV_ATTR_SET=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
+# CONFIG_CPU_FREQ_DEFAULT_GOV_POWERSAVE is not set
+# CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE is not set
+# CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL is not set
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+# CONFIG_CPU_FREQ_GOV_POWERSAVE is not set
+# CONFIG_CPU_FREQ_GOV_USERSPACE is not set
+# CONFIG_CPU_FREQ_GOV_ONDEMAND is not set
+# CONFIG_CPU_FREQ_GOV_CONSERVATIVE is not set
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+
+#
+# CPU frequency scaling drivers
+#
+CONFIG_X86_INTEL_PSTATE=y
+# CONFIG_X86_PCC_CPUFREQ is not set
+# CONFIG_X86_ACPI_CPUFREQ is not set
+# CONFIG_X86_SPEEDSTEP_CENTRINO is not set
+# CONFIG_X86_P4_CLOCKMOD is not set
+
+#
+# shared options
+#
+# end of CPU Frequency scaling
+
+#
+# CPU Idle
+#
+CONFIG_CPU_IDLE=y
+CONFIG_CPU_IDLE_GOV_LADDER=y
+CONFIG_CPU_IDLE_GOV_MENU=y
+# CONFIG_CPU_IDLE_GOV_TEO is not set
+# CONFIG_CPU_IDLE_GOV_HALTPOLL is not set
+CONFIG_HALTPOLL_CPUIDLE=y
+# end of CPU Idle
+
+CONFIG_INTEL_IDLE=y
+# end of Power management and ACPI options
+
+#
+# Bus options (PCI etc.)
+#
+CONFIG_ISA_DMA_API=y
+# CONFIG_X86_SYSFB is not set
+# end of Bus options (PCI etc.)
+
+#
+# Binary Emulations
+#
+CONFIG_IA32_EMULATION=y
+# CONFIG_X86_X32 is not set
+CONFIG_COMPAT_32=y
+CONFIG_COMPAT=y
+CONFIG_COMPAT_FOR_U64_ALIGNMENT=y
+CONFIG_SYSVIPC_COMPAT=y
+# end of Binary Emulations
+
+#
+# Firmware Drivers
+#
+# CONFIG_EDD is not set
+CONFIG_FIRMWARE_MEMMAP=y
+CONFIG_DMIID=y
+# CONFIG_DMI_SYSFS is not set
+CONFIG_DMI_SCAN_MACHINE_NON_EFI_FALLBACK=y
+# CONFIG_ISCSI_IBFT is not set
+# CONFIG_FW_CFG_SYSFS is not set
+# CONFIG_GOOGLE_FIRMWARE is not set
+
+#
+# Tegra firmware driver
+#
+# end of Tegra firmware driver
+# end of Firmware Drivers
+
+CONFIG_HAVE_KVM=y
+# CONFIG_VIRTUALIZATION is not set
+CONFIG_AS_AVX512=y
+CONFIG_AS_SHA1_NI=y
+CONFIG_AS_SHA256_NI=y
+CONFIG_AS_TPAUSE=y
+
+#
+# General architecture-dependent options
+#
+CONFIG_CRASH_CORE=y
+CONFIG_KEXEC_CORE=y
+CONFIG_HOTPLUG_SMT=y
+CONFIG_GENERIC_ENTRY=y
+# CONFIG_OPROFILE is not set
+CONFIG_HAVE_OPROFILE=y
+CONFIG_OPROFILE_NMI_TIMER=y
+CONFIG_KPROBES=y
+CONFIG_JUMP_LABEL=y
+# CONFIG_STATIC_KEYS_SELFTEST is not set
+# CONFIG_STATIC_CALL_SELFTEST is not set
+CONFIG_OPTPROBES=y
+CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y
+CONFIG_ARCH_USE_BUILTIN_BSWAP=y
+CONFIG_KRETPROBES=y
+CONFIG_HAVE_IOREMAP_PROT=y
+CONFIG_HAVE_KPROBES=y
+CONFIG_HAVE_KRETPROBES=y
+CONFIG_HAVE_OPTPROBES=y
+CONFIG_HAVE_KPROBES_ON_FTRACE=y
+CONFIG_HAVE_FUNCTION_ERROR_INJECTION=y
+CONFIG_HAVE_NMI=y
+CONFIG_HAVE_ARCH_TRACEHOOK=y
+CONFIG_HAVE_DMA_CONTIGUOUS=y
+CONFIG_GENERIC_SMP_IDLE_THREAD=y
+CONFIG_ARCH_HAS_FORTIFY_SOURCE=y
+CONFIG_ARCH_HAS_SET_MEMORY=y
+CONFIG_ARCH_HAS_SET_DIRECT_MAP=y
+CONFIG_HAVE_ARCH_THREAD_STRUCT_WHITELIST=y
+CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT=y
+CONFIG_HAVE_ASM_MODVERSIONS=y
+CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y
+CONFIG_HAVE_RSEQ=y
+CONFIG_HAVE_FUNCTION_ARG_ACCESS_API=y
+CONFIG_HAVE_HW_BREAKPOINT=y
+CONFIG_HAVE_MIXED_BREAKPOINTS_REGS=y
+CONFIG_HAVE_USER_RETURN_NOTIFIER=y
+CONFIG_HAVE_PERF_EVENTS_NMI=y
+CONFIG_HAVE_HARDLOCKUP_DETECTOR_PERF=y
+CONFIG_HAVE_PERF_REGS=y
+CONFIG_HAVE_PERF_USER_STACK_DUMP=y
+CONFIG_HAVE_ARCH_JUMP_LABEL=y
+CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE=y
+CONFIG_MMU_GATHER_TABLE_FREE=y
+CONFIG_MMU_GATHER_RCU_TABLE_FREE=y
+CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG=y
+CONFIG_HAVE_ALIGNED_STRUCT_PAGE=y
+CONFIG_HAVE_CMPXCHG_LOCAL=y
+CONFIG_HAVE_CMPXCHG_DOUBLE=y
+CONFIG_ARCH_WANT_COMPAT_IPC_PARSE_VERSION=y
+CONFIG_ARCH_WANT_OLD_COMPAT_IPC=y
+CONFIG_HAVE_ARCH_SECCOMP=y
+CONFIG_HAVE_ARCH_SECCOMP_FILTER=y
+CONFIG_SECCOMP=y
+CONFIG_SECCOMP_FILTER=y
+CONFIG_HAVE_ARCH_STACKLEAK=y
+CONFIG_HAVE_STACKPROTECTOR=y
+CONFIG_STACKPROTECTOR=y
+CONFIG_STACKPROTECTOR_STRONG=y
+CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES=y
+CONFIG_HAVE_CONTEXT_TRACKING=y
+CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y
+CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y
+CONFIG_HAVE_MOVE_PMD=y
+CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE=y
+CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD=y
+CONFIG_HAVE_ARCH_HUGE_VMAP=y
+CONFIG_ARCH_WANT_HUGE_PMD_SHARE=y
+CONFIG_HAVE_ARCH_SOFT_DIRTY=y
+CONFIG_HAVE_MOD_ARCH_SPECIFIC=y
+CONFIG_MODULES_USE_ELF_RELA=y
+CONFIG_ARCH_HAS_ELF_RANDOMIZE=y
+CONFIG_HAVE_ARCH_MMAP_RND_BITS=y
+CONFIG_HAVE_EXIT_THREAD=y
+CONFIG_ARCH_MMAP_RND_BITS=28
+CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS=y
+CONFIG_ARCH_MMAP_RND_COMPAT_BITS=8
+CONFIG_HAVE_ARCH_COMPAT_MMAP_BASES=y
+CONFIG_HAVE_STACK_VALIDATION=y
+CONFIG_HAVE_RELIABLE_STACKTRACE=y
+CONFIG_OLD_SIGSUSPEND3=y
+CONFIG_COMPAT_OLD_SIGACTION=y
+CONFIG_COMPAT_32BIT_TIME=y
+CONFIG_HAVE_ARCH_VMAP_STACK=y
+CONFIG_VMAP_STACK=y
+CONFIG_ARCH_HAS_STRICT_KERNEL_RWX=y
+CONFIG_STRICT_KERNEL_RWX=y
+CONFIG_ARCH_HAS_STRICT_MODULE_RWX=y
+CONFIG_STRICT_MODULE_RWX=y
+CONFIG_HAVE_ARCH_PREL32_RELOCATIONS=y
+CONFIG_ARCH_USE_MEMREMAP_PROT=y
+# CONFIG_LOCK_EVENT_COUNTS is not set
+CONFIG_ARCH_HAS_MEM_ENCRYPT=y
+CONFIG_ARCH_HAS_CC_PLATFORM=y
+CONFIG_HAVE_STATIC_CALL=y
+CONFIG_HAVE_STATIC_CALL_INLINE=y
+CONFIG_ARCH_WANT_LD_ORPHAN_WARN=y
+
+#
+# GCOV-based kernel profiling
+#
+# CONFIG_GCOV_KERNEL is not set
+CONFIG_ARCH_HAS_GCOV_PROFILE_ALL=y
+# end of GCOV-based kernel profiling
+
+CONFIG_HAVE_GCC_PLUGINS=y
+CONFIG_GCC_PLUGINS=y
+# CONFIG_GCC_PLUGIN_LATENT_ENTROPY is not set
+# CONFIG_GCC_PLUGIN_RANDSTRUCT is not set
+# end of General architecture-dependent options
+
+CONFIG_RT_MUTEXES=y
+CONFIG_BASE_SMALL=0
+CONFIG_MODULE_SIG_FORMAT=y
+CONFIG_MODULES=y
+CONFIG_MODULE_FORCE_LOAD=y
+CONFIG_MODULE_UNLOAD=y
+# CONFIG_MODULE_FORCE_UNLOAD is not set
+CONFIG_MODVERSIONS=y
+CONFIG_ASM_MODVERSIONS=y
+CONFIG_MODULE_SRCVERSION_ALL=y
+CONFIG_MODULE_SIG=y
+# CONFIG_MODULE_SIG_FORCE is not set
+CONFIG_MODULE_SIG_ALL=y
+# CONFIG_MODULE_SIG_SHA1 is not set
+# CONFIG_MODULE_SIG_SHA224 is not set
+# CONFIG_MODULE_SIG_SHA256 is not set
+# CONFIG_MODULE_SIG_SHA384 is not set
+CONFIG_MODULE_SIG_SHA512=y
+CONFIG_MODULE_SIG_HASH="sha512"
+# CONFIG_MODULE_COMPRESS is not set
+# CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set
+CONFIG_UNUSED_SYMBOLS=y
+CONFIG_MODULES_TREE_LOOKUP=y
+CONFIG_BLOCK=y
+CONFIG_BLK_SCSI_REQUEST=y
+CONFIG_BLK_CGROUP_RWSTAT=y
+CONFIG_BLK_DEV_BSG=y
+CONFIG_BLK_DEV_BSGLIB=y
+CONFIG_BLK_DEV_INTEGRITY=y
+# CONFIG_BLK_DEV_ZONED is not set
+CONFIG_BLK_DEV_THROTTLING=y
+# CONFIG_BLK_DEV_THROTTLING_LOW is not set
+CONFIG_BLK_CMDLINE_PARSER=y
+CONFIG_BLK_WBT=y
+# CONFIG_BLK_CGROUP_IOLATENCY is not set
+# CONFIG_BLK_CGROUP_IOCOST is not set
+CONFIG_BLK_WBT_MQ=y
+CONFIG_BLK_DEBUG_FS=y
+# CONFIG_BLK_SED_OPAL is not set
+# CONFIG_BLK_INLINE_ENCRYPTION is not set
+
+#
+# Partition Types
+#
+CONFIG_PARTITION_ADVANCED=y
+# CONFIG_ACORN_PARTITION is not set
+# CONFIG_AIX_PARTITION is not set
+# CONFIG_OSF_PARTITION is not set
+# CONFIG_AMIGA_PARTITION is not set
+# CONFIG_ATARI_PARTITION is not set
+# CONFIG_MAC_PARTITION is not set
+CONFIG_MSDOS_PARTITION=y
+# CONFIG_BSD_DISKLABEL is not set
+# CONFIG_MINIX_SUBPARTITION is not set
+# CONFIG_SOLARIS_X86_PARTITION is not set
+# CONFIG_UNIXWARE_DISKLABEL is not set
+# CONFIG_LDM_PARTITION is not set
+# CONFIG_SGI_PARTITION is not set
+# CONFIG_ULTRIX_PARTITION is not set
+# CONFIG_SUN_PARTITION is not set
+# CONFIG_KARMA_PARTITION is not set
+# CONFIG_EFI_PARTITION is not set
+# CONFIG_SYSV68_PARTITION is not set
+# CONFIG_CMDLINE_PARTITION is not set
+# end of Partition Types
+
+CONFIG_BLOCK_COMPAT=y
+CONFIG_BLK_MQ_VIRTIO=y
+CONFIG_BLK_PM=y
+
+#
+# IO Schedulers
+#
+# CONFIG_MQ_IOSCHED_DEADLINE is not set
+# CONFIG_MQ_IOSCHED_KYBER is not set
+# CONFIG_IOSCHED_BFQ is not set
+# end of IO Schedulers
+
+CONFIG_ASN1=y
+CONFIG_INLINE_SPIN_UNLOCK_IRQ=y
+CONFIG_INLINE_READ_UNLOCK=y
+CONFIG_INLINE_READ_UNLOCK_IRQ=y
+CONFIG_INLINE_WRITE_UNLOCK=y
+CONFIG_INLINE_WRITE_UNLOCK_IRQ=y
+CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=y
+CONFIG_MUTEX_SPIN_ON_OWNER=y
+CONFIG_RWSEM_SPIN_ON_OWNER=y
+CONFIG_LOCK_SPIN_ON_OWNER=y
+CONFIG_ARCH_USE_QUEUED_SPINLOCKS=y
+CONFIG_QUEUED_SPINLOCKS=y
+CONFIG_ARCH_USE_QUEUED_RWLOCKS=y
+CONFIG_QUEUED_RWLOCKS=y
+CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE=y
+CONFIG_ARCH_HAS_SYNC_CORE_BEFORE_USERMODE=y
+CONFIG_ARCH_HAS_SYSCALL_WRAPPER=y
+CONFIG_FREEZER=y
+
+#
+# Executable file formats
+#
+CONFIG_BINFMT_ELF=y
+CONFIG_COMPAT_BINFMT_ELF=y
+CONFIG_ELFCORE=y
+CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
+CONFIG_BINFMT_SCRIPT=y
+CONFIG_BINFMT_MISC=y
+CONFIG_COREDUMP=y
+# end of Executable file formats
+
+#
+# Memory Management options
+#
+CONFIG_SELECT_MEMORY_MODEL=y
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_SPARSEMEM=y
+CONFIG_NEED_MULTIPLE_NODES=y
+CONFIG_SPARSEMEM_EXTREME=y
+CONFIG_SPARSEMEM_VMEMMAP_ENABLE=y
+CONFIG_SPARSEMEM_VMEMMAP=y
+CONFIG_HAVE_FAST_GUP=y
+CONFIG_NUMA_KEEP_MEMINFO=y
+CONFIG_MEMORY_ISOLATION=y
+CONFIG_HAVE_BOOTMEM_INFO_NODE=y
+CONFIG_MEMORY_HOTPLUG=y
+CONFIG_MEMORY_HOTPLUG_SPARSE=y
+# CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE is not set
+CONFIG_MEMORY_HOTREMOVE=y
+CONFIG_SPLIT_PTLOCK_CPUS=4
+CONFIG_MEMORY_BALLOON=y
+# CONFIG_BALLOON_COMPACTION is not set
+CONFIG_COMPACTION=y
+CONFIG_PAGE_REPORTING=y
+CONFIG_MIGRATION=y
+CONFIG_CONTIG_ALLOC=y
+CONFIG_PHYS_ADDR_T_64BIT=y
+CONFIG_BOUNCE=y
+CONFIG_VIRT_TO_BUS=y
+CONFIG_KSM=y
+CONFIG_DEFAULT_MMAP_MIN_ADDR=4096
+CONFIG_TRANSPARENT_HUGEPAGE=y
+# CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS is not set
+CONFIG_TRANSPARENT_HUGEPAGE_MADVISE=y
+CONFIG_ARCH_WANTS_THP_SWAP=y
+CONFIG_THP_SWAP=y
+CONFIG_CLEANCACHE=y
+CONFIG_FRONTSWAP=y
+# CONFIG_CMA is not set
+# CONFIG_ZSWAP is not set
+CONFIG_ZPOOL=y
+# CONFIG_ZBUD is not set
+# CONFIG_Z3FOLD is not set
+# CONFIG_ZSMALLOC is not set
+CONFIG_GENERIC_EARLY_IOREMAP=y
+# CONFIG_DEFERRED_STRUCT_PAGE_INIT is not set
+# CONFIG_IDLE_PAGE_TRACKING is not set
+CONFIG_ARCH_HAS_PTE_DEVMAP=y
+# CONFIG_ZONE_DEVICE is not set
+CONFIG_ARCH_USES_HIGH_VMA_FLAGS=y
+CONFIG_ARCH_HAS_PKEYS=y
+CONFIG_PERCPU_STATS=y
+# CONFIG_GUP_BENCHMARK is not set
+# CONFIG_READ_ONLY_THP_FOR_FS is not set
+CONFIG_ARCH_HAS_PTE_SPECIAL=y
+# end of Memory Management options
+
+CONFIG_NET=y
+CONFIG_NET_INGRESS=y
+CONFIG_SKB_EXTENSIONS=y
+
+#
+# Networking options
+#
+CONFIG_PACKET=y
+# CONFIG_PACKET_DIAG is not set
+CONFIG_UNIX=y
+CONFIG_UNIX_SCM=y
+# CONFIG_UNIX_DIAG is not set
+# CONFIG_TLS is not set
+CONFIG_XFRM=y
+CONFIG_XFRM_ALGO=y
+CONFIG_XFRM_USER=y
+# CONFIG_XFRM_USER_COMPAT is not set
+# CONFIG_XFRM_INTERFACE is not set
+CONFIG_XFRM_SUB_POLICY=y
+CONFIG_XFRM_MIGRATE=y
+CONFIG_XFRM_STATISTICS=y
+# CONFIG_NET_KEY is not set
+# CONFIG_XDP_SOCKETS is not set
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_ADVANCED_ROUTER=y
+# CONFIG_IP_FIB_TRIE_STATS is not set
+CONFIG_IP_MULTIPLE_TABLES=y
+CONFIG_IP_ROUTE_MULTIPATH=y
+CONFIG_IP_ROUTE_VERBOSE=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_IP_PNP_RARP=y
+# CONFIG_NET_IPIP is not set
+# CONFIG_NET_IPGRE_DEMUX is not set
+CONFIG_IP_MROUTE_COMMON=y
+CONFIG_IP_MROUTE=y
+CONFIG_IP_MROUTE_MULTIPLE_TABLES=y
+CONFIG_IP_PIMSM_V1=y
+CONFIG_IP_PIMSM_V2=y
+CONFIG_SYN_COOKIES=y
+# CONFIG_NET_IPVTI is not set
+# CONFIG_NET_FOU is not set
+# CONFIG_INET_AH is not set
+# CONFIG_INET_ESP is not set
+# CONFIG_INET_IPCOMP is not set
+# CONFIG_INET_DIAG is not set
+CONFIG_TCP_CONG_ADVANCED=y
+# CONFIG_TCP_CONG_BIC is not set
+CONFIG_TCP_CONG_CUBIC=y
+# CONFIG_TCP_CONG_WESTWOOD is not set
+# CONFIG_TCP_CONG_HTCP is not set
+# CONFIG_TCP_CONG_HSTCP is not set
+# CONFIG_TCP_CONG_HYBLA is not set
+# CONFIG_TCP_CONG_VEGAS is not set
+# CONFIG_TCP_CONG_NV is not set
+# CONFIG_TCP_CONG_SCALABLE is not set
+# CONFIG_TCP_CONG_LP is not set
+# CONFIG_TCP_CONG_VENO is not set
+# CONFIG_TCP_CONG_YEAH is not set
+# CONFIG_TCP_CONG_ILLINOIS is not set
+# CONFIG_TCP_CONG_DCTCP is not set
+# CONFIG_TCP_CONG_CDG is not set
+# CONFIG_TCP_CONG_BBR is not set
+CONFIG_DEFAULT_CUBIC=y
+# CONFIG_DEFAULT_RENO is not set
+CONFIG_DEFAULT_TCP_CONG="cubic"
+CONFIG_TCP_MD5SIG=y
+CONFIG_IPV6=y
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_ROUTE_INFO=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+# CONFIG_INET6_AH is not set
+# CONFIG_INET6_ESP is not set
+# CONFIG_INET6_IPCOMP is not set
+# CONFIG_IPV6_MIP6 is not set
+# CONFIG_IPV6_ILA is not set
+# CONFIG_IPV6_VTI is not set
+# CONFIG_IPV6_SIT is not set
+# CONFIG_IPV6_TUNNEL is not set
+CONFIG_IPV6_MULTIPLE_TABLES=y
+CONFIG_IPV6_SUBTREES=y
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y
+CONFIG_IPV6_PIMSM_V2=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_SEG6_BPF=y
+# CONFIG_IPV6_RPL_LWTUNNEL is not set
+CONFIG_NETLABEL=y
+# CONFIG_MPTCP is not set
+CONFIG_NETWORK_SECMARK=y
+CONFIG_NET_PTP_CLASSIFY=y
+CONFIG_NETWORK_PHY_TIMESTAMPING=y
+CONFIG_NETFILTER=y
+CONFIG_NETFILTER_ADVANCED=y
+CONFIG_BRIDGE_NETFILTER=y
+
+#
+# Core Netfilter Configuration
+#
+CONFIG_NETFILTER_INGRESS=y
+CONFIG_NETFILTER_FAMILY_BRIDGE=y
+# CONFIG_NETFILTER_NETLINK_ACCT is not set
+# CONFIG_NETFILTER_NETLINK_QUEUE is not set
+# CONFIG_NETFILTER_NETLINK_LOG is not set
+# CONFIG_NETFILTER_NETLINK_OSF is not set
+CONFIG_NF_CONNTRACK=y
+CONFIG_NF_LOG_COMMON=y
+# CONFIG_NF_LOG_NETDEV is not set
+CONFIG_NF_CONNTRACK_MARK=y
+CONFIG_NF_CONNTRACK_SECMARK=y
+# CONFIG_NF_CONNTRACK_ZONES is not set
+CONFIG_NF_CONNTRACK_PROCFS=y
+CONFIG_NF_CONNTRACK_EVENTS=y
+CONFIG_NF_CONNTRACK_TIMEOUT=y
+CONFIG_NF_CONNTRACK_TIMESTAMP=y
+# CONFIG_NF_CONNTRACK_LABELS is not set
+CONFIG_NF_CT_PROTO_DCCP=y
+CONFIG_NF_CT_PROTO_SCTP=y
+CONFIG_NF_CT_PROTO_UDPLITE=y
+# CONFIG_NF_CONNTRACK_AMANDA is not set
+# CONFIG_NF_CONNTRACK_FTP is not set
+# CONFIG_NF_CONNTRACK_H323 is not set
+# CONFIG_NF_CONNTRACK_IRC is not set
+# CONFIG_NF_CONNTRACK_NETBIOS_NS is not set
+# CONFIG_NF_CONNTRACK_SNMP is not set
+# CONFIG_NF_CONNTRACK_PPTP is not set
+# CONFIG_NF_CONNTRACK_SANE is not set
+# CONFIG_NF_CONNTRACK_SIP is not set
+# CONFIG_NF_CONNTRACK_TFTP is not set
+# CONFIG_NF_CT_NETLINK is not set
+# CONFIG_NF_CT_NETLINK_TIMEOUT is not set
+CONFIG_NF_NAT=y
+CONFIG_NF_NAT_REDIRECT=y
+CONFIG_NF_NAT_MASQUERADE=y
+CONFIG_NETFILTER_SYNPROXY=y
+# CONFIG_NF_TABLES is not set
+CONFIG_NETFILTER_XTABLES=y
+
+#
+# Xtables combined modules
+#
+# CONFIG_NETFILTER_XT_MARK is not set
+# CONFIG_NETFILTER_XT_CONNMARK is not set
+
+#
+# Xtables targets
+#
+# CONFIG_NETFILTER_XT_TARGET_AUDIT is not set
+# CONFIG_NETFILTER_XT_TARGET_CHECKSUM is not set
+# CONFIG_NETFILTER_XT_TARGET_CLASSIFY is not set
+# CONFIG_NETFILTER_XT_TARGET_CONNMARK is not set
+# CONFIG_NETFILTER_XT_TARGET_CONNSECMARK is not set
+# CONFIG_NETFILTER_XT_TARGET_DSCP is not set
+# CONFIG_NETFILTER_XT_TARGET_HL is not set
+# CONFIG_NETFILTER_XT_TARGET_HMARK is not set
+# CONFIG_NETFILTER_XT_TARGET_IDLETIMER is not set
+# CONFIG_NETFILTER_XT_TARGET_LOG is not set
+# CONFIG_NETFILTER_XT_TARGET_MARK is not set
+CONFIG_NETFILTER_XT_NAT=y
+CONFIG_NETFILTER_XT_TARGET_NETMAP=y
+# CONFIG_NETFILTER_XT_TARGET_NFLOG is not set
+# CONFIG_NETFILTER_XT_TARGET_NFQUEUE is not set
+# CONFIG_NETFILTER_XT_TARGET_RATEEST is not set
+CONFIG_NETFILTER_XT_TARGET_REDIRECT=y
+CONFIG_NETFILTER_XT_TARGET_MASQUERADE=y
+# CONFIG_NETFILTER_XT_TARGET_TEE is not set
+# CONFIG_NETFILTER_XT_TARGET_TPROXY is not set
+# CONFIG_NETFILTER_XT_TARGET_SECMARK is not set
+# CONFIG_NETFILTER_XT_TARGET_TCPMSS is not set
+# CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP is not set
+
+#
+# Xtables matches
+#
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y
+# CONFIG_NETFILTER_XT_MATCH_BPF is not set
+# CONFIG_NETFILTER_XT_MATCH_CGROUP is not set
+# CONFIG_NETFILTER_XT_MATCH_CLUSTER is not set
+# CONFIG_NETFILTER_XT_MATCH_COMMENT is not set
+# CONFIG_NETFILTER_XT_MATCH_CONNBYTES is not set
+# CONFIG_NETFILTER_XT_MATCH_CONNLABEL is not set
+# CONFIG_NETFILTER_XT_MATCH_CONNLIMIT is not set
+# CONFIG_NETFILTER_XT_MATCH_CONNMARK is not set
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
+# CONFIG_NETFILTER_XT_MATCH_CPU is not set
+# CONFIG_NETFILTER_XT_MATCH_DCCP is not set
+# CONFIG_NETFILTER_XT_MATCH_DEVGROUP is not set
+# CONFIG_NETFILTER_XT_MATCH_DSCP is not set
+# CONFIG_NETFILTER_XT_MATCH_ECN is not set
+# CONFIG_NETFILTER_XT_MATCH_ESP is not set
+# CONFIG_NETFILTER_XT_MATCH_HASHLIMIT is not set
+# CONFIG_NETFILTER_XT_MATCH_HELPER is not set
+# CONFIG_NETFILTER_XT_MATCH_HL is not set
+# CONFIG_NETFILTER_XT_MATCH_IPCOMP is not set
+# CONFIG_NETFILTER_XT_MATCH_IPRANGE is not set
+# CONFIG_NETFILTER_XT_MATCH_L2TP is not set
+# CONFIG_NETFILTER_XT_MATCH_LENGTH is not set
+# CONFIG_NETFILTER_XT_MATCH_LIMIT is not set
+# CONFIG_NETFILTER_XT_MATCH_MAC is not set
+# CONFIG_NETFILTER_XT_MATCH_MARK is not set
+# CONFIG_NETFILTER_XT_MATCH_MULTIPORT is not set
+# CONFIG_NETFILTER_XT_MATCH_NFACCT is not set
+# CONFIG_NETFILTER_XT_MATCH_OSF is not set
+# CONFIG_NETFILTER_XT_MATCH_OWNER is not set
+# CONFIG_NETFILTER_XT_MATCH_POLICY is not set
+# CONFIG_NETFILTER_XT_MATCH_PHYSDEV is not set
+# CONFIG_NETFILTER_XT_MATCH_PKTTYPE is not set
+# CONFIG_NETFILTER_XT_MATCH_QUOTA is not set
+# CONFIG_NETFILTER_XT_MATCH_RATEEST is not set
+# CONFIG_NETFILTER_XT_MATCH_REALM is not set
+# CONFIG_NETFILTER_XT_MATCH_RECENT is not set
+# CONFIG_NETFILTER_XT_MATCH_SCTP is not set
+# CONFIG_NETFILTER_XT_MATCH_SOCKET is not set
+# CONFIG_NETFILTER_XT_MATCH_STATE is not set
+# CONFIG_NETFILTER_XT_MATCH_STATISTIC is not set
+# CONFIG_NETFILTER_XT_MATCH_STRING is not set
+# CONFIG_NETFILTER_XT_MATCH_TCPMSS is not set
+# CONFIG_NETFILTER_XT_MATCH_TIME is not set
+# CONFIG_NETFILTER_XT_MATCH_U32 is not set
+# end of Core Netfilter Configuration
+
+# CONFIG_IP_SET is not set
+# CONFIG_IP_VS is not set
+
+#
+# IP: Netfilter Configuration
+#
+CONFIG_NF_DEFRAG_IPV4=y
+# CONFIG_NF_SOCKET_IPV4 is not set
+# CONFIG_NF_TPROXY_IPV4 is not set
+# CONFIG_NF_DUP_IPV4 is not set
+CONFIG_NF_LOG_ARP=y
+CONFIG_NF_LOG_IPV4=y
+CONFIG_NF_REJECT_IPV4=y
+CONFIG_IP_NF_IPTABLES=y
+# CONFIG_IP_NF_MATCH_AH is not set
+# CONFIG_IP_NF_MATCH_ECN is not set
+# CONFIG_IP_NF_MATCH_RPFILTER is not set
+# CONFIG_IP_NF_MATCH_TTL is not set
+CONFIG_IP_NF_FILTER=y
+CONFIG_IP_NF_TARGET_REJECT=y
+CONFIG_IP_NF_TARGET_SYNPROXY=y
+CONFIG_IP_NF_NAT=y
+CONFIG_IP_NF_TARGET_MASQUERADE=y
+CONFIG_IP_NF_TARGET_NETMAP=y
+CONFIG_IP_NF_TARGET_REDIRECT=y
+CONFIG_IP_NF_MANGLE=y
+# CONFIG_IP_NF_TARGET_CLUSTERIP is not set
+# CONFIG_IP_NF_TARGET_ECN is not set
+# CONFIG_IP_NF_TARGET_TTL is not set
+# CONFIG_IP_NF_RAW is not set
+# CONFIG_IP_NF_SECURITY is not set
+# CONFIG_IP_NF_ARPTABLES is not set
+# end of IP: Netfilter Configuration
+
+#
+# IPv6: Netfilter Configuration
+#
+# CONFIG_NF_SOCKET_IPV6 is not set
+# CONFIG_NF_TPROXY_IPV6 is not set
+# CONFIG_NF_DUP_IPV6 is not set
+# CONFIG_NF_REJECT_IPV6 is not set
+# CONFIG_NF_LOG_IPV6 is not set
+# CONFIG_IP6_NF_IPTABLES is not set
+# end of IPv6: Netfilter Configuration
+
+CONFIG_NF_DEFRAG_IPV6=y
+# CONFIG_NF_CONNTRACK_BRIDGE is not set
+# CONFIG_BRIDGE_NF_EBTABLES is not set
+# CONFIG_BPFILTER is not set
+# CONFIG_IP_DCCP is not set
+# CONFIG_IP_SCTP is not set
+# CONFIG_RDS is not set
+# CONFIG_TIPC is not set
+# CONFIG_ATM is not set
+# CONFIG_L2TP is not set
+CONFIG_STP=y
+CONFIG_BRIDGE=y
+CONFIG_BRIDGE_IGMP_SNOOPING=y
+# CONFIG_BRIDGE_MRP is not set
+CONFIG_HAVE_NET_DSA=y
+# CONFIG_NET_DSA is not set
+# CONFIG_VLAN_8021Q is not set
+# CONFIG_DECNET is not set
+CONFIG_LLC=y
+# CONFIG_LLC2 is not set
+# CONFIG_ATALK is not set
+# CONFIG_X25 is not set
+# CONFIG_LAPB is not set
+# CONFIG_PHONET is not set
+# CONFIG_6LOWPAN is not set
+# CONFIG_IEEE802154 is not set
+CONFIG_NET_SCHED=y
+
+#
+# Queueing/Scheduling
+#
+# CONFIG_NET_SCH_CBQ is not set
+# CONFIG_NET_SCH_HTB is not set
+# CONFIG_NET_SCH_HFSC is not set
+# CONFIG_NET_SCH_PRIO is not set
+# CONFIG_NET_SCH_MULTIQ is not set
+# CONFIG_NET_SCH_RED is not set
+# CONFIG_NET_SCH_SFB is not set
+# CONFIG_NET_SCH_SFQ is not set
+# CONFIG_NET_SCH_TEQL is not set
+# CONFIG_NET_SCH_TBF is not set
+# CONFIG_NET_SCH_CBS is not set
+# CONFIG_NET_SCH_ETF is not set
+# CONFIG_NET_SCH_TAPRIO is not set
+# CONFIG_NET_SCH_GRED is not set
+# CONFIG_NET_SCH_DSMARK is not set
+# CONFIG_NET_SCH_NETEM is not set
+# CONFIG_NET_SCH_DRR is not set
+# CONFIG_NET_SCH_MQPRIO is not set
+# CONFIG_NET_SCH_SKBPRIO is not set
+# CONFIG_NET_SCH_CHOKE is not set
+# CONFIG_NET_SCH_QFQ is not set
+# CONFIG_NET_SCH_CODEL is not set
+# CONFIG_NET_SCH_FQ_CODEL is not set
+# CONFIG_NET_SCH_CAKE is not set
+# CONFIG_NET_SCH_FQ is not set
+# CONFIG_NET_SCH_HHF is not set
+# CONFIG_NET_SCH_PIE is not set
+# CONFIG_NET_SCH_INGRESS is not set
+# CONFIG_NET_SCH_PLUG is not set
+# CONFIG_NET_SCH_ETS is not set
+# CONFIG_NET_SCH_DEFAULT is not set
+
+#
+# Classification
+#
+CONFIG_NET_CLS=y
+# CONFIG_NET_CLS_BASIC is not set
+# CONFIG_NET_CLS_TCINDEX is not set
+# CONFIG_NET_CLS_ROUTE4 is not set
+# CONFIG_NET_CLS_FW is not set
+# CONFIG_NET_CLS_U32 is not set
+# CONFIG_NET_CLS_RSVP is not set
+# CONFIG_NET_CLS_RSVP6 is not set
+# CONFIG_NET_CLS_FLOW is not set
+# CONFIG_NET_CLS_CGROUP is not set
+# CONFIG_NET_CLS_BPF is not set
+# CONFIG_NET_CLS_FLOWER is not set
+# CONFIG_NET_CLS_MATCHALL is not set
+CONFIG_NET_EMATCH=y
+CONFIG_NET_EMATCH_STACK=32
+# CONFIG_NET_EMATCH_CMP is not set
+# CONFIG_NET_EMATCH_NBYTE is not set
+# CONFIG_NET_EMATCH_U32 is not set
+# CONFIG_NET_EMATCH_META is not set
+# CONFIG_NET_EMATCH_TEXT is not set
+# CONFIG_NET_EMATCH_IPT is not set
+CONFIG_NET_CLS_ACT=y
+# CONFIG_NET_ACT_POLICE is not set
+# CONFIG_NET_ACT_GACT is not set
+# CONFIG_NET_ACT_MIRRED is not set
+# CONFIG_NET_ACT_SAMPLE is not set
+# CONFIG_NET_ACT_IPT is not set
+# CONFIG_NET_ACT_NAT is not set
+# CONFIG_NET_ACT_PEDIT is not set
+# CONFIG_NET_ACT_SIMP is not set
+# CONFIG_NET_ACT_SKBEDIT is not set
+# CONFIG_NET_ACT_CSUM is not set
+# CONFIG_NET_ACT_MPLS is not set
+# CONFIG_NET_ACT_VLAN is not set
+# CONFIG_NET_ACT_BPF is not set
+# CONFIG_NET_ACT_CONNMARK is not set
+# CONFIG_NET_ACT_CTINFO is not set
+# CONFIG_NET_ACT_SKBMOD is not set
+# CONFIG_NET_ACT_IFE is not set
+# CONFIG_NET_ACT_TUNNEL_KEY is not set
+# CONFIG_NET_ACT_GATE is not set
+# CONFIG_NET_TC_SKB_EXT is not set
+CONFIG_NET_SCH_FIFO=y
+CONFIG_DCB=y
+# CONFIG_DNS_RESOLVER is not set
+# CONFIG_BATMAN_ADV is not set
+# CONFIG_OPENVSWITCH is not set
+CONFIG_VSOCKETS=y
+CONFIG_VSOCKETS_DIAG=y
+CONFIG_VSOCKETS_LOOPBACK=y
+CONFIG_VIRTIO_VSOCKETS=y
+CONFIG_VIRTIO_VSOCKETS_COMMON=y
+# CONFIG_NETLINK_DIAG is not set
+CONFIG_MPLS=y
+# CONFIG_NET_MPLS_GSO is not set
+# CONFIG_MPLS_ROUTING is not set
+# CONFIG_NET_NSH is not set
+# CONFIG_HSR is not set
+# CONFIG_NET_SWITCHDEV is not set
+# CONFIG_NET_L3_MASTER_DEV is not set
+# CONFIG_QRTR is not set
+# CONFIG_NET_NCSI is not set
+CONFIG_RPS=y
+CONFIG_RFS_ACCEL=y
+CONFIG_XPS=y
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_CGROUP_NET_CLASSID=y
+CONFIG_NET_RX_BUSY_POLL=y
+CONFIG_BQL=y
+CONFIG_BPF_JIT=y
+CONFIG_BPF_STREAM_PARSER=y
+CONFIG_NET_FLOW_LIMIT=y
+
+#
+# Network testing
+#
+# CONFIG_NET_PKTGEN is not set
+# end of Network testing
+# end of Networking options
+
+# CONFIG_HAMRADIO is not set
+# CONFIG_CAN is not set
+# CONFIG_BT is not set
+# CONFIG_AF_RXRPC is not set
+# CONFIG_AF_KCM is not set
+CONFIG_STREAM_PARSER=y
+CONFIG_FIB_RULES=y
+# CONFIG_WIRELESS is not set
+# CONFIG_WIMAX is not set
+# CONFIG_RFKILL is not set
+# CONFIG_NET_9P is not set
+# CONFIG_CAIF is not set
+# CONFIG_CEPH_LIB is not set
+# CONFIG_NFC is not set
+# CONFIG_PSAMPLE is not set
+# CONFIG_NET_IFE is not set
+CONFIG_LWTUNNEL=y
+CONFIG_LWTUNNEL_BPF=y
+CONFIG_DST_CACHE=y
+CONFIG_GRO_CELLS=y
+CONFIG_NET_SOCK_MSG=y
+CONFIG_FAILOVER=y
+CONFIG_ETHTOOL_NETLINK=y
+CONFIG_HAVE_EBPF_JIT=y
+
+#
+# Device Drivers
+#
+CONFIG_HAVE_EISA=y
+# CONFIG_EISA is not set
+CONFIG_HAVE_PCI=y
+# CONFIG_PCI is not set
+# CONFIG_PCCARD is not set
+
+#
+# Generic Driver Options
+#
+CONFIG_UEVENT_HELPER=y
+CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_STANDALONE=y
+CONFIG_PREVENT_FIRMWARE_BUILD=y
+
+#
+# Firmware loader
+#
+CONFIG_FW_LOADER=y
+CONFIG_EXTRA_FIRMWARE=""
+# CONFIG_FW_LOADER_USER_HELPER is not set
+# CONFIG_FW_LOADER_COMPRESS is not set
+CONFIG_FW_CACHE=y
+# end of Firmware loader
+
+CONFIG_ALLOW_DEV_COREDUMP=y
+# CONFIG_DEBUG_DRIVER is not set
+# CONFIG_DEBUG_DEVRES is not set
+# CONFIG_DEBUG_TEST_DRIVER_REMOVE is not set
+# CONFIG_TEST_ASYNC_DRIVER_PROBE is not set
+CONFIG_GENERIC_CPU_AUTOPROBE=y
+CONFIG_GENERIC_CPU_VULNERABILITIES=y
+CONFIG_DMA_SHARED_BUFFER=y
+# CONFIG_DMA_FENCE_TRACE is not set
+# end of Generic Driver Options
+
+#
+# Bus devices
+#
+# CONFIG_MHI_BUS is not set
+# end of Bus devices
+
+CONFIG_CONNECTOR=y
+CONFIG_PROC_EVENTS=y
+# CONFIG_GNSS is not set
+# CONFIG_MTD is not set
+# CONFIG_OF is not set
+CONFIG_ARCH_MIGHT_HAVE_PC_PARPORT=y
+# CONFIG_PARPORT is not set
+CONFIG_PNP=y
+CONFIG_PNP_DEBUG_MESSAGES=y
+
+#
+# Protocols
+#
+CONFIG_PNPACPI=y
+CONFIG_BLK_DEV=y
+# CONFIG_BLK_DEV_NULL_BLK is not set
+# CONFIG_BLK_DEV_FD is not set
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_LOOP_MIN_COUNT=8
+# CONFIG_BLK_DEV_CRYPTOLOOP is not set
+# CONFIG_BLK_DEV_DRBD is not set
+# CONFIG_BLK_DEV_NBD is not set
+# CONFIG_BLK_DEV_RAM is not set
+# CONFIG_CDROM_PKTCDVD is not set
+# CONFIG_ATA_OVER_ETH is not set
+CONFIG_VIRTIO_BLK=y
+# CONFIG_BLK_DEV_RBD is not set
+
+#
+# NVME Support
+#
+# CONFIG_NVME_FC is not set
+# CONFIG_NVME_TCP is not set
+# end of NVME Support
+
+#
+# Misc devices
+#
+# CONFIG_DUMMY_IRQ is not set
+# CONFIG_ENCLOSURE_SERVICES is not set
+# CONFIG_SRAM is not set
+# CONFIG_XILINX_SDFEC is not set
+# CONFIG_PVPANIC is not set
+# CONFIG_C2PORT is not set
+
+#
+# EEPROM support
+#
+# CONFIG_EEPROM_93CX6 is not set
+# end of EEPROM support
+
+#
+# Texas Instruments shared transport line discipline
+#
+# end of Texas Instruments shared transport line discipline
+
+#
+# Altera FPGA firmware download module (requires I2C)
+#
+# CONFIG_ECHO is not set
+# end of Misc devices
+
+CONFIG_HAVE_IDE=y
+# CONFIG_IDE is not set
+
+#
+# SCSI device support
+#
+CONFIG_SCSI_MOD=y
+# CONFIG_RAID_ATTRS is not set
+CONFIG_SCSI=y
+CONFIG_SCSI_DMA=y
+CONFIG_SCSI_PROC_FS=y
+
+#
+# SCSI support type (disk, tape, CD-ROM)
+#
+# CONFIG_BLK_DEV_SD is not set
+# CONFIG_CHR_DEV_ST is not set
+# CONFIG_BLK_DEV_SR is not set
+# CONFIG_CHR_DEV_SG is not set
+# CONFIG_CHR_DEV_SCH is not set
+CONFIG_SCSI_CONSTANTS=y
+CONFIG_SCSI_LOGGING=y
+CONFIG_SCSI_SCAN_ASYNC=y
+
+#
+# SCSI Transports
+#
+# CONFIG_SCSI_SPI_ATTRS is not set
+# CONFIG_SCSI_FC_ATTRS is not set
+CONFIG_SCSI_ISCSI_ATTRS=y
+# CONFIG_SCSI_SAS_ATTRS is not set
+# CONFIG_SCSI_SAS_LIBSAS is not set
+# CONFIG_SCSI_SRP_ATTRS is not set
+# end of SCSI Transports
+
+CONFIG_SCSI_LOWLEVEL=y
+CONFIG_ISCSI_TCP=y
+# CONFIG_ISCSI_BOOT_SYSFS is not set
+# CONFIG_SCSI_UFSHCD is not set
+# CONFIG_SCSI_DEBUG is not set
+# CONFIG_SCSI_VIRTIO is not set
+# CONFIG_SCSI_DH is not set
+# end of SCSI device support
+
+# CONFIG_ATA is not set
+# CONFIG_MD is not set
+# CONFIG_TARGET_CORE is not set
+# CONFIG_MACINTOSH_DRIVERS is not set
+CONFIG_NETDEVICES=y
+CONFIG_NET_CORE=y
+# CONFIG_BONDING is not set
+# CONFIG_DUMMY is not set
+# CONFIG_WIREGUARD is not set
+# CONFIG_EQUALIZER is not set
+# CONFIG_NET_TEAM is not set
+# CONFIG_MACVLAN is not set
+# CONFIG_IPVLAN is not set
+# CONFIG_VXLAN is not set
+# CONFIG_GENEVE is not set
+# CONFIG_BAREUDP is not set
+# CONFIG_GTP is not set
+# CONFIG_MACSEC is not set
+# CONFIG_NETCONSOLE is not set
+CONFIG_TUN=y
+# CONFIG_TUN_VNET_CROSS_LE is not set
+CONFIG_VETH=y
+CONFIG_VIRTIO_NET=y
+# CONFIG_NLMON is not set
+
+#
+# Distributed Switch Architecture drivers
+#
+# end of Distributed Switch Architecture drivers
+
+# CONFIG_ETHERNET is not set
+# CONFIG_NET_SB1000 is not set
+# CONFIG_PHYLIB is not set
+# CONFIG_MDIO_DEVICE is not set
+
+#
+# PCS device drivers
+#
+# end of PCS device drivers
+
+# CONFIG_PPP is not set
+# CONFIG_SLIP is not set
+
+#
+# Host-side USB support is needed for USB Network Adapter support
+#
+# CONFIG_WLAN is not set
+
+#
+# Enable WiMAX (Networking options) to see the WiMAX drivers
+#
+# CONFIG_WAN is not set
+# CONFIG_FUJITSU_ES is not set
+# CONFIG_NETDEVSIM is not set
+CONFIG_NET_FAILOVER=y
+# CONFIG_ISDN is not set
+
+#
+# Input device support
+#
+CONFIG_INPUT=y
+# CONFIG_INPUT_FF_MEMLESS is not set
+# CONFIG_INPUT_POLLDEV is not set
+# CONFIG_INPUT_SPARSEKMAP is not set
+# CONFIG_INPUT_MATRIXKMAP is not set
+
+#
+# Userland interfaces
+#
+# CONFIG_INPUT_MOUSEDEV is not set
+# CONFIG_INPUT_JOYDEV is not set
+# CONFIG_INPUT_EVDEV is not set
+# CONFIG_INPUT_EVBUG is not set
+
+#
+# Input Device Drivers
+#
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+# CONFIG_INPUT_JOYSTICK is not set
+# CONFIG_INPUT_TABLET is not set
+# CONFIG_INPUT_TOUCHSCREEN is not set
+# CONFIG_INPUT_MISC is not set
+# CONFIG_RMI4_CORE is not set
+
+#
+# Hardware I/O ports
+#
+# CONFIG_SERIO is not set
+CONFIG_ARCH_MIGHT_HAVE_PC_SERIO=y
+# CONFIG_GAMEPORT is not set
+# end of Hardware I/O ports
+# end of Input device support
+
+#
+# Character devices
+#
+CONFIG_TTY=y
+CONFIG_VT=y
+CONFIG_CONSOLE_TRANSLATIONS=y
+CONFIG_VT_CONSOLE=y
+CONFIG_VT_CONSOLE_SLEEP=y
+CONFIG_HW_CONSOLE=y
+CONFIG_VT_HW_CONSOLE_BINDING=y
+CONFIG_UNIX98_PTYS=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_LDISC_AUTOLOAD=y
+
+#
+# Serial drivers
+#
+CONFIG_SERIAL_EARLYCON=y
+CONFIG_SERIAL_8250=y
+# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set
+CONFIG_SERIAL_8250_PNP=y
+# CONFIG_SERIAL_8250_16550A_VARIANTS is not set
+# CONFIG_SERIAL_8250_FINTEK is not set
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DMA=y
+CONFIG_SERIAL_8250_NR_UARTS=1
+CONFIG_SERIAL_8250_RUNTIME_UARTS=1
+# CONFIG_SERIAL_8250_EXTENDED is not set
+# CONFIG_SERIAL_8250_DW is not set
+# CONFIG_SERIAL_8250_RT288X is not set
+
+#
+# Non-8250 serial port support
+#
+# CONFIG_SERIAL_UARTLITE is not set
+CONFIG_SERIAL_CORE=y
+CONFIG_SERIAL_CORE_CONSOLE=y
+# CONFIG_SERIAL_LANTIQ is not set
+# CONFIG_SERIAL_SCCNXP is not set
+# CONFIG_SERIAL_ALTERA_JTAGUART is not set
+# CONFIG_SERIAL_ALTERA_UART is not set
+# CONFIG_SERIAL_ARC is not set
+# CONFIG_SERIAL_FSL_LPUART is not set
+# CONFIG_SERIAL_FSL_LINFLEXUART is not set
+# end of Serial drivers
+
+# CONFIG_SERIAL_NONSTANDARD is not set
+# CONFIG_N_GSM is not set
+# CONFIG_NULL_TTY is not set
+# CONFIG_TRACE_SINK is not set
+CONFIG_HVC_DRIVER=y
+CONFIG_SERIAL_DEV_BUS=y
+CONFIG_SERIAL_DEV_CTRL_TTYPORT=y
+CONFIG_VIRTIO_CONSOLE=y
+# CONFIG_IPMI_HANDLER is not set
+# CONFIG_HW_RANDOM is not set
+# CONFIG_MWAVE is not set
+CONFIG_DEVMEM=y
+# CONFIG_DEVKMEM is not set
+# CONFIG_NVRAM is not set
+# CONFIG_RAW_DRIVER is not set
+# CONFIG_HPET is not set
+# CONFIG_HANGCHECK_TIMER is not set
+# CONFIG_TCG_TPM is not set
+# CONFIG_TELCLOCK is not set
+# CONFIG_RANDOM_TRUST_CPU is not set
+# CONFIG_RANDOM_TRUST_BOOTLOADER is not set
+# end of Character devices
+
+#
+# I2C support
+#
+# CONFIG_I2C is not set
+# end of I2C support
+
+# CONFIG_I3C is not set
+# CONFIG_SPI is not set
+# CONFIG_SPMI is not set
+# CONFIG_HSI is not set
+CONFIG_PPS=y
+# CONFIG_PPS_DEBUG is not set
+
+#
+# PPS clients support
+#
+# CONFIG_PPS_CLIENT_KTIMER is not set
+# CONFIG_PPS_CLIENT_LDISC is not set
+# CONFIG_PPS_CLIENT_GPIO is not set
+
+#
+# PPS generators support
+#
+
+#
+# PTP clock support
+#
+CONFIG_PTP_1588_CLOCK=y
+
+#
+# Enable PHYLIB and NETWORK_PHY_TIMESTAMPING to see the additional clocks.
+#
+CONFIG_PTP_1588_CLOCK_KVM=y
+# CONFIG_PTP_1588_CLOCK_VMW is not set
+# end of PTP clock support
+
+# CONFIG_PINCTRL is not set
+# CONFIG_GPIOLIB is not set
+# CONFIG_W1 is not set
+CONFIG_POWER_RESET=y
+# CONFIG_POWER_RESET_RESTART is not set
+CONFIG_POWER_SUPPLY=y
+# CONFIG_POWER_SUPPLY_DEBUG is not set
+# CONFIG_PDA_POWER is not set
+# CONFIG_TEST_POWER is not set
+# CONFIG_BATTERY_DS2780 is not set
+# CONFIG_BATTERY_DS2781 is not set
+# CONFIG_BATTERY_BQ27XXX is not set
+# CONFIG_CHARGER_MAX8903 is not set
+# CONFIG_HWMON is not set
+CONFIG_THERMAL=y
+# CONFIG_THERMAL_NETLINK is not set
+# CONFIG_THERMAL_STATISTICS is not set
+CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=0
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE=y
+# CONFIG_THERMAL_DEFAULT_GOV_FAIR_SHARE is not set
+# CONFIG_THERMAL_DEFAULT_GOV_USER_SPACE is not set
+CONFIG_THERMAL_GOV_FAIR_SHARE=y
+CONFIG_THERMAL_GOV_STEP_WISE=y
+# CONFIG_THERMAL_GOV_BANG_BANG is not set
+CONFIG_THERMAL_GOV_USER_SPACE=y
+# CONFIG_THERMAL_EMULATION is not set
+
+#
+# Intel thermal drivers
+#
+# CONFIG_INTEL_POWERCLAMP is not set
+
+#
+# ACPI INT340X thermal drivers
+#
+# end of ACPI INT340X thermal drivers
+# end of Intel thermal drivers
+
+# CONFIG_WATCHDOG is not set
+CONFIG_SSB_POSSIBLE=y
+# CONFIG_SSB is not set
+CONFIG_BCMA_POSSIBLE=y
+# CONFIG_BCMA is not set
+
+#
+# Multifunction device drivers
+#
+# CONFIG_MFD_MADERA is not set
+# CONFIG_HTC_PASIC3 is not set
+# CONFIG_MFD_INTEL_LPSS_ACPI is not set
+# CONFIG_MFD_INTEL_PMC_BXT is not set
+# CONFIG_MFD_KEMPLD is not set
+# CONFIG_MFD_MT6397 is not set
+# CONFIG_MFD_SM501 is not set
+# CONFIG_ABX500_CORE is not set
+# CONFIG_MFD_SYSCON is not set
+# CONFIG_MFD_TI_AM335X_TSCADC is not set
+# CONFIG_MFD_TQMX86 is not set
+# CONFIG_RAVE_SP_CORE is not set
+# end of Multifunction device drivers
+
+# CONFIG_REGULATOR is not set
+# CONFIG_RC_CORE is not set
+# CONFIG_MEDIA_CEC_SUPPORT is not set
+# CONFIG_MEDIA_SUPPORT is not set
+
+#
+# Graphics support
+#
+# CONFIG_DRM is not set
+
+#
+# ARM devices
+#
+# end of ARM devices
+
+#
+# Frame buffer Devices
+#
+# CONFIG_FB is not set
+# end of Frame buffer Devices
+
+#
+# Backlight & LCD device support
+#
+# CONFIG_LCD_CLASS_DEVICE is not set
+# CONFIG_BACKLIGHT_CLASS_DEVICE is not set
+# end of Backlight & LCD device support
+
+#
+# Console display driver support
+#
+CONFIG_VGA_CONSOLE=y
+CONFIG_DUMMY_CONSOLE=y
+CONFIG_DUMMY_CONSOLE_COLUMNS=80
+CONFIG_DUMMY_CONSOLE_ROWS=25
+# end of Console display driver support
+# end of Graphics support
+
+# CONFIG_SOUND is not set
+
+#
+# HID support
+#
+CONFIG_HID=y
+# CONFIG_HID_BATTERY_STRENGTH is not set
+CONFIG_HIDRAW=y
+# CONFIG_UHID is not set
+# CONFIG_HID_GENERIC is not set
+
+#
+# Special HID drivers
+#
+# CONFIG_HID_A4TECH is not set
+# CONFIG_HID_ACRUX is not set
+# CONFIG_HID_APPLE is not set
+# CONFIG_HID_AUREAL is not set
+# CONFIG_HID_BELKIN is not set
+# CONFIG_HID_CHERRY is not set
+# CONFIG_HID_COUGAR is not set
+# CONFIG_HID_MACALLY is not set
+# CONFIG_HID_CMEDIA is not set
+# CONFIG_HID_CYPRESS is not set
+# CONFIG_HID_DRAGONRISE is not set
+# CONFIG_HID_EMS_FF is not set
+# CONFIG_HID_ELECOM is not set
+# CONFIG_HID_EZKEY is not set
+# CONFIG_HID_GEMBIRD is not set
+# CONFIG_HID_GFRM is not set
+# CONFIG_HID_GLORIOUS is not set
+# CONFIG_HID_VIVALDI is not set
+# CONFIG_HID_KEYTOUCH is not set
+# CONFIG_HID_KYE is not set
+# CONFIG_HID_WALTOP is not set
+# CONFIG_HID_VIEWSONIC is not set
+# CONFIG_HID_GYRATION is not set
+# CONFIG_HID_ICADE is not set
+# CONFIG_HID_ITE is not set
+# CONFIG_HID_JABRA is not set
+# CONFIG_HID_TWINHAN is not set
+# CONFIG_HID_KENSINGTON is not set
+# CONFIG_HID_LCPOWER is not set
+# CONFIG_HID_LENOVO is not set
+# CONFIG_HID_MAGICMOUSE is not set
+# CONFIG_HID_MALTRON is not set
+# CONFIG_HID_MAYFLASH is not set
+CONFIG_HID_REDRAGON=y
+# CONFIG_HID_MICROSOFT is not set
+# CONFIG_HID_MONTEREY is not set
+# CONFIG_HID_MULTITOUCH is not set
+# CONFIG_HID_NTI is not set
+# CONFIG_HID_ORTEK is not set
+# CONFIG_HID_PANTHERLORD is not set
+# CONFIG_HID_PETALYNX is not set
+# CONFIG_HID_PICOLCD is not set
+# CONFIG_HID_PLANTRONICS is not set
+# CONFIG_HID_PRIMAX is not set
+# CONFIG_HID_SAITEK is not set
+# CONFIG_HID_SPEEDLINK is not set
+# CONFIG_HID_STEAM is not set
+# CONFIG_HID_STEELSERIES is not set
+# CONFIG_HID_SUNPLUS is not set
+# CONFIG_HID_RMI is not set
+# CONFIG_HID_GREENASIA is not set
+# CONFIG_HID_SMARTJOYPLUS is not set
+# CONFIG_HID_TIVO is not set
+# CONFIG_HID_TOPSEED is not set
+# CONFIG_HID_THRUSTMASTER is not set
+# CONFIG_HID_UDRAW_PS3 is not set
+# CONFIG_HID_XINMO is not set
+# CONFIG_HID_ZEROPLUS is not set
+# CONFIG_HID_ZYDACRON is not set
+# CONFIG_HID_SENSOR_HUB is not set
+# CONFIG_HID_ALPS is not set
+# end of Special HID drivers
+# end of HID support
+
+CONFIG_USB_OHCI_LITTLE_ENDIAN=y
+# CONFIG_USB_SUPPORT is not set
+# CONFIG_MMC is not set
+# CONFIG_MEMSTICK is not set
+# CONFIG_NEW_LEDS is not set
+# CONFIG_ACCESSIBILITY is not set
+# CONFIG_INFINIBAND is not set
+CONFIG_EDAC_ATOMIC_SCRUB=y
+CONFIG_EDAC_SUPPORT=y
+# CONFIG_EDAC is not set
+CONFIG_RTC_LIB=y
+CONFIG_RTC_MC146818_LIB=y
+# CONFIG_RTC_CLASS is not set
+CONFIG_DMADEVICES=y
+# CONFIG_DMADEVICES_DEBUG is not set
+
+#
+# DMA Devices
+#
+CONFIG_DMA_ACPI=y
+# CONFIG_ALTERA_MSGDMA is not set
+# CONFIG_INTEL_IDMA64 is not set
+# CONFIG_QCOM_HIDMA_MGMT is not set
+# CONFIG_QCOM_HIDMA is not set
+# CONFIG_DW_DMAC is not set
+# CONFIG_SF_PDMA is not set
+
+#
+# DMABUF options
+#
+CONFIG_SYNC_FILE=y
+# CONFIG_SW_SYNC is not set
+# CONFIG_UDMABUF is not set
+# CONFIG_DMABUF_MOVE_NOTIFY is not set
+# CONFIG_DMABUF_SELFTESTS is not set
+# CONFIG_DMABUF_HEAPS is not set
+# end of DMABUF options
+
+CONFIG_AUXDISPLAY=y
+# CONFIG_IMG_ASCII_LCD is not set
+# CONFIG_CHARLCD_BL_OFF is not set
+# CONFIG_CHARLCD_BL_ON is not set
+CONFIG_CHARLCD_BL_FLASH=y
+# CONFIG_UIO is not set
+# CONFIG_VFIO is not set
+CONFIG_VIRT_DRIVERS=y
+CONFIG_VIRTIO=y
+CONFIG_VIRTIO_MENU=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_MEM=m
+# CONFIG_VIRTIO_INPUT is not set
+CONFIG_VIRTIO_MMIO=y
+CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
+# CONFIG_VDPA is not set
+CONFIG_VHOST_MENU=y
+# CONFIG_VHOST_NET is not set
+# CONFIG_VHOST_VSOCK is not set
+# CONFIG_VHOST_CROSS_ENDIAN_LEGACY is not set
+
+#
+# Microsoft Hyper-V guest support
+#
+# CONFIG_HYPERV is not set
+# end of Microsoft Hyper-V guest support
+
+# CONFIG_GREYBUS is not set
+CONFIG_STAGING=y
+# CONFIG_COMEDI is not set
+# CONFIG_STAGING_MEDIA is not set
+
+#
+# Android
+#
+# end of Android
+
+# CONFIG_GS_FPGABOOT is not set
+# CONFIG_UNISYSSPAR is not set
+
+#
+# Gasket devices
+#
+# end of Gasket devices
+
+# CONFIG_FIELDBUS_DEV is not set
+CONFIG_X86_PLATFORM_DEVICES=y
+# CONFIG_ACPI_WMI is not set
+# CONFIG_ACERHDF is not set
+# CONFIG_ACER_WIRELESS is not set
+# CONFIG_ASUS_WIRELESS is not set
+# CONFIG_DCDBAS is not set
+# CONFIG_DELL_SMBIOS is not set
+# CONFIG_DELL_RBU is not set
+# CONFIG_DELL_SMO8800 is not set
+# CONFIG_FUJITSU_TABLET is not set
+# CONFIG_GPD_POCKET_FAN is not set
+# CONFIG_HP_WIRELESS is not set
+# CONFIG_SENSORS_HDAPS is not set
+# CONFIG_INTEL_HID_EVENT is not set
+# CONFIG_INTEL_MENLOW is not set
+# CONFIG_INTEL_VBTN is not set
+# CONFIG_SURFACE_PRO3_BUTTON is not set
+# CONFIG_SAMSUNG_Q10 is not set
+# CONFIG_TOSHIBA_BT_RFKILL is not set
+# CONFIG_TOSHIBA_HAPS is not set
+# CONFIG_ACPI_CMPC is not set
+# CONFIG_SYSTEM76_ACPI is not set
+# CONFIG_TOPSTAR_LAPTOP is not set
+# CONFIG_INTEL_RST is not set
+# CONFIG_INTEL_SMARTCONNECT is not set
+CONFIG_INTEL_TURBO_MAX_3=y
+# CONFIG_INTEL_UNCORE_FREQ_CONTROL is not set
+# CONFIG_INTEL_PUNIT_IPC is not set
+# CONFIG_INTEL_SCU_PLATFORM is not set
+# CONFIG_CHROME_PLATFORMS is not set
+# CONFIG_MELLANOX_PLATFORM is not set
+# CONFIG_COMMON_CLK is not set
+# CONFIG_HWSPINLOCK is not set
+
+#
+# Clock Source drivers
+#
+CONFIG_CLKEVT_I8253=y
+CONFIG_I8253_LOCK=y
+CONFIG_CLKBLD_I8253=y
+# end of Clock Source drivers
+
+CONFIG_MAILBOX=y
+CONFIG_PCC=y
+# CONFIG_ALTERA_MBOX is not set
+CONFIG_IOMMU_SUPPORT=y
+
+#
+# Generic IOMMU Pagetable Support
+#
+# end of Generic IOMMU Pagetable Support
+
+# CONFIG_IOMMU_DEBUGFS is not set
+
+#
+# Remoteproc drivers
+#
+# CONFIG_REMOTEPROC is not set
+# end of Remoteproc drivers
+
+#
+# Rpmsg drivers
+#
+# CONFIG_RPMSG_QCOM_GLINK_RPM is not set
+# CONFIG_RPMSG_VIRTIO is not set
+# end of Rpmsg drivers
+
+# CONFIG_SOUNDWIRE is not set
+
+#
+# SOC (System On Chip) specific Drivers
+#
+
+#
+# Amlogic SoC drivers
+#
+# end of Amlogic SoC drivers
+
+#
+# Aspeed SoC drivers
+#
+# end of Aspeed SoC drivers
+
+#
+# Broadcom SoC drivers
+#
+# end of Broadcom SoC drivers
+
+#
+# NXP/Freescale QorIQ SoC drivers
+#
+# end of NXP/Freescale QorIQ SoC drivers
+
+#
+# i.MX SoC drivers
+#
+# end of i.MX SoC drivers
+
+#
+# Qualcomm SoC drivers
+#
+# end of Qualcomm SoC drivers
+
+# CONFIG_SOC_TI is not set
+
+#
+# Xilinx SoC drivers
+#
+# CONFIG_XILINX_VCU is not set
+# end of Xilinx SoC drivers
+# end of SOC (System On Chip) specific Drivers
+
+# CONFIG_PM_DEVFREQ is not set
+# CONFIG_EXTCON is not set
+# CONFIG_MEMORY is not set
+# CONFIG_IIO is not set
+# CONFIG_PWM is not set
+
+#
+# IRQ chip support
+#
+# end of IRQ chip support
+
+# CONFIG_IPACK_BUS is not set
+# CONFIG_RESET_CONTROLLER is not set
+
+#
+# PHY Subsystem
+#
+# CONFIG_GENERIC_PHY is not set
+# CONFIG_BCM_KONA_USB2_PHY is not set
+# CONFIG_PHY_PXA_28NM_HSIC is not set
+# CONFIG_PHY_PXA_28NM_USB2 is not set
+# CONFIG_PHY_INTEL_LGM_EMMC is not set
+# end of PHY Subsystem
+
+# CONFIG_POWERCAP is not set
+# CONFIG_MCB is not set
+
+#
+# Performance monitor support
+#
+# end of Performance monitor support
+
+CONFIG_RAS=y
+
+#
+# Android
+#
+# CONFIG_ANDROID is not set
+# end of Android
+
+# CONFIG_LIBNVDIMM is not set
+# CONFIG_DAX is not set
+# CONFIG_NVMEM is not set
+
+#
+# HW tracing support
+#
+# CONFIG_STM is not set
+# CONFIG_INTEL_TH is not set
+# end of HW tracing support
+
+# CONFIG_FPGA is not set
+# CONFIG_TEE is not set
+# CONFIG_UNISYS_VISORBUS is not set
+# CONFIG_SIOX is not set
+# CONFIG_SLIMBUS is not set
+# CONFIG_INTERCONNECT is not set
+# CONFIG_COUNTER is not set
+# end of Device Drivers
+
+#
+# File systems
+#
+CONFIG_DCACHE_WORD_ACCESS=y
+# CONFIG_VALIDATE_FS_PARSER is not set
+CONFIG_FS_IOMAP=y
+# CONFIG_EXT2_FS is not set
+# CONFIG_EXT3_FS is not set
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_USE_FOR_EXT2=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_EXT4_DEBUG=y
+CONFIG_JBD2=y
+CONFIG_JBD2_DEBUG=y
+CONFIG_FS_MBCACHE=y
+# CONFIG_REISERFS_FS is not set
+# CONFIG_JFS_FS is not set
+# CONFIG_XFS_FS is not set
+# CONFIG_GFS2_FS is not set
+# CONFIG_BTRFS_FS is not set
+# CONFIG_NILFS2_FS is not set
+# CONFIG_F2FS_FS is not set
+# CONFIG_FS_DAX is not set
+CONFIG_FS_POSIX_ACL=y
+CONFIG_EXPORTFS=y
+# CONFIG_EXPORTFS_BLOCK_OPS is not set
+CONFIG_FILE_LOCKING=y
+CONFIG_MANDATORY_FILE_LOCKING=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FS_ENCRYPTION_ALGS=y
+# CONFIG_FS_VERITY is not set
+CONFIG_FSNOTIFY=y
+CONFIG_DNOTIFY=y
+CONFIG_INOTIFY_USER=y
+CONFIG_FANOTIFY=y
+CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y
+CONFIG_QUOTA=y
+CONFIG_QUOTA_NETLINK_INTERFACE=y
+# CONFIG_PRINT_QUOTA_WARNING is not set
+# CONFIG_QUOTA_DEBUG is not set
+# CONFIG_QFMT_V1 is not set
+# CONFIG_QFMT_V2 is not set
+CONFIG_QUOTACTL=y
+# CONFIG_AUTOFS4_FS is not set
+# CONFIG_AUTOFS_FS is not set
+# CONFIG_FUSE_FS is not set
+CONFIG_OVERLAY_FS=y
+# CONFIG_OVERLAY_FS_REDIRECT_DIR is not set
+CONFIG_OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW=y
+# CONFIG_OVERLAY_FS_INDEX is not set
+# CONFIG_OVERLAY_FS_XINO_AUTO is not set
+# CONFIG_OVERLAY_FS_METACOPY is not set
+
+#
+# Caches
+#
+# CONFIG_FSCACHE is not set
+# end of Caches
+
+#
+# CD-ROM/DVD Filesystems
+#
+# CONFIG_ISO9660_FS is not set
+# CONFIG_UDF_FS is not set
+# end of CD-ROM/DVD Filesystems
+
+#
+# DOS/FAT/EXFAT/NT Filesystems
+#
+# CONFIG_MSDOS_FS is not set
+# CONFIG_VFAT_FS is not set
+# CONFIG_EXFAT_FS is not set
+# CONFIG_NTFS_FS is not set
+# end of DOS/FAT/EXFAT/NT Filesystems
+
+#
+# Pseudo filesystems
+#
+CONFIG_PROC_FS=y
+CONFIG_PROC_KCORE=y
+CONFIG_PROC_SYSCTL=y
+CONFIG_PROC_PAGE_MONITOR=y
+CONFIG_PROC_CHILDREN=y
+CONFIG_PROC_PID_ARCH_STATUS=y
+CONFIG_KERNFS=y
+CONFIG_SYSFS=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_TMPFS_XATTR=y
+# CONFIG_TMPFS_INODE64 is not set
+CONFIG_HUGETLBFS=y
+CONFIG_HUGETLB_PAGE=y
+CONFIG_MEMFD_CREATE=y
+CONFIG_ARCH_HAS_GIGANTIC_PAGE=y
+# CONFIG_CONFIGFS_FS is not set
+# end of Pseudo filesystems
+
+CONFIG_MISC_FILESYSTEMS=y
+# CONFIG_ORANGEFS_FS is not set
+# CONFIG_ADFS_FS is not set
+# CONFIG_AFFS_FS is not set
+# CONFIG_ECRYPT_FS is not set
+# CONFIG_HFS_FS is not set
+# CONFIG_HFSPLUS_FS is not set
+# CONFIG_BEFS_FS is not set
+# CONFIG_BFS_FS is not set
+# CONFIG_EFS_FS is not set
+# CONFIG_CRAMFS is not set
+CONFIG_SQUASHFS=y
+CONFIG_SQUASHFS_FILE_CACHE=y
+# CONFIG_SQUASHFS_FILE_DIRECT is not set
+CONFIG_SQUASHFS_DECOMP_SINGLE=y
+# CONFIG_SQUASHFS_DECOMP_MULTI is not set
+# CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU is not set
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_ZLIB=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+# CONFIG_SQUASHFS_4K_DEVBLK_SIZE is not set
+# CONFIG_SQUASHFS_EMBEDDED is not set
+CONFIG_SQUASHFS_FRAGMENT_CACHE_SIZE=3
+# CONFIG_VXFS_FS is not set
+# CONFIG_MINIX_FS is not set
+# CONFIG_OMFS_FS is not set
+# CONFIG_HPFS_FS is not set
+# CONFIG_QNX4FS_FS is not set
+# CONFIG_QNX6FS_FS is not set
+# CONFIG_ROMFS_FS is not set
+CONFIG_PSTORE=y
+CONFIG_PSTORE_DEFLATE_COMPRESS=y
+# CONFIG_PSTORE_LZO_COMPRESS is not set
+# CONFIG_PSTORE_LZ4_COMPRESS is not set
+# CONFIG_PSTORE_LZ4HC_COMPRESS is not set
+# CONFIG_PSTORE_842_COMPRESS is not set
+# CONFIG_PSTORE_ZSTD_COMPRESS is not set
+CONFIG_PSTORE_COMPRESS=y
+CONFIG_PSTORE_DEFLATE_COMPRESS_DEFAULT=y
+CONFIG_PSTORE_COMPRESS_DEFAULT="deflate"
+# CONFIG_PSTORE_CONSOLE is not set
+# CONFIG_PSTORE_PMSG is not set
+# CONFIG_PSTORE_RAM is not set
+# CONFIG_SYSV_FS is not set
+# CONFIG_UFS_FS is not set
+# CONFIG_EROFS_FS is not set
+CONFIG_NETWORK_FILESYSTEMS=y
+# CONFIG_NFS_FS is not set
+# CONFIG_NFSD is not set
+# CONFIG_CEPH_FS is not set
+# CONFIG_CIFS is not set
+# CONFIG_CODA_FS is not set
+# CONFIG_AFS_FS is not set
+CONFIG_NLS=y
+CONFIG_NLS_DEFAULT="utf8"
+# CONFIG_NLS_CODEPAGE_437 is not set
+# CONFIG_NLS_CODEPAGE_737 is not set
+# CONFIG_NLS_CODEPAGE_775 is not set
+# CONFIG_NLS_CODEPAGE_850 is not set
+# CONFIG_NLS_CODEPAGE_852 is not set
+# CONFIG_NLS_CODEPAGE_855 is not set
+# CONFIG_NLS_CODEPAGE_857 is not set
+# CONFIG_NLS_CODEPAGE_860 is not set
+# CONFIG_NLS_CODEPAGE_861 is not set
+# CONFIG_NLS_CODEPAGE_862 is not set
+# CONFIG_NLS_CODEPAGE_863 is not set
+# CONFIG_NLS_CODEPAGE_864 is not set
+# CONFIG_NLS_CODEPAGE_865 is not set
+# CONFIG_NLS_CODEPAGE_866 is not set
+# CONFIG_NLS_CODEPAGE_869 is not set
+# CONFIG_NLS_CODEPAGE_936 is not set
+# CONFIG_NLS_CODEPAGE_950 is not set
+# CONFIG_NLS_CODEPAGE_932 is not set
+# CONFIG_NLS_CODEPAGE_949 is not set
+# CONFIG_NLS_CODEPAGE_874 is not set
+# CONFIG_NLS_ISO8859_8 is not set
+# CONFIG_NLS_CODEPAGE_1250 is not set
+# CONFIG_NLS_CODEPAGE_1251 is not set
+# CONFIG_NLS_ASCII is not set
+# CONFIG_NLS_ISO8859_1 is not set
+# CONFIG_NLS_ISO8859_2 is not set
+# CONFIG_NLS_ISO8859_3 is not set
+# CONFIG_NLS_ISO8859_4 is not set
+# CONFIG_NLS_ISO8859_5 is not set
+# CONFIG_NLS_ISO8859_6 is not set
+# CONFIG_NLS_ISO8859_7 is not set
+# CONFIG_NLS_ISO8859_9 is not set
+# CONFIG_NLS_ISO8859_13 is not set
+# CONFIG_NLS_ISO8859_14 is not set
+# CONFIG_NLS_ISO8859_15 is not set
+# CONFIG_NLS_KOI8_R is not set
+# CONFIG_NLS_KOI8_U is not set
+# CONFIG_NLS_MAC_ROMAN is not set
+# CONFIG_NLS_MAC_CELTIC is not set
+# CONFIG_NLS_MAC_CENTEURO is not set
+# CONFIG_NLS_MAC_CROATIAN is not set
+# CONFIG_NLS_MAC_CYRILLIC is not set
+# CONFIG_NLS_MAC_GAELIC is not set
+# CONFIG_NLS_MAC_GREEK is not set
+# CONFIG_NLS_MAC_ICELAND is not set
+# CONFIG_NLS_MAC_INUIT is not set
+# CONFIG_NLS_MAC_ROMANIAN is not set
+# CONFIG_NLS_MAC_TURKISH is not set
+# CONFIG_NLS_UTF8 is not set
+# CONFIG_UNICODE is not set
+CONFIG_IO_WQ=y
+# end of File systems
+
+#
+# Security options
+#
+CONFIG_KEYS=y
+# CONFIG_KEYS_REQUEST_CACHE is not set
+CONFIG_PERSISTENT_KEYRINGS=y
+CONFIG_ENCRYPTED_KEYS=y
+# CONFIG_KEY_DH_OPERATIONS is not set
+# CONFIG_SECURITY_DMESG_RESTRICT is not set
+CONFIG_SECURITY=y
+CONFIG_SECURITY_WRITABLE_HOOKS=y
+CONFIG_SECURITYFS=y
+CONFIG_SECURITY_NETWORK=y
+CONFIG_PAGE_TABLE_ISOLATION=y
+CONFIG_SECURITY_NETWORK_XFRM=y
+# CONFIG_SECURITY_PATH is not set
+CONFIG_LSM_MMAP_MIN_ADDR=65536
+CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR=y
+# CONFIG_HARDENED_USERCOPY is not set
+CONFIG_FORTIFY_SOURCE=y
+# CONFIG_STATIC_USERMODEHELPER is not set
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_SELINUX_DISABLE=y
+CONFIG_SECURITY_SELINUX_DEVELOP=y
+CONFIG_SECURITY_SELINUX_AVC_STATS=y
+CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE=1
+CONFIG_SECURITY_SELINUX_SIDTAB_HASH_BITS=9
+CONFIG_SECURITY_SELINUX_SID2STR_CACHE_SIZE=256
+# CONFIG_SECURITY_SMACK is not set
+# CONFIG_SECURITY_TOMOYO is not set
+# CONFIG_SECURITY_APPARMOR is not set
+# CONFIG_SECURITY_LOADPIN is not set
+# CONFIG_SECURITY_YAMA is not set
+# CONFIG_SECURITY_SAFESETID is not set
+# CONFIG_SECURITY_LOCKDOWN_LSM is not set
+CONFIG_INTEGRITY=y
+# CONFIG_INTEGRITY_SIGNATURE is not set
+CONFIG_INTEGRITY_AUDIT=y
+# CONFIG_IMA is not set
+# CONFIG_EVM is not set
+CONFIG_DEFAULT_SECURITY_SELINUX=y
+# CONFIG_DEFAULT_SECURITY_DAC is not set
+CONFIG_LSM="lockdown,yama,loadpin,safesetid,integrity,selinux,smack,tomoyo,apparmor,bpf"
+
+#
+# Kernel hardening options
+#
+
+#
+# Memory initialization
+#
+CONFIG_CC_HAS_AUTO_VAR_INIT_PATTERN=y
+CONFIG_CC_HAS_AUTO_VAR_INIT_ZERO=y
+CONFIG_INIT_STACK_NONE=y
+# CONFIG_GCC_PLUGIN_STRUCTLEAK_USER is not set
+# CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF is not set
+# CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL is not set
+# CONFIG_INIT_STACK_ALL_PATTERN is not set
+# CONFIG_INIT_STACK_ALL_ZERO is not set
+# CONFIG_GCC_PLUGIN_STACKLEAK is not set
+# CONFIG_INIT_ON_ALLOC_DEFAULT_ON is not set
+# CONFIG_INIT_ON_FREE_DEFAULT_ON is not set
+# end of Memory initialization
+# end of Kernel hardening options
+# end of Security options
+
+CONFIG_CRYPTO=y
+
+#
+# Crypto core or helper
+#
+CONFIG_CRYPTO_ALGAPI=y
+CONFIG_CRYPTO_ALGAPI2=y
+CONFIG_CRYPTO_AEAD=y
+CONFIG_CRYPTO_AEAD2=y
+CONFIG_CRYPTO_SKCIPHER=y
+CONFIG_CRYPTO_SKCIPHER2=y
+CONFIG_CRYPTO_HASH=y
+CONFIG_CRYPTO_HASH2=y
+CONFIG_CRYPTO_RNG=y
+CONFIG_CRYPTO_RNG2=y
+CONFIG_CRYPTO_RNG_DEFAULT=y
+CONFIG_CRYPTO_AKCIPHER2=y
+CONFIG_CRYPTO_AKCIPHER=y
+CONFIG_CRYPTO_KPP2=y
+CONFIG_CRYPTO_KPP=y
+CONFIG_CRYPTO_ACOMP2=y
+CONFIG_CRYPTO_MANAGER=y
+CONFIG_CRYPTO_MANAGER2=y
+# CONFIG_CRYPTO_USER is not set
+CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=y
+CONFIG_CRYPTO_NULL=y
+CONFIG_CRYPTO_NULL2=y
+# CONFIG_CRYPTO_PCRYPT is not set
+# CONFIG_CRYPTO_CRYPTD is not set
+# CONFIG_CRYPTO_AUTHENC is not set
+# CONFIG_CRYPTO_TEST is not set
+
+#
+# Public-key cryptography
+#
+CONFIG_CRYPTO_RSA=y
+CONFIG_CRYPTO_DH=y
+CONFIG_CRYPTO_ECC=y
+CONFIG_CRYPTO_ECDH=y
+# CONFIG_CRYPTO_ECRDSA is not set
+# CONFIG_CRYPTO_SM2 is not set
+# CONFIG_CRYPTO_CURVE25519 is not set
+# CONFIG_CRYPTO_CURVE25519_X86 is not set
+
+#
+# Authenticated Encryption with Associated Data
+#
+# CONFIG_CRYPTO_CCM is not set
+# CONFIG_CRYPTO_GCM is not set
+# CONFIG_CRYPTO_CHACHA20POLY1305 is not set
+# CONFIG_CRYPTO_AEGIS128 is not set
+# CONFIG_CRYPTO_AEGIS128_AESNI_SSE2 is not set
+CONFIG_CRYPTO_SEQIV=y
+# CONFIG_CRYPTO_ECHAINIV is not set
+
+#
+# Block modes
+#
+CONFIG_CRYPTO_CBC=y
+# CONFIG_CRYPTO_CFB is not set
+CONFIG_CRYPTO_CTR=y
+CONFIG_CRYPTO_CTS=y
+CONFIG_CRYPTO_ECB=y
+# CONFIG_CRYPTO_LRW is not set
+# CONFIG_CRYPTO_OFB is not set
+# CONFIG_CRYPTO_PCBC is not set
+CONFIG_CRYPTO_XTS=y
+# CONFIG_CRYPTO_KEYWRAP is not set
+# CONFIG_CRYPTO_NHPOLY1305_SSE2 is not set
+# CONFIG_CRYPTO_NHPOLY1305_AVX2 is not set
+# CONFIG_CRYPTO_ADIANTUM is not set
+# CONFIG_CRYPTO_ESSIV is not set
+
+#
+# Hash modes
+#
+# CONFIG_CRYPTO_CMAC is not set
+CONFIG_CRYPTO_HMAC=y
+# CONFIG_CRYPTO_XCBC is not set
+# CONFIG_CRYPTO_VMAC is not set
+
+#
+# Digest
+#
+CONFIG_CRYPTO_CRC32C=y
+# CONFIG_CRYPTO_CRC32C_INTEL is not set
+# CONFIG_CRYPTO_CRC32 is not set
+# CONFIG_CRYPTO_CRC32_PCLMUL is not set
+# CONFIG_CRYPTO_XXHASH is not set
+# CONFIG_CRYPTO_BLAKE2B is not set
+# CONFIG_CRYPTO_BLAKE2S is not set
+# CONFIG_CRYPTO_BLAKE2S_X86 is not set
+CONFIG_CRYPTO_CRCT10DIF=y
+CONFIG_CRYPTO_CRCT10DIF_PCLMUL=y
+# CONFIG_CRYPTO_GHASH is not set
+# CONFIG_CRYPTO_POLY1305 is not set
+# CONFIG_CRYPTO_POLY1305_X86_64 is not set
+# CONFIG_CRYPTO_MD4 is not set
+CONFIG_CRYPTO_MD5=y
+# CONFIG_CRYPTO_MICHAEL_MIC is not set
+# CONFIG_CRYPTO_RMD128 is not set
+# CONFIG_CRYPTO_RMD160 is not set
+# CONFIG_CRYPTO_RMD256 is not set
+# CONFIG_CRYPTO_RMD320 is not set
+CONFIG_CRYPTO_SHA1=y
+# CONFIG_CRYPTO_SHA1_SSSE3 is not set
+# CONFIG_CRYPTO_SHA256_SSSE3 is not set
+# CONFIG_CRYPTO_SHA512_SSSE3 is not set
+CONFIG_CRYPTO_SHA256=y
+CONFIG_CRYPTO_SHA512=y
+# CONFIG_CRYPTO_SHA3 is not set
+# CONFIG_CRYPTO_SM3 is not set
+# CONFIG_CRYPTO_STREEBOG is not set
+# CONFIG_CRYPTO_TGR192 is not set
+# CONFIG_CRYPTO_WP512 is not set
+# CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL is not set
+
+#
+# Ciphers
+#
+CONFIG_CRYPTO_AES=y
+CONFIG_CRYPTO_AES_TI=y
+# CONFIG_CRYPTO_AES_NI_INTEL is not set
+# CONFIG_CRYPTO_BLOWFISH is not set
+# CONFIG_CRYPTO_BLOWFISH_X86_64 is not set
+# CONFIG_CRYPTO_CAMELLIA is not set
+# CONFIG_CRYPTO_CAMELLIA_X86_64 is not set
+# CONFIG_CRYPTO_CAMELLIA_AESNI_AVX_X86_64 is not set
+# CONFIG_CRYPTO_CAMELLIA_AESNI_AVX2_X86_64 is not set
+# CONFIG_CRYPTO_CAST5 is not set
+# CONFIG_CRYPTO_CAST5_AVX_X86_64 is not set
+# CONFIG_CRYPTO_CAST6 is not set
+# CONFIG_CRYPTO_CAST6_AVX_X86_64 is not set
+# CONFIG_CRYPTO_DES is not set
+# CONFIG_CRYPTO_DES3_EDE_X86_64 is not set
+# CONFIG_CRYPTO_FCRYPT is not set
+# CONFIG_CRYPTO_SALSA20 is not set
+# CONFIG_CRYPTO_CHACHA20 is not set
+# CONFIG_CRYPTO_CHACHA20_X86_64 is not set
+# CONFIG_CRYPTO_SERPENT is not set
+# CONFIG_CRYPTO_SERPENT_SSE2_X86_64 is not set
+# CONFIG_CRYPTO_SERPENT_AVX_X86_64 is not set
+# CONFIG_CRYPTO_SERPENT_AVX2_X86_64 is not set
+# CONFIG_CRYPTO_SM4 is not set
+# CONFIG_CRYPTO_TWOFISH is not set
+# CONFIG_CRYPTO_TWOFISH_X86_64 is not set
+# CONFIG_CRYPTO_TWOFISH_X86_64_3WAY is not set
+# CONFIG_CRYPTO_TWOFISH_AVX_X86_64 is not set
+
+#
+# Compression
+#
+CONFIG_CRYPTO_DEFLATE=y
+CONFIG_CRYPTO_LZO=y
+# CONFIG_CRYPTO_842 is not set
+# CONFIG_CRYPTO_LZ4 is not set
+# CONFIG_CRYPTO_LZ4HC is not set
+# CONFIG_CRYPTO_ZSTD is not set
+
+#
+# Random Number Generation
+#
+# CONFIG_CRYPTO_ANSI_CPRNG is not set
+CONFIG_CRYPTO_DRBG_MENU=y
+CONFIG_CRYPTO_DRBG_HMAC=y
+CONFIG_CRYPTO_DRBG_HASH=y
+CONFIG_CRYPTO_DRBG_CTR=y
+CONFIG_CRYPTO_DRBG=y
+CONFIG_CRYPTO_JITTERENTROPY=y
+# CONFIG_CRYPTO_USER_API_HASH is not set
+# CONFIG_CRYPTO_USER_API_SKCIPHER is not set
+# CONFIG_CRYPTO_USER_API_RNG is not set
+# CONFIG_CRYPTO_USER_API_AEAD is not set
+CONFIG_CRYPTO_HASH_INFO=y
+# CONFIG_CRYPTO_HW is not set
+CONFIG_ASYMMETRIC_KEY_TYPE=y
+CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y
+CONFIG_X509_CERTIFICATE_PARSER=y
+# CONFIG_PKCS8_PRIVATE_KEY_PARSER is not set
+CONFIG_PKCS7_MESSAGE_PARSER=y
+# CONFIG_PKCS7_TEST_KEY is not set
+CONFIG_SIGNED_PE_FILE_VERIFICATION=y
+
+#
+# Certificates for signature checking
+#
+CONFIG_MODULE_SIG_KEY="certs/signing_key.pem"
+CONFIG_SYSTEM_TRUSTED_KEYRING=y
+CONFIG_SYSTEM_TRUSTED_KEYS=""
+# CONFIG_SYSTEM_EXTRA_CERTIFICATE is not set
+# CONFIG_SECONDARY_TRUSTED_KEYRING is not set
+CONFIG_SYSTEM_BLACKLIST_KEYRING=y
+CONFIG_SYSTEM_BLACKLIST_HASH_LIST=""
+# CONFIG_SYSTEM_REVOCATION_LIST is not set
+# end of Certificates for signature checking
+
+#
+# Library routines
+#
+# CONFIG_PACKING is not set
+CONFIG_BITREVERSE=y
+CONFIG_GENERIC_STRNCPY_FROM_USER=y
+CONFIG_GENERIC_STRNLEN_USER=y
+CONFIG_GENERIC_NET_UTILS=y
+CONFIG_GENERIC_FIND_FIRST_BIT=y
+# CONFIG_CORDIC is not set
+# CONFIG_PRIME_NUMBERS is not set
+CONFIG_GENERIC_PCI_IOMAP=y
+CONFIG_GENERIC_IOMAP=y
+CONFIG_ARCH_USE_CMPXCHG_LOCKREF=y
+CONFIG_ARCH_HAS_FAST_MULTIPLIER=y
+CONFIG_ARCH_USE_SYM_ANNOTATIONS=y
+
+#
+# Crypto library routines
+#
+CONFIG_CRYPTO_LIB_AES=y
+CONFIG_CRYPTO_LIB_BLAKE2S_GENERIC=y
+# CONFIG_CRYPTO_LIB_CHACHA is not set
+# CONFIG_CRYPTO_LIB_CURVE25519 is not set
+CONFIG_CRYPTO_LIB_POLY1305_RSIZE=11
+# CONFIG_CRYPTO_LIB_POLY1305 is not set
+# CONFIG_CRYPTO_LIB_CHACHA20POLY1305 is not set
+CONFIG_CRYPTO_LIB_SHA256=y
+# end of Crypto library routines
+
+CONFIG_LIB_MEMNEQ=y
+CONFIG_CRC_CCITT=y
+CONFIG_CRC16=y
+CONFIG_CRC_T10DIF=y
+# CONFIG_CRC_ITU_T is not set
+CONFIG_CRC32=y
+# CONFIG_CRC32_SELFTEST is not set
+CONFIG_CRC32_SLICEBY8=y
+# CONFIG_CRC32_SLICEBY4 is not set
+# CONFIG_CRC32_SARWATE is not set
+# CONFIG_CRC32_BIT is not set
+# CONFIG_CRC64 is not set
+# CONFIG_CRC4 is not set
+# CONFIG_CRC7 is not set
+CONFIG_LIBCRC32C=y
+# CONFIG_CRC8 is not set
+CONFIG_XXHASH=y
+# CONFIG_RANDOM32_SELFTEST is not set
+CONFIG_ZLIB_INFLATE=y
+CONFIG_ZLIB_DEFLATE=y
+CONFIG_LZO_COMPRESS=y
+CONFIG_LZO_DECOMPRESS=y
+CONFIG_LZ4_DECOMPRESS=y
+CONFIG_ZSTD_DECOMPRESS=y
+CONFIG_XZ_DEC=y
+CONFIG_XZ_DEC_X86=y
+CONFIG_XZ_DEC_POWERPC=y
+CONFIG_XZ_DEC_IA64=y
+CONFIG_XZ_DEC_ARM=y
+CONFIG_XZ_DEC_ARMTHUMB=y
+CONFIG_XZ_DEC_SPARC=y
+CONFIG_XZ_DEC_BCJ=y
+# CONFIG_XZ_DEC_TEST is not set
+CONFIG_DECOMPRESS_GZIP=y
+CONFIG_DECOMPRESS_BZIP2=y
+CONFIG_DECOMPRESS_LZMA=y
+CONFIG_DECOMPRESS_XZ=y
+CONFIG_DECOMPRESS_LZO=y
+CONFIG_DECOMPRESS_LZ4=y
+CONFIG_DECOMPRESS_ZSTD=y
+CONFIG_GENERIC_ALLOCATOR=y
+CONFIG_XARRAY_MULTI=y
+CONFIG_ASSOCIATIVE_ARRAY=y
+CONFIG_HAS_IOMEM=y
+CONFIG_HAS_IOPORT_MAP=y
+CONFIG_HAS_DMA=y
+CONFIG_NEED_SG_DMA_LENGTH=y
+CONFIG_NEED_DMA_MAP_STATE=y
+CONFIG_ARCH_DMA_ADDR_T_64BIT=y
+CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED=y
+CONFIG_SWIOTLB=y
+CONFIG_DMA_COHERENT_POOL=y
+# CONFIG_DMA_API_DEBUG is not set
+CONFIG_SGL_ALLOC=y
+CONFIG_CPU_RMAP=y
+CONFIG_DQL=y
+CONFIG_NLATTR=y
+CONFIG_CLZ_TAB=y
+CONFIG_IRQ_POLL=y
+CONFIG_MPILIB=y
+CONFIG_OID_REGISTRY=y
+CONFIG_HAVE_GENERIC_VDSO=y
+CONFIG_GENERIC_GETTIMEOFDAY=y
+CONFIG_GENERIC_VDSO_TIME_NS=y
+CONFIG_SG_POOL=y
+CONFIG_ARCH_HAS_PMEM_API=y
+CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE=y
+CONFIG_ARCH_HAS_COPY_MC=y
+CONFIG_ARCH_STACKWALK=y
+CONFIG_SBITMAP=y
+# CONFIG_STRING_SELFTEST is not set
+# end of Library routines
+
+#
+# Kernel hacking
+#
+
+#
+# printk and dmesg options
+#
+CONFIG_PRINTK_TIME=y
+# CONFIG_PRINTK_CALLER is not set
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=7
+CONFIG_CONSOLE_LOGLEVEL_QUIET=4
+CONFIG_MESSAGE_LOGLEVEL_DEFAULT=4
+# CONFIG_BOOT_PRINTK_DELAY is not set
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_DYNAMIC_DEBUG_CORE=y
+CONFIG_SYMBOLIC_ERRNAME=y
+CONFIG_DEBUG_BUGVERBOSE=y
+# end of printk and dmesg options
+
+#
+# Compile-time checks and compiler options
+#
+# CONFIG_DEBUG_INFO is not set
+# CONFIG_ENABLE_MUST_CHECK is not set
+CONFIG_FRAME_WARN=2048
+CONFIG_STRIP_ASM_SYMS=y
+# CONFIG_READABLE_ASM is not set
+# CONFIG_HEADERS_INSTALL is not set
+CONFIG_DEBUG_SECTION_MISMATCH=y
+CONFIG_SECTION_MISMATCH_WARN_ONLY=y
+CONFIG_FRAME_POINTER=y
+CONFIG_STACK_VALIDATION=y
+# CONFIG_DEBUG_FORCE_WEAK_PER_CPU is not set
+# end of Compile-time checks and compiler options
+
+#
+# Generic Kernel Debugging Instruments
+#
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE=0x1
+CONFIG_MAGIC_SYSRQ_SERIAL=y
+CONFIG_MAGIC_SYSRQ_SERIAL_SEQUENCE=""
+CONFIG_DEBUG_FS=y
+CONFIG_DEBUG_FS_ALLOW_ALL=y
+# CONFIG_DEBUG_FS_DISALLOW_MOUNT is not set
+# CONFIG_DEBUG_FS_ALLOW_NONE is not set
+CONFIG_HAVE_ARCH_KGDB=y
+# CONFIG_KGDB is not set
+CONFIG_ARCH_HAS_UBSAN_SANITIZE_ALL=y
+# CONFIG_UBSAN is not set
+CONFIG_HAVE_ARCH_KCSAN=y
+CONFIG_HAVE_KCSAN_COMPILER=y
+# CONFIG_KCSAN is not set
+# end of Generic Kernel Debugging Instruments
+
+CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_MISC=y
+
+#
+# Memory Debugging
+#
+# CONFIG_PAGE_EXTENSION is not set
+# CONFIG_DEBUG_PAGEALLOC is not set
+# CONFIG_PAGE_OWNER is not set
+# CONFIG_PAGE_POISONING is not set
+# CONFIG_DEBUG_RODATA_TEST is not set
+CONFIG_ARCH_HAS_DEBUG_WX=y
+# CONFIG_DEBUG_WX is not set
+CONFIG_GENERIC_PTDUMP=y
+# CONFIG_PTDUMP_DEBUGFS is not set
+# CONFIG_DEBUG_OBJECTS is not set
+# CONFIG_SLUB_DEBUG_ON is not set
+# CONFIG_SLUB_STATS is not set
+CONFIG_HAVE_DEBUG_KMEMLEAK=y
+# CONFIG_DEBUG_KMEMLEAK is not set
+# CONFIG_DEBUG_STACK_USAGE is not set
+# CONFIG_SCHED_STACK_END_CHECK is not set
+CONFIG_ARCH_HAS_DEBUG_VM_PGTABLE=y
+# CONFIG_DEBUG_VM is not set
+# CONFIG_DEBUG_VM_PGTABLE is not set
+CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y
+# CONFIG_DEBUG_VIRTUAL is not set
+CONFIG_DEBUG_MEMORY_INIT=y
+# CONFIG_DEBUG_PER_CPU_MAPS is not set
+CONFIG_HAVE_ARCH_KASAN=y
+CONFIG_HAVE_ARCH_KASAN_VMALLOC=y
+CONFIG_CC_HAS_KASAN_GENERIC=y
+CONFIG_CC_HAS_WORKING_NOSANITIZE_ADDRESS=y
+# CONFIG_KASAN is not set
+# end of Memory Debugging
+
+# CONFIG_DEBUG_SHIRQ is not set
+
+#
+# Debug Oops, Lockups and Hangs
+#
+# CONFIG_PANIC_ON_OOPS is not set
+CONFIG_PANIC_ON_OOPS_VALUE=0
+CONFIG_PANIC_TIMEOUT=0
+# CONFIG_SOFTLOCKUP_DETECTOR is not set
+CONFIG_HARDLOCKUP_CHECK_TIMESTAMP=y
+# CONFIG_HARDLOCKUP_DETECTOR is not set
+# CONFIG_DETECT_HUNG_TASK is not set
+# CONFIG_WQ_WATCHDOG is not set
+# CONFIG_TEST_LOCKUP is not set
+# end of Debug Oops, Lockups and Hangs
+
+#
+# Scheduler Debugging
+#
+# CONFIG_SCHED_DEBUG is not set
+CONFIG_SCHED_INFO=y
+# CONFIG_SCHEDSTATS is not set
+# end of Scheduler Debugging
+
+# CONFIG_DEBUG_TIMEKEEPING is not set
+
+#
+# Lock Debugging (spinlocks, mutexes, etc...)
+#
+CONFIG_LOCK_DEBUGGING_SUPPORT=y
+# CONFIG_PROVE_LOCKING is not set
+# CONFIG_LOCK_STAT is not set
+# CONFIG_DEBUG_RT_MUTEXES is not set
+# CONFIG_DEBUG_SPINLOCK is not set
+# CONFIG_DEBUG_MUTEXES is not set
+# CONFIG_DEBUG_WW_MUTEX_SLOWPATH is not set
+# CONFIG_DEBUG_RWSEMS is not set
+# CONFIG_DEBUG_LOCK_ALLOC is not set
+# CONFIG_DEBUG_ATOMIC_SLEEP is not set
+# CONFIG_DEBUG_LOCKING_API_SELFTESTS is not set
+# CONFIG_LOCK_TORTURE_TEST is not set
+# CONFIG_WW_MUTEX_SELFTEST is not set
+# CONFIG_SCF_TORTURE_TEST is not set
+# CONFIG_CSD_LOCK_WAIT_DEBUG is not set
+# end of Lock Debugging (spinlocks, mutexes, etc...)
+
+CONFIG_STACKTRACE=y
+# CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set
+# CONFIG_DEBUG_KOBJECT is not set
+
+#
+# Debug kernel data structures
+#
+CONFIG_DEBUG_LIST=y
+# CONFIG_DEBUG_PLIST is not set
+# CONFIG_DEBUG_SG is not set
+# CONFIG_DEBUG_NOTIFIERS is not set
+CONFIG_BUG_ON_DATA_CORRUPTION=y
+# end of Debug kernel data structures
+
+# CONFIG_DEBUG_CREDENTIALS is not set
+
+#
+# RCU Debugging
+#
+# CONFIG_RCU_SCALE_TEST is not set
+# CONFIG_RCU_TORTURE_TEST is not set
+# CONFIG_RCU_REF_SCALE_TEST is not set
+CONFIG_RCU_CPU_STALL_TIMEOUT=59
+# CONFIG_RCU_TRACE is not set
+# CONFIG_RCU_EQS_DEBUG is not set
+# end of RCU Debugging
+
+# CONFIG_DEBUG_WQ_FORCE_RR_CPU is not set
+# CONFIG_DEBUG_BLOCK_EXT_DEVT is not set
+# CONFIG_CPU_HOTPLUG_STATE_CONTROL is not set
+# CONFIG_LATENCYTOP is not set
+CONFIG_USER_STACKTRACE_SUPPORT=y
+CONFIG_HAVE_FUNCTION_TRACER=y
+CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
+CONFIG_HAVE_DYNAMIC_FTRACE=y
+CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS=y
+CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS=y
+CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
+CONFIG_HAVE_SYSCALL_TRACEPOINTS=y
+CONFIG_HAVE_FENTRY=y
+CONFIG_HAVE_C_RECORDMCOUNT=y
+CONFIG_TRACING_SUPPORT=y
+# CONFIG_FTRACE is not set
+# CONFIG_SAMPLES is not set
+CONFIG_ARCH_HAS_DEVMEM_IS_ALLOWED=y
+CONFIG_STRICT_DEVMEM=y
+# CONFIG_IO_STRICT_DEVMEM is not set
+
+#
+# x86 Debugging
+#
+CONFIG_TRACE_IRQFLAGS_SUPPORT=y
+CONFIG_TRACE_IRQFLAGS_NMI_SUPPORT=y
+CONFIG_X86_VERBOSE_BOOTUP=y
+CONFIG_EARLY_PRINTK=y
+# CONFIG_DEBUG_TLBFLUSH is not set
+CONFIG_HAVE_MMIOTRACE_SUPPORT=y
+# CONFIG_X86_DECODER_SELFTEST is not set
+CONFIG_IO_DELAY_0X80=y
+# CONFIG_IO_DELAY_0XED is not set
+# CONFIG_IO_DELAY_UDELAY is not set
+# CONFIG_IO_DELAY_NONE is not set
+# CONFIG_DEBUG_BOOT_PARAMS is not set
+# CONFIG_CPA_DEBUG is not set
+# CONFIG_DEBUG_ENTRY is not set
+# CONFIG_DEBUG_NMI_SELFTEST is not set
+# CONFIG_X86_DEBUG_FPU is not set
+# CONFIG_UNWINDER_ORC is not set
+CONFIG_UNWINDER_FRAME_POINTER=y
+# end of x86 Debugging
+
+#
+# Kernel Testing and Coverage
+#
+# CONFIG_KUNIT is not set
+# CONFIG_NOTIFIER_ERROR_INJECTION is not set
+CONFIG_FUNCTION_ERROR_INJECTION=y
+# CONFIG_FAULT_INJECTION is not set
+CONFIG_ARCH_HAS_KCOV=y
+CONFIG_CC_HAS_SANCOV_TRACE_PC=y
+# CONFIG_KCOV is not set
+CONFIG_RUNTIME_TESTING_MENU=y
+# CONFIG_LKDTM is not set
+# CONFIG_TEST_LIST_SORT is not set
+# CONFIG_TEST_MIN_HEAP is not set
+# CONFIG_TEST_SORT is not set
+# CONFIG_KPROBES_SANITY_TEST is not set
+# CONFIG_BACKTRACE_SELF_TEST is not set
+# CONFIG_RBTREE_TEST is not set
+# CONFIG_REED_SOLOMON_TEST is not set
+# CONFIG_INTERVAL_TREE_TEST is not set
+# CONFIG_PERCPU_TEST is not set
+# CONFIG_ATOMIC64_SELFTEST is not set
+# CONFIG_TEST_HEXDUMP is not set
+# CONFIG_TEST_STRING_HELPERS is not set
+# CONFIG_TEST_STRSCPY is not set
+# CONFIG_TEST_KSTRTOX is not set
+# CONFIG_TEST_PRINTF is not set
+# CONFIG_TEST_BITMAP is not set
+# CONFIG_TEST_UUID is not set
+# CONFIG_TEST_XARRAY is not set
+# CONFIG_TEST_OVERFLOW is not set
+# CONFIG_TEST_RHASHTABLE is not set
+# CONFIG_TEST_HASH is not set
+# CONFIG_TEST_IDA is not set
+# CONFIG_TEST_LKM is not set
+# CONFIG_TEST_BITOPS is not set
+# CONFIG_TEST_VMALLOC is not set
+# CONFIG_TEST_USER_COPY is not set
+# CONFIG_TEST_BPF is not set
+# CONFIG_TEST_BLACKHOLE_DEV is not set
+# CONFIG_FIND_BIT_BENCHMARK is not set
+# CONFIG_TEST_FIRMWARE is not set
+# CONFIG_TEST_SYSCTL is not set
+# CONFIG_TEST_UDELAY is not set
+# CONFIG_TEST_STATIC_KEYS is not set
+# CONFIG_TEST_KMOD is not set
+# CONFIG_TEST_MEMCAT_P is not set
+# CONFIG_TEST_STACKINIT is not set
+# CONFIG_TEST_MEMINIT is not set
+# CONFIG_TEST_FREE_PAGES is not set
+# CONFIG_TEST_FPU is not set
+# CONFIG_MEMTEST is not set
+# end of Kernel Testing and Coverage
+# end of Kernel hacking
diff --git a/packaging/Makefile b/packaging/Makefile
index 66c789282..22bfb4404 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -38,7 +38,7 @@ firecracker-bins: target-dir build-dir
 
 vmlinux:
 	#curl -fsSL -o ./target/vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin
-	curl -fsSL -o ./target/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/vmlinux.bin
+	curl -fsSL -o ./target/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.2.2/vmlinux.bin
 	#cp ../kernels/vmlinux.bin ./target/vmlinux.bin
 
 version:

From 9b2b52ad2d1041138c9f36d039e6c24cb8fb7cb1 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 4 Jul 2022 19:22:36 +0200
Subject: [PATCH 316/990] Fix: Pubsub subscribers did not get all relevant
 messages

Subscribers did not get triggered on 'item_hash' from message with a 'ref'.
---
 vm_supervisor/tasks.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py
index f8ff3c112..9b09fb064 100644
--- a/vm_supervisor/tasks.py
+++ b/vm_supervisor/tasks.py
@@ -99,11 +99,9 @@ async def watch_for_messages(dispatcher: PubSub, reactor: Reactor):
         logger.info(f"Websocket received message: {message.item_hash}")
 
         # Dispatch update to running VMs
+        await dispatcher.publish(key=message.item_hash, value=message)
         if hasattr(message.content, "ref") and message.content.ref:
-            ref = message.content.ref
-        else:
-            ref = message.item_hash
-        await dispatcher.publish(key=ref, value=message)
+            await dispatcher.publish(key=message.content.ref, value=message)
 
         # Register new VM to run on future messages:
         if isinstance(message, ProgramMessage):

From 649aaa8b24eba646b2670fed5a569be7d38efbed Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 4 Jul 2022 19:27:54 +0200
Subject: [PATCH 317/990] Doc: Format of Caddy GPG is now armored, did not work
 anymore

---
 CONFIGURE_CADDY.md          | 2 +-
 doc/INSTALL-Debian-11.md    | 4 ++--
 doc/INSTALL-Ubuntu-20.04.md | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/CONFIGURE_CADDY.md b/CONFIGURE_CADDY.md
index 472e8fb41..4efafca9b 100644
--- a/CONFIGURE_CADDY.md
+++ b/CONFIGURE_CADDY.md
@@ -38,7 +38,7 @@ To install on Debian/Ubuntu, according to the
 [official instructions](https://caddyserver.com/docs/install#debian-ubuntu-raspbian):
 ```shell
 sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo apt-key add -
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
 curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list
 sudo apt update
 sudo apt install caddy
diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index 234a6a6ca..f7a9c5113 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -86,8 +86,8 @@ This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CON
 
 Again, run these commands as `root`:
 ```shell
-apt install -y debian-keyring debian-archive-keyring apt-transport-https gnupg
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | apt-key add -
+ apt install -y debian-keyring debian-archive-keyring apt-transport-https
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
 curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list
 apt update
 apt install caddy
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
index 173e5b1d5..46169c818 100644
--- a/doc/INSTALL-Ubuntu-20.04.md
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -85,8 +85,8 @@ This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CON
 
 Again, run these commands as `root`:
 ```shell
-sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https gnupg
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo apt-key add -
+sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
 curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list
 sudo apt update
 sudo apt install caddy

From 0e2fad4ec6612538dc5a42cb95f33f019246132e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 4 Jul 2022 19:20:09 +0200
Subject: [PATCH 318/990] Chore: Cleanup internal logging

---
 vm_supervisor/pubsub.py | 1 -
 vm_supervisor/status.py | 4 +++-
 vm_supervisor/tasks.py  | 2 --
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/vm_supervisor/pubsub.py b/vm_supervisor/pubsub.py
index e594c553c..3ef1d04a3 100644
--- a/vm_supervisor/pubsub.py
+++ b/vm_supervisor/pubsub.py
@@ -31,6 +31,5 @@ async def msubscribe(self, *keys):
         return await queue.get()
 
     async def publish(self, key, value):
-        logger.debug(f"publish({key}, ...)")
         for queue in self.subscribers.get(key, tuple()):
             await queue.put(value)
diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py
index a1625b93d..ddddacca3 100644
--- a/vm_supervisor/status.py
+++ b/vm_supervisor/status.py
@@ -2,13 +2,15 @@
 Used to check that the example_fastapi program works as expected
 in a deployed supervisor.
 """
-
+import logging
 from typing import Dict, Any, List
 
 from aiohttp import ClientSession, ClientResponseError
 
 from vm_supervisor.conf import settings
 
+logger = logging.getLogger(__name__)
+
 CHECK_VM_URL = f"http://{settings.SUPERVISOR_HOST}:{settings.SUPERVISOR_PORT}/vm/{settings.CHECK_FASTAPI_VM_ID}"
 
 
diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py
index 9b09fb064..b2b59b365 100644
--- a/vm_supervisor/tasks.py
+++ b/vm_supervisor/tasks.py
@@ -41,7 +41,6 @@ async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]:
         async with session.ws_connect(url) as ws:
             logger.debug(f"Websocket connected on {url}")
             async for msg in ws:
-                logger.debug(f"Websocket received data...")
                 if msg.type == aiohttp.WSMsgType.TEXT:
                     try:
                         data = json.loads(msg.data)
@@ -96,7 +95,6 @@ async def watch_for_messages(dispatcher: PubSub, reactor: Reactor):
     )
 
     async for message in retry_generator(subscribe_via_ws(url)):
-        logger.info(f"Websocket received message: {message.item_hash}")
 
         # Dispatch update to running VMs
         await dispatcher.publish(key=message.item_hash, value=message)

From bfeca7a9d2fc67ee0b92d1f2bb884af955066b58 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 4 Jul 2022 22:47:22 +0200
Subject: [PATCH 319/990] Fix: VM config files were never closed

---
 firecracker/microvm.py | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 7a5ebd855..d7aec587b 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -67,7 +67,7 @@ class MicroVM:
     network_interface: Optional[str] = None
     stdout_task: Optional[Task] = None
     stderr_task: Optional[Task] = None
-    config_file = None
+    config_file_path: Optional[Path] = None
     drives: List[Drive]
     init_timeout: float
 
@@ -160,13 +160,13 @@ async def start_firecracker(
         if os.path.exists(self.socket_path):
             os.remove(self.socket_path)
 
-        config_file = NamedTemporaryFile()
-        config_file.write(
-            config.json(by_alias=True, exclude_none=True, indent=4).encode()
-        )
-        config_file.flush()
-        self.config_file = config_file
-        print(self.config_file)
+        with NamedTemporaryFile(delete=False) as config_file:
+            config_file.write(
+                config.json(by_alias=True, exclude_none=True, indent=4).encode()
+            )
+            config_file.flush()
+            os.chmod(config_file.name, 0o644)
+            self.config_file_path = Path(config_file.name)
 
         logger.debug(
             " ".join(
@@ -200,14 +200,13 @@ async def start_jailed_firecracker(
         uid = str(getpwnam("jailman").pw_uid)
         gid = str(getpwnam("jailman").pw_gid)
 
-        # config_file = NamedTemporaryFile(dir=f"{self.jailer_path}/tmp/", suffix='.json')
-        config_file = open(f"{self.jailer_path}/tmp/config.json", "wb")
-        config_file.write(
-            config.json(by_alias=True, exclude_none=True, indent=4).encode()
-        )
-        config_file.flush()
-        os.chmod(config_file.name, 0o644)
-        self.config_file = config_file
+        with open(f"{self.jailer_path}/tmp/config.json", "wb") as config_file:
+            config_file.write(
+                config.json(by_alias=True, exclude_none=True, indent=4).encode()
+            )
+            config_file.flush()
+            os.chmod(config_file.name, 0o644)
+            self.config_file_path = Path(config_file.name)
 
         logger.debug(
             " ".join(
@@ -444,6 +443,8 @@ async def teardown(self):
             )
 
         logger.debug("Removing files")
+        if self.config_file_path:
+            self.config_file_path.unlink(missing_ok=True)
         system(f"rm -fr {self.namespace_path}")
 
     def __del__(self):

From 0dca97ce6ec3567b80b542a323eabcb45122a050 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 4 Jul 2022 19:26:29 +0200
Subject: [PATCH 320/990] Fix: VM id counter values too high could cause issues
 in IP ranges

---
 vm_supervisor/pool.py | 37 ++++++++++++++++++++++++++++++++++---
 1 file changed, 34 insertions(+), 3 deletions(-)

diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index dc2f07c09..e3b25e393 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -1,8 +1,9 @@
 import asyncio
 import logging
-from typing import Dict, Optional, List
+from typing import Dict, Optional, Iterable
 
 from aleph_message.models import ProgramContent, ProgramMessage
+
 from .conf import settings
 from .models import VmHash, VmExecution
 
@@ -33,10 +34,34 @@ async def create_a_vm(
         execution = VmExecution(vm_hash=vm_hash, program=program, original=original)
         self.executions[vm_hash] = execution
         await execution.prepare()
-        self.counter += 1
-        await execution.create(address=self.counter)
+        vm_id = self.get_unique_vm_id()
+        await execution.create(vm_id=vm_id)
         return execution
 
+    def get_unique_vm_id(self) -> int:
+        """Get a unique identifier for the VM.
+
+        This identifier is used to name the network interface and in the IPv4 range
+        dedicated to the VM.
+        """
+        self.counter += 1
+        if self.counter < 255**2:
+            # In common cases, use the counter itself as the vm_id. This makes it
+            # easier to debug.
+            return self.counter
+        else:
+            # The value of the counter is too high and some functions such as the
+            # IPv4 range dedicated to the VM do not support such high values.
+            #
+            # We therefore recycle vm_id values from executions that are not running
+            # anymore.
+            currently_used_vm_ids = set(execution.vm.vm_id
+                                        for execution in self.executions.values()
+                                        if execution.is_running)
+            for i in range(settings.START_ID_INDEX, 255**2):
+                if i not in currently_used_vm_ids:
+                    return i
+
     async def get_running_vm(self, vm_hash: VmHash) -> Optional[VmExecution]:
         """Return a running VM or None. Disables the VM expiration task."""
         execution = self.executions.get(vm_hash)
@@ -47,6 +72,12 @@ async def get_running_vm(self, vm_hash: VmHash) -> Optional[VmExecution]:
             return None
 
     def forget_vm(self, vm_hash: VmHash) -> None:
+        """Remove a VM from the executions pool.
+
+        Used after self.create_a_vm(...) raised an error in order to
+        completely forget about the execution and enforce a new execution
+        when attempted again.
+        """
         try:
             del self.executions[vm_hash]
         except KeyError:

From 2e3cdba185654ac3bf78c78757ea1f78c679a6c7 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 4 Jul 2022 22:45:37 +0200
Subject: [PATCH 321/990] Fix: VM unix socket were never closed

---
 firecracker/microvm.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index d7aec587b..86afc0965 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -4,6 +4,7 @@
 import os.path
 import string
 from asyncio import Task
+from asyncio.base_events import Server
 from os import getuid
 from pathlib import Path
 from pwd import getpwnam
@@ -71,6 +72,8 @@ class MicroVM:
     drives: List[Drive]
     init_timeout: float
 
+    _unix_socket: Server
+
     @property
     def namespace_path(self):
         firecracker_bin_name = os.path.basename(self.firecracker_bin_path)
@@ -359,7 +362,7 @@ async def wait_for_init(self):
         async def unix_client_connected(*_):
             await queue.put(True)
 
-        await asyncio.start_unix_server(
+        self._unix_socket = await asyncio.start_unix_server(
             unix_client_connected, path=f"{self.vsock_path}_52"
         )
         os.system(f"chown jailman:jailman {self.vsock_path}_52")
@@ -442,6 +445,11 @@ async def teardown(self):
                 f"iptables -D FORWARD -i {self.network_tap} -o {self.network_interface} -j ACCEPT"
             )
 
+        if self._unix_socket:
+            logger.debug("Closing unix socket")
+            self._unix_socket.close()
+            await self._unix_socket.wait_closed()
+
         logger.debug("Removing files")
         if self.config_file_path:
             self.config_file_path.unlink(missing_ok=True)

From 19d3472c4801ac0b4eb3590de385758f6496bf47 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 4 Jul 2022 22:43:19 +0200
Subject: [PATCH 322/990] Fix: Pubsub queues were not discarded after use

---
 vm_supervisor/pubsub.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/pubsub.py b/vm_supervisor/pubsub.py
index 3ef1d04a3..2d6b03a53 100644
--- a/vm_supervisor/pubsub.py
+++ b/vm_supervisor/pubsub.py
@@ -25,10 +25,25 @@ async def msubscribe(self, *keys):
         """Subscribe to multiple keys"""
         keys = tuple(key for key in keys if key is not None)
         logger.debug(f"msubscribe({keys})")
+
         queue = asyncio.Queue()
+
+        # Register the queue on all keys
         for key in keys:
             self.subscribers.setdefault(key, set()).add(queue)
-        return await queue.get()
+
+        # Wait for any subscription
+        await queue.get()
+
+        # Cleanup: remove the queue from the subscribers
+        for key in keys:
+            for subscriber in self.subscribers.values():
+                subscriber.discard(queue)
+                # Remove keys with no remaining queue
+                if not self.subscribers.get(key):
+                    self.subscribers.pop(key)
+
+        return
 
     async def publish(self, key, value):
         for queue in self.subscribers.get(key, tuple()):

From fd218b1e917977a6be553a6e8385e85affcc701e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 5 Jul 2022 17:54:29 +0200
Subject: [PATCH 323/990] Fix: VM id counter values too high could cause issues
 in IP ranges

---
 vm_supervisor/pubsub.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/pubsub.py b/vm_supervisor/pubsub.py
index 2d6b03a53..88460ac1a 100644
--- a/vm_supervisor/pubsub.py
+++ b/vm_supervisor/pubsub.py
@@ -19,7 +19,15 @@ def __init__(self):
     async def subscribe(self, key):
         queue = asyncio.Queue()
         self.subscribers.setdefault(key, set()).add(queue)
-        return await queue.get()
+        await queue.get()
+
+        # Cleanup: remove the queue from the subscribers
+        self.subscribers.get(key).discard(queue)
+        # Remove keys with no remaining queue
+        if not self.subscribers.get(key):
+            self.subscribers.pop(key)
+
+        return
 
     async def msubscribe(self, *keys):
         """Subscribe to multiple keys"""

From d6c3b9e01a2299d0475df29cfbab8e088eaaae68 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Jul 2022 00:12:34 +0200
Subject: [PATCH 324/990] Release 0.2.3

---
 doc/INSTALL-Debian-11.md    | 2 +-
 doc/INSTALL-Ubuntu-20.04.md | 2 +-
 vm_supervisor/README.md     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index f7a9c5113..4a723c7e3 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.1/aleph-vm.debian-11.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.3/aleph-vm.debian-11.deb
 apt install /opt/aleph-vm.debian-11.deb
 ```
 
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
index 46169c818..653968c25 100644
--- a/doc/INSTALL-Ubuntu-20.04.md
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.1/aleph-vm.ubuntu-20.04.deb
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.3/aleph-vm.ubuntu-20.04.deb
 sudo apt install /opt/aleph-vm.ubuntu-20.04.deb
 ```
 
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index 72b0362f7..ac06beccf 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -104,7 +104,7 @@ A more optimized kernel may be made available in the future.
 See section _Compile your kernel_ below to build your own.
 
 ```shell
-curl -fsSL -o /opt/firecracker/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/vmlinux.bin
+curl -fsSL -o /opt/firecracker/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.2.2/vmlinux.bin
 ```
 
 ## 3. Running

From 207cb529e32935dce15c7e1c2359c94a0f2acd4f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Jul 2022 00:22:44 +0200
Subject: [PATCH 325/990] Fix: Error in variable name

---
 vm_supervisor/models.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index e66fa1341..71f93e9e5 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -93,12 +93,12 @@ async def prepare(self):
         self.times.prepared_at = datetime.now()
         self.resources = resources
 
-    async def create(self, address: int) -> AlephFirecrackerVM:
+    async def create(self, vm_id: int) -> AlephFirecrackerVM:
         if not self.resources:
             raise ValueError("Execution resources must be configured first")
         self.times.starting_at = datetime.now()
         self.vm = vm = AlephFirecrackerVM(
-            vm_id=address,
+            vm_id=vm_id,
             vm_hash=self.vm_hash,
             resources=self.resources,
             enable_networking=self.program.environment.internet,

From 4ff685ca751179254aa5bd48eb28f25b240aabbf Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Jul 2022 14:33:23 +0100
Subject: [PATCH 326/990] Fix: Stability issues when checking crash handling in
 monitored endpoint

# Problem:
The endpoint '/status/check/fastapi' is monitored by multiple operators
and therefore called very frequently. The crash handling check results
in a new virtual machine being started at every call, and memory leaks
(likely pipes accumulating) can cause the supervisor to crash after
some time with 'Too many open files'.

# Solution:
In the short term, avoid calling the crash check in the monitored endpoint.
In the longer term, investigate and fix the accumulating file descriptors.
---
 vm_supervisor/views.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index 1c40427df..f065e7f06 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -141,7 +141,6 @@ async def status_check_fastapi(request: web.Request):
             "cache": await status.check_cache(session),
             "persistent_storage": await status.check_persistent_storage(session),
             "error_handling": await status.check_error_raised(session),
-            "crash_handling": await status.check_crash_and_restart(session),
         }
         return web.json_response(result, status=200 if all(result.values()) else 503)
 

From abc19314b9887b493ffe71a920f73b3f763dc604 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Jul 2022 15:13:59 +0100
Subject: [PATCH 327/990] Release 0.2.4

---
 doc/INSTALL-Debian-11.md    | 2 +-
 doc/INSTALL-Ubuntu-20.04.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index 4a723c7e3..4c3d80b32 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.3/aleph-vm.debian-11.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.4/aleph-vm.debian-11.deb
 apt install /opt/aleph-vm.debian-11.deb
 ```
 
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
index 653968c25..cfcfc8110 100644
--- a/doc/INSTALL-Ubuntu-20.04.md
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.3/aleph-vm.ubuntu-20.04.deb
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.4/aleph-vm.ubuntu-20.04.deb
 sudo apt install /opt/aleph-vm.ubuntu-20.04.deb
 ```
 

From 1dfc6d3d19a7848f8b1b08ea1a6dda98bd2d56e9 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 1 Aug 2022 21:12:25 +0200
Subject: [PATCH 328/990] CI: Building example volume using Docker failed

Due to a change in GitHub Actions
---
 .github/workflows/build-deb-package.yml | 21 +++++++++++++++++++--
 examples/volumes/Dockerfile             |  2 +-
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 0c1ebd364..000b51f6e 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -44,9 +44,26 @@ jobs:
           sudo apt update
           sudo apt install -y debootstrap
           cd runtimes/aleph-debian-11-python && sudo ./create_disk_image.sh && cd ../..
-          cd examples/volumes && sudo ./build_squashfs.sh && cd ../..
 
       - uses: actions/upload-artifact@v2
         with:
           name: aleph-debian-11-python.squashfs
-          path: runtimes/aleph-debian-11-python/rootfs.squashfs
\ No newline at end of file
+          path: runtimes/aleph-debian-11-python/rootfs.squashfs
+
+
+  build_example_venv_volume:
+    name: "Build example squashfs volume using Docker"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - run: |
+          docker build -t aleph-vm-build-squashfs -f examples/volumes/Dockerfile examples/volumes
+          docker run --rm -v "$(pwd)":/mnt aleph-vm-build-squashfs
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: example-volume-venv.squashfs
+          path: volume-venv.squashfs
diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile
index d8d72e780..f80fdea60 100644
--- a/examples/volumes/Dockerfile
+++ b/examples/volumes/Dockerfile
@@ -1,4 +1,4 @@
-FROM debian:buster
+FROM debian:bullseye
 
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     python3-venv \

From 0e086f4def008f31754745d9d893b61155bfa5c8 Mon Sep 17 00:00:00 2001
From: Reza Rahemtola 
Date: Tue, 2 Aug 2022 00:12:50 +0200
Subject: [PATCH 329/990] Fix: Examples doc typos

---
 examples/example_http_js/README.md   | 9 ++++-----
 examples/example_http_rust/README.md | 4 ++--
 tutorials/REQUIREMENTS.md            | 4 ++--
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/examples/example_http_js/README.md b/examples/example_http_js/README.md
index 144aa189b..8c04304f7 100644
--- a/examples/example_http_js/README.md
+++ b/examples/example_http_js/README.md
@@ -1,7 +1,6 @@
-# Aleph VM Rust Example
+# Aleph VM JS Example
 
-A simple example program written in Rust that 
-can run in an Aleph VM.
+A simple example program written in JS that can run in an Aleph VM.
 
 ## About
 
@@ -28,6 +27,6 @@ make podman-publish
 ### Using Docker
 
 ```shell
-make prepare
-make publish
+make docker-prepare
+make docker-publish
 ```
diff --git a/examples/example_http_rust/README.md b/examples/example_http_rust/README.md
index 144aa189b..3b9f325df 100644
--- a/examples/example_http_rust/README.md
+++ b/examples/example_http_rust/README.md
@@ -28,6 +28,6 @@ make podman-publish
 ### Using Docker
 
 ```shell
-make prepare
-make publish
+make docker-prepare
+make docker-publish
 ```
diff --git a/tutorials/REQUIREMENTS.md b/tutorials/REQUIREMENTS.md
index 1ddbdf6e7..905ddfbbd 100644
--- a/tutorials/REQUIREMENTS.md
+++ b/tutorials/REQUIREMENTS.md
@@ -47,7 +47,7 @@ Once the command is down, your virtual machine will be booted and ready!
 
 ### Set Vagrantfile configuration
 
-Open the vagrantfile and add following `config.vm.box``
+Open the vagrantfile and add following `config.vm.box`
 
 ```shell
 config.vm.network "forwarded_port", guest:8000, host:8000
@@ -78,7 +78,7 @@ aleph upload packages.squashfs
 ```
 
 ```shell
-ipfs add venv.squashfs
+ipfs add packages.squashfs
 ```
 | added QmWWX6BaaRkRSr2iNdwH5e29ACPg2nCHHXTRTfuBmVm3Ga venv.squashfs
 

From a722351b2a7b9d81ae63e68219cea1fe9b59a16c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 29 Jul 2022 10:32:50 +0200
Subject: [PATCH 330/990] Fix: Network was depending on system logs print

---
 vm_supervisor/vm/firecracker_microvm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index dc7c59d14..9bcd18f3f 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -387,7 +387,7 @@ async def configure(self):
         reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
         config = ConfigurationPayload(
             ip=self.fvm.guest_ip if self.enable_networking else None,
-            route=self.fvm.host_ip if self.enable_console else None,
+            route=self.fvm.host_ip if self.enable_networking else None,
             dns_servers=settings.DNS_NAMESERVERS,
             code=code,
             encoding=self.resources.code_encoding,

From f568134aa410e995ee94bb91984e0eb52e03805f Mon Sep 17 00:00:00 2001
From: Mehdi <75360886+mrhouzlane@users.noreply.github.com>
Date: Sun, 14 Aug 2022 23:57:06 +0200
Subject: [PATCH 331/990] Doc: Add macOS instructions in README.md

Changed to make a single command working for both Linux/macOs
---
 tutorials/README.md | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/tutorials/README.md b/tutorials/README.md
index 49505662f..cf1b87dc6 100644
--- a/tutorials/README.md
+++ b/tutorials/README.md
@@ -36,12 +36,25 @@ Some cryptographic functionalities of Aleph use curve secp256k1 and require inst
 Archiving programs and volumes requires
 [Squashfs user space tools](https://github.com/plougher/squashfs-tools).
 
-    sudo apt-get install -y python3-pip libsecp256k1-dev squashfs-tools
+- Linux : 
+``` 
+sudo apt-get install -y python3-pip libsecp256k1-dev squashfs-tools
+``` 
+
+- macOs : 
+``` 
+brew tap cuber/homebrew-libsecp256k1
+brew install libsecp256k1 squashfs
+```
 
 You will also need [Uvicorn](https://www.uvicorn.org/) for local testing 
 and the [Python Aleph client](https://github.com/aleph-im/aleph-client) for it's command-line tools:
 
-    pip3 install uvicorn[standard] aleph-client fastapi eth_account
+- Linux/ macOs :
+
+``` 
+pip3 install "uvicorn[standard]" aleph-client fastapi eth_account
+``` 
 
 ## 1. Understanding the VMs
 

From a46af2e96924c9e6c95723ba336252216827f6ec Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 29 Jun 2022 17:28:47 +0200
Subject: [PATCH 332/990] Fix: Watching for updates could be started multiple
 times

This would have been the source of many duplicate asyncio tasks.
---
 vm_supervisor/models.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 71f93e9e5..75070b7dc 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -55,6 +55,7 @@ class VmExecution:
     concurrent_runs: int
     runs_done_event: asyncio.Event
     expire_task: Optional[asyncio.Task] = None
+    update_task: Optional[asyncio.Task] = None
 
     @property
     def is_running(self):
@@ -147,6 +148,13 @@ def cancel_expiration(self) -> bool:
         else:
             return False
 
+    def cancel_update(self) -> bool:
+        if self.update_task:
+            self.update_task.cancel()
+            return True
+        else:
+            return False
+
     async def stop(self):
         if self.times.stopped_at is not None:
             logger.debug(f"VM={self.vm.vm_id} already stopped")
@@ -157,10 +165,12 @@ async def stop(self):
         await self.vm.teardown()
         self.times.stopped_at = datetime.now()
         self.cancel_expiration()
+        self.cancel_update()
 
     def start_watching_for_updates(self, pubsub: PubSub):
-        pool = asyncio.get_running_loop()
-        pool.create_task(self.watch_for_updates(pubsub=pubsub))
+        if not self.update_task:
+            loop = asyncio.get_running_loop()
+            self.update_task = loop.create_task(self.watch_for_updates(pubsub=pubsub))
 
     async def watch_for_updates(self, pubsub: PubSub):
         await pubsub.msubscribe(

From 796243021b6ba560a0799cb817766bfdca27df1c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 18 Aug 2022 12:12:16 +0200
Subject: [PATCH 333/990] Fix: Dockerfile relied on old version of Firecracker

---
 docker/vm_supervisor-dev.dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 5503869a4..42f9fdfcd 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -12,8 +12,8 @@ RUN useradd jailman
 
 RUN mkdir /opt/firecracker
 RUN chown $(whoami) /opt/firecracker
-RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.0.0/firecracker-v1.0.0-x86_64.tgz | tar -xz --directory /opt/firecracker
-RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.1.0/vmlinux.bin
+RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.1.1/firecracker-v1.1.1-x86_64.tgz | tar -xz --directory /opt/firecracker
+RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin
 
 # Link binaries on version-agnostic paths:
 RUN ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker

From ac23b55bdeb36e9f6c071155a16555dd5f1ded28 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 18 Aug 2022 12:11:43 +0200
Subject: [PATCH 334/990] Fix: Minor inconsistencies and improve logging

---
 vm_supervisor/messages.py               | 4 ++--
 vm_supervisor/storage.py                | 2 ++
 vm_supervisor/vm/firecracker_microvm.py | 2 +-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/vm_supervisor/messages.py b/vm_supervisor/messages.py
index b408685d5..68d0e3b27 100644
--- a/vm_supervisor/messages.py
+++ b/vm_supervisor/messages.py
@@ -18,7 +18,7 @@ async def try_get_message(ref: str) -> ProgramMessage:
         raise HTTPServiceUnavailable(reason="Aleph Connector unavailable")
     except ClientResponseError as error:
         if error.status == 404:
-            raise HTTPNotFound(reason="Hash not found", body=f"Hash not found: {ref}")
+            raise HTTPNotFound(reason="Hash not found", text=f"Hash not found: {ref}")
         else:
             raise
 
@@ -31,7 +31,7 @@ async def get_latest_ref(item_hash: str) -> str:
     except ClientResponseError as error:
         if error.status == 404:
             raise HTTPNotFound(
-                reason="Hash not found", body=f"Hash not found: {item_hash}"
+                reason="Hash not found", text=f"Hash not found: {item_hash}"
             )
         else:
             raise
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 0e3aeb07b..12c75f6ed 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -106,9 +106,11 @@ async def get_code_path(ref: str) -> FilePath:
             if os.path.exists(f"{archive_path}.squashfs"):
                 os.remove(f"{archive_path}.squashfs")
             os.system(f"mksquashfs {archive_path} {archive_path}.squashfs")
+            logger.debug(f"Squashfs generated on {archive_path}.squashfs")
             return FilePath(f"{archive_path}.squashfs")
         elif encoding == Encoding.zip:
             make_archive(archive_path, "zip", root_dir=archive_path)
+            logger.debug(f"Zip generated on {archive_path}.zip")
             return FilePath(f"{archive_path}.zip")
         else:
             raise ValueError(f"Unsupported encoding: {encoding}")
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 9bcd18f3f..c56d1e06f 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -148,7 +148,7 @@ async def download_code(self):
             self.code_path = await get_code_path(code_ref)
         except ClientResponseError as error:
             raise ResourceDownloadError(error)
-        assert isfile(self.code_path)
+        assert isfile(self.code_path), f"Code not found on '{self.code_path}'"
 
     async def download_runtime(self):
         runtime_ref: str = self.message_content.runtime.ref

From 7ada28ea001688f87e1125e82fb812406ce3c03e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 29 Jun 2022 16:39:31 +0200
Subject: [PATCH 335/990] Cleanup: Remove code duplication using
 `create_vm_execution`

---
 vm_supervisor/run.py | 33 ++-------------------------------
 1 file changed, 2 insertions(+), 31 deletions(-)

diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 285cf0d14..3280276ee 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -170,39 +170,10 @@ async def run_code_on_event(vm_hash: VmHash, event, pubsub: PubSub):
     Execute code in response to an event.
     """
 
-    try:
-        execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
-    except Exception as error:
-        logger.exception(error)
-        raise
+    execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
-        message, original_message = await load_updated_message(vm_hash)
-        pool.message_cache[vm_hash] = message
-
-        try:
-            execution = await pool.create_a_vm(
-                vm_hash=vm_hash,
-                program=message.content,
-                original=original_message.content,
-            )
-        except ResourceDownloadError as error:
-            logger.exception(error)
-            pool.forget_vm(vm_hash=vm_hash)
-            raise HTTPBadRequest(reason="Code, runtime or data not available")
-        except FileTooLargeError as error:
-            raise HTTPInternalServerError(reason=error.args[0])
-        except VmSetupError as error:
-            logger.exception(error)
-            pool.forget_vm(vm_hash=vm_hash)
-            raise HTTPInternalServerError(reason="Error during program initialisation")
-        except MicroVMFailedInit as error:
-            logger.exception(error)
-            pool.forget_vm(vm_hash=vm_hash)
-            raise HTTPInternalServerError(reason="Error during runtime initialisation")
-
-    if not execution.vm:
-        raise ValueError("The VM has not been created")
+        execution = await create_vm_execution(vm_hash=vm_hash)
 
     logger.debug(f"Using vm={execution.vm.vm_id}")
 

From e3a45d8c1f97f73138d7ccf4a892d8622876a427 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 6 Sep 2022 14:05:02 +0200
Subject: [PATCH 336/990] Internal: FilePath type can be replaced with
 pathlib.Path

The custom type did not bring any value to the codebase.
---
 firecracker/config.py                   |  6 ++--
 firecracker/microvm.py                  |  3 +-
 firecracker/models.py                   |  3 --
 vm_supervisor/conf.py                   | 36 ++++++++++-----------
 vm_supervisor/storage.py                | 42 ++++++++++++-------------
 vm_supervisor/vm/firecracker_microvm.py | 20 ++++++------
 6 files changed, 51 insertions(+), 59 deletions(-)
 delete mode 100644 firecracker/models.py

diff --git a/firecracker/config.py b/firecracker/config.py
index 01acb4498..94154595f 100644
--- a/firecracker/config.py
+++ b/firecracker/config.py
@@ -1,13 +1,13 @@
+from pathlib import Path
 from typing import List, Optional
 
-from firecracker.models import FilePath
 from pydantic import BaseModel, PositiveInt
 
 VSOCK_PATH = "/tmp/v.sock"
 
 
 class BootSource(BaseModel):
-    kernel_image_path: FilePath = FilePath("vmlinux.bin")
+    kernel_image_path: Path = Path("vmlinux.bin")
     boot_args: str = (
         "console=ttyS0 reboot=k panic=1 pci=off "
         "ro noapic nomodules random.trust_cpu=on"
@@ -24,7 +24,7 @@ def args(enable_console: bool = True):
 
 class Drive(BaseModel):
     drive_id: str = "rootfs"
-    path_on_host: FilePath = FilePath("./runtimes/aleph-alpine-3.13-python/rootfs.ext4")
+    path_on_host: Path = Path("./runtimes/aleph-alpine-3.13-python/rootfs.ext4")
     is_root_device: bool = True
     is_read_only: bool = True
 
diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 86afc0965..f101d50ec 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -12,7 +12,6 @@
 from typing import Optional, Tuple, List
 
 from .config import FirecrackerConfig
-from .models import FilePath
 from .config import Drive
 
 logger = logging.getLogger(__name__)
@@ -297,7 +296,7 @@ def enable_drive(self, drive_path: str, read_only: bool = True) -> Drive:
 
         drive = Drive(
             drive_id=device_name,
-            path_on_host=FilePath(drive_path),
+            path_on_host=Path(drive_path),
             is_root_device=False,
             is_read_only=read_only,
         )
diff --git a/firecracker/models.py b/firecracker/models.py
deleted file mode 100644
index b79743982..000000000
--- a/firecracker/models.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from typing import NewType
-
-FilePath = NewType("FilePath", str)
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 219de32e8..f26b22546 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -3,10 +3,10 @@
 import re
 from enum import Enum
 from os.path import isfile, join, exists, abspath, isdir
+from pathlib import Path
 from subprocess import check_output
-from typing import NewType, Optional, List, Dict, Any
+from typing import NewType, Optional, List
 
-from firecracker.models import FilePath
 from pydantic import BaseSettings, Field
 
 logger = logging.getLogger(__name__)
@@ -89,39 +89,37 @@ class Settings(BaseSettings):
 
     CONNECTOR_URL = Url("http://localhost:4021")
 
-    CACHE_ROOT = FilePath("/var/cache/aleph/vm")
-    MESSAGE_CACHE = FilePath(join(CACHE_ROOT, "message"))
-    CODE_CACHE = FilePath(join(CACHE_ROOT, "code"))
-    RUNTIME_CACHE = FilePath(join(CACHE_ROOT, "runtime"))
-    DATA_CACHE = FilePath(join(CACHE_ROOT, "data"))
+    CACHE_ROOT = Path("/var/cache/aleph/vm")
+    MESSAGE_CACHE = CACHE_ROOT / "message"
+    CODE_CACHE = CACHE_ROOT / "code"
+    RUNTIME_CACHE = CACHE_ROOT / "runtime"
+    DATA_CACHE = CACHE_ROOT / "data"
 
-    EXECUTION_ROOT = FilePath("/var/lib/aleph/vm")
-    EXECUTION_DATABASE = FilePath(join(EXECUTION_ROOT, "executions.sqlite3"))
+    EXECUTION_ROOT = Path("/var/lib/aleph/vm")
+    EXECUTION_DATABASE = EXECUTION_ROOT / "executions.sqlite3"
     EXECUTION_LOG_ENABLED = False
-    EXECUTION_LOG_DIRECTORY = FilePath(join(EXECUTION_ROOT, "executions"))
+    EXECUTION_LOG_DIRECTORY = EXECUTION_ROOT / "executions"
 
-    PERSISTENT_VOLUMES_DIR = FilePath(
-        join("/var/lib/aleph/vm", "volumes", "persistent")
-    )
+    PERSISTENT_VOLUMES_DIR = EXECUTION_ROOT / "volumes" / "persistent"
 
     MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000  # 10 MB
     MAX_DATA_ARCHIVE_SIZE = 10_000_000  # 10 MB
 
-    FAKE_DATA_PROGRAM: Optional[FilePath] = None
-    BENCHMARK_FAKE_DATA_PROGRAM = FilePath(
+    FAKE_DATA_PROGRAM: Optional[Path] = None
+    BENCHMARK_FAKE_DATA_PROGRAM = Path(
         abspath(join(__file__, "../../examples/example_fastapi"))
     )
 
-    FAKE_DATA_MESSAGE = FilePath(
+    FAKE_DATA_MESSAGE = Path(
         abspath(join(__file__, "../../examples/message_from_aleph.json"))
     )
-    FAKE_DATA_DATA: Optional[FilePath] = FilePath(
+    FAKE_DATA_DATA: Optional[Path] = Path(
         abspath(join(__file__, "../../examples/data/"))
     )
-    FAKE_DATA_RUNTIME = FilePath(
+    FAKE_DATA_RUNTIME = Path(
         abspath(join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs"))
     )
-    FAKE_DATA_VOLUME: Optional[FilePath] = FilePath(
+    FAKE_DATA_VOLUME: Optional[Path] = Path(
         abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs"))
     )
 
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 12c75f6ed..8a71dfb6f 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -5,17 +5,17 @@
 In the future, it should connect to an Aleph node and retrieve the code from there.
 """
 import asyncio
-import json
 import hashlib
+import json
 import logging
 import os
 import re
 import sys
-from os.path import isfile, join, dirname
+from os.path import isfile, join
+from pathlib import Path
 from shutil import make_archive
 
 import aiohttp
-
 from aleph_message.models import ProgramMessage
 from aleph_message.models.program import (
     Encoding,
@@ -24,13 +24,13 @@
     PersistentVolume,
     VolumePersistence,
 )
+
 from .conf import settings
-from firecracker.models import FilePath
 
 logger = logging.getLogger(__name__)
 
 
-async def download_file(url: str, local_path: FilePath) -> None:
+async def download_file(url: str, local_path: Path) -> None:
     # TODO: Limit max size of download to the message specification
     if isfile(local_path):
         logger.debug(f"File already exists: {local_path}")
@@ -81,7 +81,7 @@ async def get_message(ref: str) -> ProgramMessage:
     if settings.FAKE_DATA_PROGRAM:
         cache_path = settings.FAKE_DATA_MESSAGE
     else:
-        cache_path = FilePath(join(settings.MESSAGE_CACHE, ref) + ".json")
+        cache_path = Path(join(settings.MESSAGE_CACHE, ref) + ".json")
         url = f"{settings.CONNECTOR_URL}/download/message/{ref}"
         await download_file(url, cache_path)
 
@@ -95,7 +95,7 @@ async def get_message(ref: str) -> ProgramMessage:
         return ProgramMessage(**msg)
 
 
-async def get_code_path(ref: str) -> FilePath:
+async def get_code_path(ref: str) -> Path:
     if settings.FAKE_DATA_PROGRAM:
         archive_path = settings.FAKE_DATA_PROGRAM
 
@@ -107,43 +107,43 @@ async def get_code_path(ref: str) -> FilePath:
                 os.remove(f"{archive_path}.squashfs")
             os.system(f"mksquashfs {archive_path} {archive_path}.squashfs")
             logger.debug(f"Squashfs generated on {archive_path}.squashfs")
-            return FilePath(f"{archive_path}.squashfs")
+            return Path(f"{archive_path}.squashfs")
         elif encoding == Encoding.zip:
             make_archive(archive_path, "zip", root_dir=archive_path)
             logger.debug(f"Zip generated on {archive_path}.zip")
-            return FilePath(f"{archive_path}.zip")
+            return Path(f"{archive_path}.zip")
         else:
             raise ValueError(f"Unsupported encoding: {encoding}")
 
-    cache_path = FilePath(join(settings.CODE_CACHE, ref))
+    cache_path = Path(join(settings.CODE_CACHE, ref))
     url = f"{settings.CONNECTOR_URL}/download/code/{ref}"
     await download_file(url, cache_path)
     return cache_path
 
 
-async def get_data_path(ref: str) -> FilePath:
+async def get_data_path(ref: str) -> Path:
     if settings.FAKE_DATA_PROGRAM and settings.FAKE_DATA_DATA:
         data_dir = settings.FAKE_DATA_DATA
         make_archive(data_dir, "zip", data_dir)
-        return FilePath(f"{data_dir}.zip")
+        return Path(f"{data_dir}.zip")
 
-    cache_path = FilePath(join(settings.DATA_CACHE, ref))
+    cache_path = Path(join(settings.DATA_CACHE, ref))
     url = f"{settings.CONNECTOR_URL}/download/data/{ref}"
     await download_file(url, cache_path)
     return cache_path
 
 
-async def get_runtime_path(ref: str) -> FilePath:
+async def get_runtime_path(ref: str) -> Path:
     if settings.FAKE_DATA_PROGRAM:
-        return FilePath(settings.FAKE_DATA_RUNTIME)
+        return Path(settings.FAKE_DATA_RUNTIME)
 
-    cache_path = FilePath(join(settings.RUNTIME_CACHE, ref))
+    cache_path = Path(join(settings.RUNTIME_CACHE, ref))
     url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}"
     await download_file(url, cache_path)
     return cache_path
 
 
-def create_ext4(path: FilePath, size_mib: int) -> bool:
+def create_ext4(path: Path, size_mib: int) -> bool:
     if os.path.isfile(path):
         return False
     tmp_path = f"{path}.tmp"
@@ -155,13 +155,13 @@ def create_ext4(path: FilePath, size_mib: int) -> bool:
     return True
 
 
-async def get_volume_path(volume: MachineVolume, namespace: str) -> FilePath:
+async def get_volume_path(volume: MachineVolume, namespace: str) -> Path:
     if isinstance(volume, ImmutableVolume):
         ref = volume.ref
         if settings.FAKE_DATA_PROGRAM and settings.FAKE_DATA_VOLUME:
-            return FilePath(settings.FAKE_DATA_VOLUME)
+            return Path(settings.FAKE_DATA_VOLUME)
 
-        cache_path = FilePath(join(settings.DATA_CACHE, ref))
+        cache_path = Path(join(settings.DATA_CACHE, ref))
         url = f"{settings.CONNECTOR_URL}/download/data/{ref}"
         await download_file(url, cache_path)
         return cache_path
@@ -171,7 +171,7 @@ async def get_volume_path(volume: MachineVolume, namespace: str) -> FilePath:
         if not re.match(r"^[\w\-_/]+$", volume.name):
             raise ValueError(f"Invalid value for volume name: {volume.name}")
         os.makedirs(join(settings.PERSISTENT_VOLUMES_DIR, namespace), exist_ok=True)
-        volume_path = FilePath(
+        volume_path = Path(
             join(settings.PERSISTENT_VOLUMES_DIR, namespace, f"{volume.name}.ext4")
         )
         await asyncio.get_event_loop().run_in_executor(
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index c56d1e06f..525e7ea74 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -8,6 +8,7 @@
 from multiprocessing import Process, set_start_method
 from os.path import isfile, exists
 from typing import Optional, Dict, List
+from pathlib import Path
 
 import msgpack
 
@@ -28,7 +29,6 @@
     NetworkInterface,
 )
 from firecracker.microvm import MicroVM, setfacl
-from firecracker.models import FilePath
 from guest_api.__main__ import run_guest_api
 from ..conf import settings
 from ..storage import get_code_path, get_runtime_path, get_data_path, get_volume_path
@@ -37,7 +37,7 @@
 set_start_method("spawn")
 
 
-def load_file_content(path: FilePath) -> bytes:
+def load_file_content(path: Path) -> bytes:
     if path:
         with open(path, "rb") as fd:
             return fd.read()
@@ -118,14 +118,14 @@ class AlephFirecrackerResources:
 
     message_content: ProgramContent
 
-    kernel_image_path: FilePath
-    code_path: FilePath
+    kernel_image_path: Path
+    code_path: Path
     code_encoding: Encoding
     code_entrypoint: str
-    rootfs_path: FilePath
+    rootfs_path: Path
     volumes: List[HostVolume]
-    volume_paths: Dict[str, FilePath]
-    data_path: Optional[FilePath]
+    volume_paths: Dict[str, Path]
+    data_path: Optional[Path]
     namespace: str
 
     def __init__(self, message_content: ProgramContent, namespace: str):
@@ -271,7 +271,7 @@ async def setup(self):
 
         config = FirecrackerConfig(
             boot_source=BootSource(
-                kernel_image_path=FilePath(
+                kernel_image_path=Path(
                     fvm.enable_kernel(self.resources.kernel_image_path)
                 ),
                 boot_args=BootSource.args(enable_console=self.enable_console),
@@ -279,9 +279,7 @@ async def setup(self):
             drives=[
                 Drive(
                     drive_id="rootfs",
-                    path_on_host=FilePath(
-                        fvm.enable_rootfs(self.resources.rootfs_path)
-                    ),
+                    path_on_host=Path(fvm.enable_rootfs(self.resources.rootfs_path)),
                     is_root_device=True,
                     is_read_only=True,
                 ),

From 02abd716357eb0f3c67f90f25a005993889693bd Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 9 Sep 2022 10:30:47 +0200
Subject: [PATCH 337/990] Fix: Internal PubSub `msubscribe` failed

Mutating the object being iterated caused inconsistency when iterating over subscribers.
---
 vm_supervisor/pubsub.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/vm_supervisor/pubsub.py b/vm_supervisor/pubsub.py
index 88460ac1a..7ef36056f 100644
--- a/vm_supervisor/pubsub.py
+++ b/vm_supervisor/pubsub.py
@@ -11,7 +11,7 @@
 
 
 class PubSub:
-    subscribers: Dict[Hashable, Set[asyncio.Queue]]
+    subscribers: Dict[Hashable, Set[asyncio.Queue[set]]]
 
     def __init__(self):
         self.subscribers = {}
@@ -45,12 +45,11 @@ async def msubscribe(self, *keys):
 
         # Cleanup: remove the queue from the subscribers
         for key in keys:
-            for subscriber in self.subscribers.values():
+            for subscriber in list(self.subscribers.values()):
                 subscriber.discard(queue)
-                # Remove keys with no remaining queue
-                if not self.subscribers.get(key):
+                # Remove keys with no remaining queue (empty set remaining)
+                if self.subscribers.get(key) == set():
                     self.subscribers.pop(key)
-
         return
 
     async def publish(self, key, value):

From 75425cb64b10e3566d702037511fd00f586a07e1 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 1 Jun 2022 19:05:29 +0200
Subject: [PATCH 338/990] Feature: System resources were not exposed

The scheduling of persistent VMs requires external services to fetch
the available system resources of the host.

Solution: Add a new HTTP endpoint on `/about/usage/system`
that exposes system resources and system properties of
the host machine.
---
 .github/workflows/test-on-droplet.yml |   1 +
 docker/vm_supervisor-dev.dockerfile   |   2 +-
 packaging/aleph-vm/DEBIAN/control     |   2 +-
 vm_supervisor/resources.py            | 119 ++++++++++++++++++++++++++
 vm_supervisor/supervisor.py           |   2 +
 5 files changed, 124 insertions(+), 2 deletions(-)
 create mode 100644 vm_supervisor/resources.py

diff --git a/.github/workflows/test-on-droplet.yml b/.github/workflows/test-on-droplet.yml
index 04039fe1b..0d4e6a656 100644
--- a/.github/workflows/test-on-droplet.yml
+++ b/.github/workflows/test-on-droplet.yml
@@ -65,6 +65,7 @@ jobs:
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           
           sleep 3
+          curl --retry 5 "http://${DROPLET_IPV4}:4020/about/usage/system"
           curl --retry 5 "http://${DROPLET_IPV4}:4020/status/check/fastapi"
 
       - name: Cleanup
diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 42f9fdfcd..e4be2a238 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -5,7 +5,7 @@ FROM debian:bullseye
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     sudo acl curl squashfs-tools git \
     python3 python3-aiohttp python3-msgpack python3-pip python3-aiodns python3-aioredis  \
-    python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging \
+    python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging python3-cpuinfo \
     && rm -rf /var/lib/apt/lists/*
 
 RUN useradd jailman
diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index 0b213d9d9..1bbd7d5bb 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo
 Section: aleph-im
 Priority: Extra
diff --git a/vm_supervisor/resources.py b/vm_supervisor/resources.py
new file mode 100644
index 000000000..b2fd8968f
--- /dev/null
+++ b/vm_supervisor/resources.py
@@ -0,0 +1,119 @@
+from datetime import datetime, timezone
+from functools import lru_cache
+from typing import Tuple
+
+import cpuinfo
+import psutil
+from aiohttp import web
+from aleph_message.models.program import CpuProperties
+from pydantic import BaseModel
+
+from .conf import settings
+
+
+class Period(BaseModel):
+    datetime: datetime
+
+
+class LoadAverage(BaseModel):
+    load1: float
+    load5: float
+    load15: float
+
+    @classmethod
+    def from_psutil(cls, psutil_loadavg: Tuple[float, float, float]):
+        return cls(
+            load1=psutil_loadavg[0],
+            load5=psutil_loadavg[1],
+            load15=psutil_loadavg[2],
+        )
+
+
+class CoreFrequencies(BaseModel):
+    min: float
+    max: float
+
+    @classmethod
+    def from_psutil(cls, psutil_freq: psutil._common.scpufreq):
+        min = psutil_freq.min or psutil_freq.current
+        max = psutil_freq.max or psutil_freq.current
+        return cls(min=min, max=max)
+
+
+class CpuUsage(BaseModel):
+    count: int
+    load_average: LoadAverage
+    core_frequencies: CoreFrequencies
+
+
+class MemoryUsage(BaseModel):
+    total_kB: int
+    available_kB: int
+
+
+class DiskUsage(BaseModel):
+    total_kB: int
+    available_kB: int
+
+
+class UsagePeriod(BaseModel):
+    start_timestamp: datetime
+    duration_seconds: float
+
+
+class MachineProperties(BaseModel):
+    cpu: CpuProperties
+
+
+class MachineUsage(BaseModel):
+    cpu: CpuUsage
+    mem: MemoryUsage
+    disk: DiskUsage
+    period: UsagePeriod
+    properties: MachineProperties
+    active: bool = True
+
+
+@lru_cache
+def get_machine_properties() -> MachineProperties:
+    """Fetch machine properties such as architecture, CPU vendor, ...
+    These should not change while the supervisor is running.
+
+    In the future, some properties may have to be fetched from within a VM.
+    """
+    cpu_info = cpuinfo.get_cpu_info()  # Slow
+    return MachineProperties(
+        cpu=CpuProperties(
+            architecture=cpu_info["raw_arch_string"],
+            vendor=cpu_info["vendor_id"],
+        ),
+    )
+
+
+async def about_system_usage(request: web.Request):
+    period_start = datetime.now(timezone.utc).replace(second=0, microsecond=0)
+
+    usage: MachineUsage = MachineUsage(
+        cpu=CpuUsage(
+            count=psutil.cpu_count(),
+            load_average=LoadAverage.from_psutil(psutil.getloadavg()),
+            core_frequencies=CoreFrequencies.from_psutil(psutil.cpu_freq()),
+        ),
+        mem=MemoryUsage(
+            total_kB=psutil.virtual_memory().total / 1000,
+            available_kB=psutil.virtual_memory().available / 1000,
+        ),
+        disk=DiskUsage(
+            total_kB=psutil.disk_usage(settings.PERSISTENT_VOLUMES_DIR).total // 1000,
+            available_kB=psutil.disk_usage(settings.PERSISTENT_VOLUMES_DIR).free
+            // 1000,
+        ),
+        period=UsagePeriod(
+            start_timestamp=period_start,
+            duration_seconds=60,
+        ),
+        properties=get_machine_properties(),
+    )
+    return web.json_response(
+        text=usage.json(exclude_none=True),
+    )
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 49a0708c5..b70d891ae 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -14,6 +14,7 @@
 from . import __version__
 from . import metrics
 from .conf import settings
+from .resources import about_system_usage
 from .run import pool
 from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task
 from .views import (
@@ -49,6 +50,7 @@ async def server_version_middleware(
         web.get("/about/login", about_login),
         web.get("/about/executions", about_executions),
         web.get("/about/executions/records", about_execution_records),
+        web.get("/about/usage/system", about_system_usage),
         web.get("/about/config", about_config),
         web.get("/status/check/fastapi", status_check_fastapi),
         web.get("/status/check/version", status_check_version),

From 61fa49c6871c727c245fd196ce7287f3b874d845 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 15 Sep 2022 17:25:32 +0200
Subject: [PATCH 339/990] Fix: Typo prevented stopping service on upgrade

---
 packaging/aleph-vm/DEBIAN/preinst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/aleph-vm/DEBIAN/preinst b/packaging/aleph-vm/DEBIAN/preinst
index 5dd10fac4..c76d9f6a4 100755
--- a/packaging/aleph-vm/DEBIAN/preinst
+++ b/packaging/aleph-vm/DEBIAN/preinst
@@ -5,7 +5,7 @@ set -uf -o pipefail
 if ! [[ -v container ]]; then
   # Stop the service during an upgrade.
   # The service does not exist during a new install and will fail, this is okay
-  systemctl stop aleph-vm-supervisorz.service
+  systemctl stop aleph-vm-supervisor.service
 fi
 
 set -e

From 8d5d1d7d58434b511a2f9f332f0af3907382d70d Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 9 Sep 2022 10:24:08 +0200
Subject: [PATCH 340/990] Debug: Operators could not see asyncio debug messages

The traceback for these exceptions did not appear anywhere.

Solution: Introduce a new option that allows developers and operators to enable
asyncio debug mode.

Add helpers to run coroutines in tasks with exception catching and logging
and helps with spotting errors in coroutines.

Fixup
---
 vm_supervisor/__main__.py | 15 ++++++++++++++-
 vm_supervisor/conf.py     |  1 +
 vm_supervisor/models.py   |  9 ++++-----
 vm_supervisor/reactor.py  |  9 ++++-----
 vm_supervisor/tasks.py    |  3 ++-
 vm_supervisor/utils.py    | 20 +++++++++++++++++++-
 6 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index 218d00633..a8ae23efa 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -72,6 +72,14 @@ def parse_args(args):
         action="store_const",
         const=logging.DEBUG,
     )
+    parser.add_argument(
+        "-d",
+        "--debug-asyncio",
+        dest="debug_asyncio",
+        help="Enable asyncio debugging",
+        action="store_true",
+        default=settings.DEBUG_ASYNCIO,
+    )
     parser.add_argument(
         "-p",
         "--print-settings",
@@ -214,6 +222,7 @@ def main():
         PREALLOC_VM_COUNT=args.prealloc_vm_count,
         ALLOW_VM_NETWORKING=args.allow_vm_networking,
         FAKE_DATA_PROGRAM=args.fake_data_program,
+        DEBUG_ASYNCIO=args.debug_asyncio,
     )
 
     if sentry_sdk:
@@ -240,8 +249,12 @@ def main():
 
     settings.check()
 
+    loop = asyncio.get_event_loop()
+
+    if args.debug_asyncio:
+        loop.set_debug(True)
+
     if args.benchmark > 0:
-        loop = asyncio.get_event_loop()
         loop.run_until_complete(benchmark(runs=args.benchmark))
         print("Finished")
     elif args.do_not_run:
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index f26b22546..168fb88c4 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -80,6 +80,7 @@ class Settings(BaseSettings):
     USE_JAILER = True
     # System logs make boot ~2x slower
     PRINT_SYSTEM_LOGS = False
+    DEBUG_ASYNCIO = False
     # Networking does not work inside Docker/Podman
     ALLOW_VM_NETWORKING = True
     FIRECRACKER_PATH = "/opt/firecracker/firecracker"
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 75070b7dc..3e8365945 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -11,7 +11,7 @@
 
 from .metrics import save_record, save_execution_data, ExecutionRecord
 from .pubsub import PubSub
-from .utils import dumps_for_json
+from .utils import dumps_for_json, create_task_log_exceptions
 from .vm import AlephFirecrackerVM
 from .vm.firecracker_microvm import AlephFirecrackerResources
 from .conf import settings
@@ -126,11 +126,11 @@ def stop_after_timeout(self, timeout: float = 5.0) -> Task:
         if sys.version_info.major >= 3 and sys.version_info.minor >= 8:
             # Task can be named
             vm_id: str = str(self.vm.vm_id if self.vm else None)
-            self.expire_task = loop.create_task(
+            self.expire_task = create_task_log_exceptions(
                 self.expire(timeout), name=f"expire {vm_id}"
             )
         else:
-            self.expire_task = loop.create_task(self.expire(timeout))
+            self.expire_task = create_task_log_exceptions(self.expire(timeout))
         return self.expire_task
 
     async def expire(self, timeout: float) -> None:
@@ -169,8 +169,7 @@ async def stop(self):
 
     def start_watching_for_updates(self, pubsub: PubSub):
         if not self.update_task:
-            loop = asyncio.get_running_loop()
-            self.update_task = loop.create_task(self.watch_for_updates(pubsub=pubsub))
+            self.update_task = create_task_log_exceptions(self.watch_for_updates(pubsub=pubsub))
 
     async def watch_for_updates(self, pubsub: PubSub):
         await pubsub.msubscribe(
diff --git a/vm_supervisor/reactor.py b/vm_supervisor/reactor.py
index db604a2d6..00d66a143 100644
--- a/vm_supervisor/reactor.py
+++ b/vm_supervisor/reactor.py
@@ -1,12 +1,12 @@
-import asyncio
 import logging
-from typing import List, Dict, Coroutine
+from typing import List, Coroutine
 
+from aleph_message.models import Message, ProgramMessage
 from aleph_message.models.program import Subscription
 
-from aleph_message.models import Message, ProgramMessage
 from vm_supervisor.pubsub import PubSub
 from vm_supervisor.run import run_code_on_event
+from vm_supervisor.utils import create_task_log_exceptions
 
 logger = logging.getLogger(__name__)
 
@@ -64,9 +64,8 @@ async def trigger(self, message: Message):
                     break
 
         # Call all listeners asynchronously from the event loop:
-        loop = asyncio.get_event_loop()
         for coroutine in coroutines:
-            loop.create_task(coroutine)
+            create_task_log_exceptions(coroutine)
 
     def register(self, message: ProgramMessage):
         if message.content.on.message:
diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py
index b2b59b365..803885559 100644
--- a/vm_supervisor/tasks.py
+++ b/vm_supervisor/tasks.py
@@ -17,6 +17,7 @@
 from .models import VmHash
 from .pubsub import PubSub
 from .reactor import Reactor
+from .utils import create_task_log_exceptions
 
 logger = logging.getLogger(__name__)
 
@@ -123,7 +124,7 @@ async def start_watch_for_messages_task(app: web.Application):
 
     app["pubsub"] = pubsub
     app["reactor"] = reactor
-    app["messages_listener"] = asyncio.create_task(watch_for_messages(pubsub, reactor))
+    app["messages_listener"] = create_task_log_exceptions(watch_for_messages(pubsub, reactor))
 
 
 async def stop_watch_for_messages_task(app: web.Application):
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index c128b8369..10a4f2051 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -1,10 +1,14 @@
+import asyncio
 import json
+import logging
 from base64 import b32decode, b16encode
 from dataclasses import is_dataclass, asdict as dataclass_as_dict
-from typing import Any, Optional
+from typing import Any, Optional, Coroutine, Dict
 
 import aiodns
 
+logger = logging.getLogger(__name__)
+
 
 def b32_to_b16(hash: str) -> bytes:
     """Convert base32 encoded bytes to base16 encoded bytes."""
@@ -33,3 +37,17 @@ def to_json(o: Any):
 
 def dumps_for_json(o: Any, indent: Optional[int] = None):
     return json.dumps(o, default=to_json, indent=indent)
+
+
+async def run_and_log_exception(coro: Coroutine):
+    """Exceptions in coroutines may go unnoticed if they are not handled."""
+    try:
+        return await coro
+    except Exception as error:
+        logger.exception(error)
+        raise
+
+
+def create_task_log_exceptions(coro: Coroutine, *, name=None):
+    """Ensure that exceptions running in coroutines are logged."""
+    return asyncio.create_task(run_and_log_exception(coro), name=name)

From 893be83ff84f13a94fb9cd185e280ae8dcd23db5 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 5 Oct 2022 23:52:49 +0200
Subject: [PATCH 341/990] Cleanup: Code cleanup (#240)

Clean-up multiple warnings without affecting the behaviour.
---
 firecracker/microvm.py                  | 19 +++++--
 guest_api/__main__.py                   |  3 +-
 packaging/Makefile                      |  2 +-
 vm_supervisor/__init__.py               | 73 +++++++++++++------------
 vm_supervisor/__main__.py               |  2 +-
 vm_supervisor/conf.py                   |  4 +-
 vm_supervisor/messages.py               |  2 +-
 vm_supervisor/models.py                 | 11 +++-
 vm_supervisor/pool.py                   | 12 ++--
 vm_supervisor/reactor.py                |  6 +-
 vm_supervisor/run.py                    | 17 +++---
 vm_supervisor/status.py                 |  6 +-
 vm_supervisor/storage.py                |  4 +-
 vm_supervisor/supervisor.py             |  5 +-
 vm_supervisor/tasks.py                  |  4 +-
 vm_supervisor/version.py                | 34 ++++++++++++
 vm_supervisor/views.py                  |  7 ++-
 vm_supervisor/vm/__init__.py            |  3 +-
 vm_supervisor/vm/firecracker_microvm.py |  6 +-
 19 files changed, 142 insertions(+), 78 deletions(-)
 create mode 100644 vm_supervisor/version.py

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index f101d50ec..8b79899c1 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -19,6 +19,7 @@
 VSOCK_PATH = "/tmp/v.sock"
 JAILER_BASE_DIRECTORY = "/var/lib/aleph/vm/jailer"
 
+
 class MicroVMFailedInit(Exception):
     pass
 
@@ -376,8 +377,14 @@ async def shutdown(self):
         logger.debug(f"Shutdown vm={self.vm_id}")
         try:
             reader, writer = await asyncio.open_unix_connection(path=self.vsock_path)
-        except (FileNotFoundError, ConnectionResetError, ConnectionRefusedError) as error:
-            logger.warning(f"VM={self.vm_id} cannot receive shutdown signal: {error.args}")
+        except (
+            FileNotFoundError,
+            ConnectionResetError,
+            ConnectionRefusedError,
+        ) as error:
+            logger.warning(
+                f"VM={self.vm_id} cannot receive shutdown signal: {error.args}"
+            )
             return
 
         try:
@@ -390,15 +397,17 @@ async def shutdown(self):
             logger.debug(f"ack={ack.decode()}")
 
             msg: bytes = await reader.readline()
-            logger.debug(f"msg={msg}")
+            logger.debug(f"msg={msg!r}")
 
             msg2: bytes = await reader.readline()
-            logger.debug(f"msg2={msg2}")
+            logger.debug(f"msg2={msg2!r}")
 
             if msg2 != b"STOPZ\n":
                 logger.warning(f"Unexpected response from VM: {msg2[:20]}")
         except ConnectionResetError as error:
-            logger.warning(f"ConnectionResetError in shutdown of {self.vm_id}: {error.args}")
+            logger.warning(
+                f"ConnectionResetError in shutdown of {self.vm_id}: {error.args}"
+            )
 
     async def stop(self):
         if self.proc:
diff --git a/guest_api/__main__.py b/guest_api/__main__.py
index b2a2f2f62..11249f0b7 100644
--- a/guest_api/__main__.py
+++ b/guest_api/__main__.py
@@ -71,6 +71,7 @@ async def repost(request: web.Request):
 
 async def properties(request: web.Request):
     logger.debug("Forwarding signing properties")
+    _ = request
 
     url = f"{ALEPH_VM_CONNECTOR}/properties"
     async with aiohttp.ClientSession() as session:
@@ -150,7 +151,7 @@ async def list_keys_from_cache(request: web.Request):
 
     redis: aioredis.Redis = await get_redis()
     result = await redis.keys(f"{prefix}:{pattern}")
-    keys = [key.decode()[len(prefix) + 1 :] for key in result]
+    keys = [key.decode()[len(prefix) + 1:] for key in result]
     return web.json_response(keys)
 
 
diff --git a/packaging/Makefile b/packaging/Makefile
index 22bfb4404..d06527cfc 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -43,7 +43,7 @@ vmlinux:
 
 version:
 	python3 ./version_from_git.py --inplace deb aleph-vm/DEBIAN/control
-	python3 ./version_from_git.py --inplace __version__ ../vm_supervisor/__init__.py
+	python3 ./version_from_git.py --inplace __version__ ../vm_supervisor/version.py
 
 build-dir:
 	mkdir -p target
diff --git a/vm_supervisor/__init__.py b/vm_supervisor/__init__.py
index 88495ea73..ae44191e8 100644
--- a/vm_supervisor/__init__.py
+++ b/vm_supervisor/__init__.py
@@ -1,36 +1,39 @@
-import logging
-from subprocess import check_output, CalledProcessError
-from typing import Optional
-
-logger = logging.getLogger(__name__)
-
-
-def get_version_from_git() -> Optional[str]:
-    try:
-        return check_output(("git", "describe", "--tags")).strip().decode()
-    except FileNotFoundError:
-        logger.warning("git not found")
-        return None
-    except CalledProcessError:
-        logger.warning("git description not available")
-        return None
-
-
-def get_version_from_apt() -> Optional[str]:
-    try:
-        import apt
-        return apt.Cache().get('aleph-vm').installed.version
-    except ImportError:
-        logger.warning("apt version not available")
-        return None
-
-
-def get_version() -> Optional[str]:
-    return get_version_from_git() or get_version_from_apt()
-
-
-# The version number is harcoded in the following line when packaging the software
-__version__ = get_version() or "version-unavailable"
-
-
+from . import conf
+from . import messages
+from . import metrics
+from . import models
+from . import pool
+from . import pubsub
+from . import reactor
+from . import resources
+from . import run
+from . import status
+from . import storage
 from . import supervisor
+from . import tasks
+from . import utils
+from . import version
+from . import views
+from . import vm
+
+__version__ = version.__version__
+
+__all__ = (
+    "conf",
+    "messages",
+    "metrics",
+    "models",
+    "pool",
+    "pubsub",
+    "reactor",
+    "resources",
+    "run",
+    "status",
+    "storage",
+    "supervisor",
+    "tasks",
+    "utils",
+    "version",
+    "views",
+    "vm",
+)
diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index a8ae23efa..d150c3d83 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -13,7 +13,7 @@
 except ImportError:
     sentry_sdk = None
 
-from vm_supervisor.pubsub import PubSub
+from .pubsub import PubSub
 from . import supervisor, metrics
 from .conf import settings
 from .models import VmHash
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 168fb88c4..792832e47 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -5,7 +5,7 @@
 from os.path import isfile, join, exists, abspath, isdir
 from pathlib import Path
 from subprocess import check_output
-from typing import NewType, Optional, List
+from typing import NewType, Optional, List, Dict, Any
 
 from pydantic import BaseSettings, Field
 
@@ -28,7 +28,7 @@ def etc_resolv_conf_dns_servers():
 
 
 def systemd_resolved_dns_servers(interface):
-    ## Example output format from systemd-resolve --status {interface}:
+    # Example output format from systemd-resolve --status {interface}:
     # Link 2 (enp7s0)
     #       Current Scopes: DNS
     # DefaultRoute setting: yes
diff --git a/vm_supervisor/messages.py b/vm_supervisor/messages.py
index 68d0e3b27..7b78d070d 100644
--- a/vm_supervisor/messages.py
+++ b/vm_supervisor/messages.py
@@ -4,8 +4,8 @@
 
 from aiohttp import ClientConnectorError, ClientResponseError
 from aiohttp.web_exceptions import HTTPServiceUnavailable, HTTPNotFound
-
 from aleph_message.models import ProgramMessage
+
 from .models import VmHash
 from .storage import get_message, get_latest_amend
 
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 3e8365945..cc5de0bc8 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -9,12 +9,12 @@
 
 from aleph_message.models import ProgramContent
 
+from .conf import settings
 from .metrics import save_record, save_execution_data, ExecutionRecord
 from .pubsub import PubSub
 from .utils import dumps_for_json, create_task_log_exceptions
 from .vm import AlephFirecrackerVM
 from .vm.firecracker_microvm import AlephFirecrackerResources
-from .conf import settings
 
 logger = logging.getLogger(__name__)
 
@@ -65,6 +65,10 @@ def is_running(self):
     def becomes_ready(self):
         return self.ready_event.wait
 
+    @property
+    def vm_id(self) -> Optional[int]:
+        return self.vm.vm_id if self.vm else None
+
     def __init__(
         self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent
     ):
@@ -122,7 +126,6 @@ def stop_after_timeout(self, timeout: float = 5.0) -> Task:
             logger.debug("VM already has a timeout. Extending it.")
             self.expire_task.cancel()
 
-        loop = asyncio.get_event_loop()
         if sys.version_info.major >= 3 and sys.version_info.minor >= 8:
             # Task can be named
             vm_id: str = str(self.vm.vm_id if self.vm else None)
@@ -169,7 +172,9 @@ async def stop(self):
 
     def start_watching_for_updates(self, pubsub: PubSub):
         if not self.update_task:
-            self.update_task = create_task_log_exceptions(self.watch_for_updates(pubsub=pubsub))
+            self.update_task = create_task_log_exceptions(
+                self.watch_for_updates(pubsub=pubsub)
+            )
 
     async def watch_for_updates(self, pubsub: PubSub):
         await pubsub.msubscribe(
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index e3b25e393..8016b01d1 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -1,6 +1,6 @@
 import asyncio
 import logging
-from typing import Dict, Optional, Iterable
+from typing import Dict, Optional
 
 from aleph_message.models import ProgramContent, ProgramMessage
 
@@ -55,12 +55,16 @@ def get_unique_vm_id(self) -> int:
             #
             # We therefore recycle vm_id values from executions that are not running
             # anymore.
-            currently_used_vm_ids = set(execution.vm.vm_id
-                                        for execution in self.executions.values()
-                                        if execution.is_running)
+            currently_used_vm_ids = set(
+                execution.vm_id
+                for execution in self.executions.values()
+                if execution.is_running
+            )
             for i in range(settings.START_ID_INDEX, 255**2):
                 if i not in currently_used_vm_ids:
                     return i
+            else:
+                raise ValueError("No available value for vm_id.")
 
     async def get_running_vm(self, vm_hash: VmHash) -> Optional[VmExecution]:
         """Return a running VM or None. Disables the VM expiration task."""
diff --git a/vm_supervisor/reactor.py b/vm_supervisor/reactor.py
index 00d66a143..b4e499bc4 100644
--- a/vm_supervisor/reactor.py
+++ b/vm_supervisor/reactor.py
@@ -4,9 +4,9 @@
 from aleph_message.models import Message, ProgramMessage
 from aleph_message.models.program import Subscription
 
-from vm_supervisor.pubsub import PubSub
-from vm_supervisor.run import run_code_on_event
-from vm_supervisor.utils import create_task_log_exceptions
+from .pubsub import PubSub
+from .run import run_code_on_event
+from .utils import create_task_log_exceptions
 
 logger = logging.getLogger(__name__)
 
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 3280276ee..4850b5a3b 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -87,7 +87,7 @@ async def run_code_on_request(
     if not execution:
         execution = await create_vm_execution(vm_hash=vm_hash)
 
-    logger.debug(f"Using vm={execution.vm.vm_id}")
+    logger.debug(f"Using vm={execution.vm_id}")
 
     scope: Dict = await build_asgi_scope(path, request)
 
@@ -95,7 +95,7 @@ async def run_code_on_request(
         await execution.becomes_ready()
         result_raw: bytes = await execution.run_code(scope=scope)
 
-        if result_raw == b'':
+        if result_raw == b"":
             # Missing result from the init process of the virtual machine, not even an error message.
             # It may have completely crashed.
 
@@ -103,13 +103,14 @@ async def run_code_on_request(
             # It will be restarted on a future request.
             await execution.stop()
 
-            return web.Response(status=502, reason="No response from VM",
-                                text="VM did not respond and was shut down")
+            return web.Response(
+                status=502,
+                reason="No response from VM",
+                text="VM did not respond and was shut down",
+            )
 
     except asyncio.TimeoutError:
-        logger.warning(
-            f"VM{execution.vm.vm_id} did not respond within `resource.seconds`"
-        )
+        logger.warning(f"VM{execution.vm_id} did not respond within `resource.seconds`")
         return web.HTTPGatewayTimeout(
             body="Program did not respond within `resource.seconds`"
         )
@@ -175,7 +176,7 @@ async def run_code_on_event(vm_hash: VmHash, event, pubsub: PubSub):
     if not execution:
         execution = await create_vm_execution(vm_hash=vm_hash)
 
-    logger.debug(f"Using vm={execution.vm.vm_id}")
+    logger.debug(f"Using vm={execution.vm_id}")
 
     scope: Dict = await build_event_scope(event)
 
diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py
index ddddacca3..d4b4a5de9 100644
--- a/vm_supervisor/status.py
+++ b/vm_supervisor/status.py
@@ -7,7 +7,7 @@
 
 from aiohttp import ClientSession, ClientResponseError
 
-from vm_supervisor.conf import settings
+from .conf import settings
 
 logger = logging.getLogger(__name__)
 
@@ -67,7 +67,7 @@ async def check_internet(session: ClientSession) -> bool:
 async def check_cache(session: ClientSession) -> bool:
     try:
         result1: bool = await get_json_from_vm(session, "/cache/set/a/42")
-        assert result1 == True
+        assert result1 is True
         result2: int = await get_json_from_vm(session, "/cache/get/a")
         assert result2 == "42"
         keys: List[str] = await get_json_from_vm(session, "/cache/keys")
@@ -95,7 +95,7 @@ async def check_error_raised(session: ClientSession) -> bool:
     try:
         async with session.get(f"{CHECK_VM_URL}/raise") as resp:
             text = await resp.text()
-            return (resp.status == 500 and "Traceback" in text)
+            return resp.status == 500 and "Traceback" in text
     except ClientResponseError:
         return False
 
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 8a71dfb6f..2945b249a 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -109,7 +109,7 @@ async def get_code_path(ref: str) -> Path:
             logger.debug(f"Squashfs generated on {archive_path}.squashfs")
             return Path(f"{archive_path}.squashfs")
         elif encoding == Encoding.zip:
-            make_archive(archive_path, "zip", root_dir=archive_path)
+            make_archive(str(archive_path), "zip", root_dir=archive_path)
             logger.debug(f"Zip generated on {archive_path}.zip")
             return Path(f"{archive_path}.zip")
         else:
@@ -124,7 +124,7 @@ async def get_code_path(ref: str) -> Path:
 async def get_data_path(ref: str) -> Path:
     if settings.FAKE_DATA_PROGRAM and settings.FAKE_DATA_DATA:
         data_dir = settings.FAKE_DATA_DATA
-        make_archive(data_dir, "zip", data_dir)
+        make_archive(str(data_dir), "zip", data_dir)
         return Path(f"{data_dir}.zip")
 
     cache_path = Path(join(settings.DATA_CACHE, ref))
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index b70d891ae..9e4001ab2 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -11,12 +11,12 @@
 
 from aiohttp import web
 
-from . import __version__
 from . import metrics
 from .conf import settings
 from .resources import about_system_usage
 from .run import pool
 from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task
+from .version import __version__
 from .views import (
     run_code_from_path,
     run_code_from_hostname,
@@ -24,7 +24,8 @@
     about_executions,
     about_config,
     status_check_fastapi,
-    about_execution_records, status_check_version,
+    about_execution_records,
+    status_check_version,
 )
 
 logger = logging.getLogger(__name__)
diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py
index 803885559..5821363fb 100644
--- a/vm_supervisor/tasks.py
+++ b/vm_supervisor/tasks.py
@@ -124,7 +124,9 @@ async def start_watch_for_messages_task(app: web.Application):
 
     app["pubsub"] = pubsub
     app["reactor"] = reactor
-    app["messages_listener"] = create_task_log_exceptions(watch_for_messages(pubsub, reactor))
+    app["messages_listener"] = create_task_log_exceptions(
+        watch_for_messages(pubsub, reactor)
+    )
 
 
 async def stop_watch_for_messages_task(app: web.Application):
diff --git a/vm_supervisor/version.py b/vm_supervisor/version.py
new file mode 100644
index 000000000..c64544bb1
--- /dev/null
+++ b/vm_supervisor/version.py
@@ -0,0 +1,34 @@
+import logging
+from subprocess import check_output, CalledProcessError
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+def get_version_from_git() -> Optional[str]:
+    try:
+        return check_output(("git", "describe", "--tags")).strip().decode()
+    except FileNotFoundError:
+        logger.warning("git not found")
+        return None
+    except CalledProcessError:
+        logger.warning("git description not available")
+        return None
+
+
+def get_version_from_apt() -> Optional[str]:
+    try:
+        import apt
+
+        return apt.Cache().get("aleph-vm").installed.version
+    except ImportError:
+        logger.warning("apt version not available")
+        return None
+
+
+def get_version() -> Optional[str]:
+    return get_version_from_git() or get_version_from_apt()
+
+
+# The version number is hardcoded in the following line when packaging the software
+__version__ = get_version() or "version-unavailable"
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index f065e7f06..b21b91974 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -3,14 +3,15 @@
 import os.path
 from string import Template
 from typing import Awaitable, Optional
-from packaging.version import Version, InvalidVersion
 
 import aiodns
 import aiohttp
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPNotFound
+from packaging.version import Version, InvalidVersion
 
-from . import status, __version__
+from . import status
+from .version import __version__
 from .conf import settings
 from .metrics import get_execution_records
 from .models import VmHash
@@ -75,7 +76,7 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
     return await run_code_on_request(message_ref, path, request)
 
 
-def authenticate_request(request: web.Request) -> web.Response:
+def authenticate_request(request: web.Request) -> None:
     """Check that the token in the cookies matches the app's secret token."""
     if request.cookies.get("token") != request.app["secret_token"]:
         raise web.HTTPUnauthorized(reason="Invalid token", text="401 Invalid token")
diff --git a/vm_supervisor/vm/__init__.py b/vm_supervisor/vm/__init__.py
index ae7b3635f..414fd5790 100644
--- a/vm_supervisor/vm/__init__.py
+++ b/vm_supervisor/vm/__init__.py
@@ -1,3 +1,4 @@
+from . import firecracker_microvm
 from .firecracker_microvm import AlephFirecrackerVM
 
-assert AlephFirecrackerVM
+__all__ = ("firecracker_microvm", "AlephFirecrackerVM")
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 525e7ea74..50f5620d1 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -77,7 +77,7 @@ class Volume:
 @dataclass
 class HostVolume:
     mount: str
-    path_on_host: str
+    path_on_host: Path
     read_only: bool
 
 
@@ -454,7 +454,9 @@ async def communicate(reader, writer, scope):
             return response
 
         try:
-            reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
+            reader, writer = await asyncio.open_unix_connection(
+                path=self.fvm.vsock_path
+            )
         except ConnectionRefusedError:
             raise VmInitNotConnected("MicroVM may have crashed")
         try:

From 23f05fb250a2a8c739d1ba18bc581a0b4edf504b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 6 Oct 2022 00:23:34 +0200
Subject: [PATCH 342/990] Feature: VMs could not be persisted (#241)

Feature: VMs could not be persisted

The supervisor only supported on request execution of
virtual machines.

This add support for an external scheduler to send a
list of VMs that should be started and persisted.

A temporary simple HTTP authentication method is
used to authenticate the scheduler, this is planned to move
to P2P messages from Core Channel Nodes.
---
 docker/vm_supervisor-dev.dockerfile |  2 +-
 examples/volumes/Dockerfile         |  2 +-
 guest_api/__main__.py               |  2 +-
 packaging/Makefile                  |  2 +-
 vm_supervisor/README.md             |  2 +-
 vm_supervisor/conf.py               |  5 +++
 vm_supervisor/models.py             |  8 ++++-
 vm_supervisor/pool.py               |  7 +++-
 vm_supervisor/resources.py          |  7 ++++
 vm_supervisor/run.py                | 27 +++++++++++++++
 vm_supervisor/supervisor.py         |  2 ++
 vm_supervisor/utils.py              |  2 +-
 vm_supervisor/views.py              | 53 ++++++++++++++++++++++++++++-
 13 files changed, 112 insertions(+), 9 deletions(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index e4be2a238..8fcad9484 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -19,7 +19,7 @@ RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://s3.amazonaws.com/spec.ccf
 RUN ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker
 RUN ln /opt/firecracker/release-*/jailer-v* /opt/firecracker/jailer
 
-RUN pip3 install typing-extensions 'aleph-message>=0.1.19'
+RUN pip3 install typing-extensions 'aleph-message==0.2.2'
 
 RUN mkdir -p /var/lib/aleph/vm/jailer
 
diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile
index f80fdea60..4ce6a6c10 100644
--- a/examples/volumes/Dockerfile
+++ b/examples/volumes/Dockerfile
@@ -6,6 +6,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 RUN python3 -m venv /opt/venv
-RUN /opt/venv/bin/pip install 'aleph-message>=0.1.19'
+RUN /opt/venv/bin/pip install 'aleph-message==0.2.2'
 
 CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs
diff --git a/guest_api/__main__.py b/guest_api/__main__.py
index 11249f0b7..829680bbf 100644
--- a/guest_api/__main__.py
+++ b/guest_api/__main__.py
@@ -151,7 +151,7 @@ async def list_keys_from_cache(request: web.Request):
 
     redis: aioredis.Redis = await get_redis()
     result = await redis.keys(f"{prefix}:{pattern}")
-    keys = [key.decode()[len(prefix) + 1:] for key in result]
+    keys = [key.decode()[len(prefix) + 1 :] for key in result]
     return web.json_response(keys)
 
 
diff --git a/packaging/Makefile b/packaging/Makefile
index d06527cfc..87563ed20 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -16,7 +16,7 @@ debian-package-code:
 	cp ../examples/message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message>=0.1.19'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.2.2'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index ac06beccf..d125cb79d 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -87,7 +87,7 @@ is used to parse and validate Aleph messages.
 ```shell
 apt install -y --no-install-recommends --no-install-suggests python3-pip
 pip3 install pydantic[dotenv]
-pip3 install aleph-message
+pip3 install 'aleph-message==0.2.2'
 ```
 
 ### 2.f. Create the jailer working directory:
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 792832e47..fa18b6431 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -106,6 +106,11 @@ class Settings(BaseSettings):
     MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000  # 10 MB
     MAX_DATA_ARCHIVE_SIZE = 10_000_000  # 10 MB
 
+    # hashlib.sha256(b"secret-token").hexdigest()
+    ALLOCATION_TOKEN_HASH = (
+        "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda"
+    )
+
     FAKE_DATA_PROGRAM: Optional[Path] = None
     BENCHMARK_FAKE_DATA_PROGRAM = Path(
         abspath(join(__file__, "../../examples/example_fastapi"))
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index cc5de0bc8..abbd23c8d 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -57,6 +57,8 @@ class VmExecution:
     expire_task: Optional[asyncio.Task] = None
     update_task: Optional[asyncio.Task] = None
 
+    persistent: bool = False
+
     @property
     def is_running(self):
         return self.times.starting_at and not self.times.stopping_at
@@ -121,7 +123,11 @@ async def create(self, vm_id: int) -> AlephFirecrackerVM:
             await vm.teardown()
             raise
 
-    def stop_after_timeout(self, timeout: float = 5.0) -> Task:
+    def stop_after_timeout(self, timeout: float = 5.0) -> Optional[Task]:
+        if self.persistent:
+            logger.debug("VM marked as long running. Ignoring timeout.")
+            return
+
         if self.expire_task:
             logger.debug("VM already has a timeout. Extending it.")
             self.expire_task.cancel()
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 8016b01d1..0a2398174 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -1,6 +1,6 @@
 import asyncio
 import logging
-from typing import Dict, Optional
+from typing import Dict, Optional, Iterable
 
 from aleph_message.models import ProgramContent, ProgramMessage
 
@@ -94,3 +94,8 @@ async def stop(self):
         await asyncio.gather(
             *(execution.stop() for vm_hash, execution in self.executions.items())
         )
+
+    def get_persistent_executions(self) -> Iterable[VmExecution]:
+        for vm_hash, execution in self.executions.items():
+            if execution.persistent and execution.is_running:
+                yield execution
diff --git a/vm_supervisor/resources.py b/vm_supervisor/resources.py
index b2fd8968f..1f9c06050 100644
--- a/vm_supervisor/resources.py
+++ b/vm_supervisor/resources.py
@@ -1,5 +1,6 @@
 from datetime import datetime, timezone
 from functools import lru_cache
+from typing import Set, Optional
 from typing import Tuple
 
 import cpuinfo
@@ -117,3 +118,9 @@ async def about_system_usage(request: web.Request):
     return web.json_response(
         text=usage.json(exclude_none=True),
     )
+
+
+class Allocation(BaseModel):
+    persistent_vms: Set[str]
+    on_demand_vms: Optional[Set[str]] = None
+    jobs: Optional[Set] = None
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 4850b5a3b..2df96cf64 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -214,3 +214,30 @@ async def run_code_on_event(vm_hash: VmHash, event, pubsub: PubSub):
             execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT)
         else:
             await execution.stop()
+
+
+async def start_persistent_vm(vm_hash: VmHash, pubsub: PubSub) -> VmExecution:
+    execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
+
+    if not execution:
+        logger.info(f"Starting persistent VM {vm_hash}")
+        execution = await create_vm_execution(vm_hash=vm_hash)
+    # If the VM was already running in lambda mode, it should not expire
+    # as long as it is also scheduled as long-running
+    execution.persistent = True
+    execution.cancel_expiration()
+
+    await execution.becomes_ready()
+
+    if settings.WATCH_FOR_UPDATES:
+        execution.start_watching_for_updates(pubsub=pubsub)
+
+    return execution
+
+
+async def stop_persistent_vm(vm_hash: VmHash) -> Optional[VmExecution]:
+    logger.info(f"Stopping persistent VM {vm_hash}")
+    execution = await pool.get_running_vm(vm_hash)
+    if execution:
+        await execution.stop()
+    return execution
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 9e4001ab2..83259c61a 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -26,6 +26,7 @@
     status_check_fastapi,
     about_execution_records,
     status_check_version,
+    update_allocations,
 )
 
 logger = logging.getLogger(__name__)
@@ -53,6 +54,7 @@ async def server_version_middleware(
         web.get("/about/executions/records", about_execution_records),
         web.get("/about/usage/system", about_system_usage),
         web.get("/about/config", about_config),
+        web.post("/control/allocations", update_allocations),
         web.get("/status/check/fastapi", status_check_fastapi),
         web.get("/status/check/version", status_check_version),
         web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index 10a4f2051..3c247af4d 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -3,7 +3,7 @@
 import logging
 from base64 import b32decode, b16encode
 from dataclasses import is_dataclass, asdict as dataclass_as_dict
-from typing import Any, Optional, Coroutine, Dict
+from typing import Any, Optional, Coroutine
 
 import aiodns
 
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index b21b91974..e894687d1 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -1,6 +1,7 @@
 import binascii
 import logging
 import os.path
+from hashlib import sha256
 from string import Template
 from typing import Awaitable, Optional
 
@@ -9,13 +10,16 @@
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPNotFound
 from packaging.version import Version, InvalidVersion
+from pydantic import ValidationError
 
 from . import status
 from .version import __version__
 from .conf import settings
 from .metrics import get_execution_records
 from .models import VmHash
-from .run import run_code_on_request, pool
+from .pubsub import PubSub
+from .resources import Allocation
+from .run import run_code_on_request, pool, start_persistent_vm
 from .utils import b32_to_b16, get_ref_from_dns, dumps_for_json
 
 logger = logging.getLogger(__name__)
@@ -167,3 +171,50 @@ async def status_check_version(request: web.Request):
         )
     else:
         return web.HTTPForbidden(text=f"Outdated: version {current} < {reference}")
+
+
+def authenticate_api_request(request: web.Request) -> bool:
+    """Authenticate an API request to update the VM allocations."""
+    signature: bytes = request.headers.get("X-Auth-Signature").encode()
+
+    if not signature:
+        raise web.HTTPUnauthorized(text="Authentication token is missing")
+
+    # Use a simple authentication method: the hash of the signature should match the value in the settings
+    return sha256(signature).hexdigest() == settings.ALLOCATION_TOKEN_HASH
+
+
+async def update_allocations(request: web.Request):
+    if not authenticate_api_request(request):
+        return web.HTTPUnauthorized(text="Authentication token received is invalid")
+
+    try:
+        data = await request.json()
+        allocation = Allocation.parse_obj(data)
+    except ValidationError as error:
+        return web.json_response(
+            data=error.json(), status=web.HTTPBadRequest.status_code
+        )
+
+    pubsub: PubSub = request.app["pubsub"]
+
+    # Start VMs
+    for vm_hash in allocation.persistent_vms:
+        vm_hash = VmHash(vm_hash)
+        logger.info(f"Starting long running VM {vm_hash}")
+        await start_persistent_vm(vm_hash, pubsub)
+
+    # Stop VMs
+    for execution in pool.get_persistent_executions():
+        if execution.vm_hash not in allocation.persistent_vms:
+            logger.info(f"Stopping long running VM {execution.vm_hash}")
+            await execution.stop()
+            execution.persistent = False
+
+    # Log unsupported features
+    if allocation.on_demand_vms:
+        logger.warning("Not supported yet: 'allocation.on_demand_vms'")
+    if allocation.jobs:
+        logger.warning("Not supported yet: 'allocation.on_demand_vms'")
+
+    return web.json_response(data={"success": True})

From 7c742b07d2203acddd1dcee800d7aff49e568cc8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 6 Oct 2022 12:56:55 +0200
Subject: [PATCH 343/990] Fix: Annotation was not supported by Python 3.8

We still support Ubuntu 20.04 that ships with Python 3.8.

Solution: Use a different typing annotation depending on the version of Python, making it easier to ditch the simpler annotation when Python 3.8 goes out of support.
---
 vm_supervisor/pubsub.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/pubsub.py b/vm_supervisor/pubsub.py
index 7ef36056f..ae9a40318 100644
--- a/vm_supervisor/pubsub.py
+++ b/vm_supervisor/pubsub.py
@@ -5,13 +5,18 @@
 
 import asyncio
 import logging
+import sys
 from typing import Dict, Hashable, Set
 
 logger = logging.getLogger(__name__)
 
 
 class PubSub:
-    subscribers: Dict[Hashable, Set[asyncio.Queue[set]]]
+    if sys.version_info >= (3, 9):
+        subscribers: Dict[Hashable, Set[asyncio.Queue[Set]]]
+    else:
+        # Support for Python 3.8 (Ubuntu 20.04)
+        subscribers: Dict[Hashable, Set[asyncio.Queue]]
 
     def __init__(self):
         self.subscribers = {}

From 9cfe03f4f31f37512d36aed4ce7e7569e0bd21c0 Mon Sep 17 00:00:00 2001
From: tomribbens 
Date: Wed, 12 Oct 2022 15:29:43 +0200
Subject: [PATCH 344/990] Fix: Remove hardcoded 172.0.0.0/8 range from use to
 comply with RFC1918

Previously, 172.0.0.0/8 range was hardcoded to be used as the pool for which to assign IP addresses out of. As this was in violation of RFC1918, this needed to change. As this meant reworking the code anyway, it was changed so the range is configurable.
---
 firecracker/microvm.py                     | 16 +++----
 runtimes/aleph-alpine-3.13-python/init1.py |  4 +-
 vm_supervisor/conf.py                      | 11 +++--
 vm_supervisor/pool.py                      |  4 +-
 vm_supervisor/utils.py                     | 52 +++++++++++++++++++++-
 vm_supervisor/vm/firecracker_microvm.py    | 11 +++++
 6 files changed, 82 insertions(+), 16 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 8b79899c1..ff3db5202 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -71,6 +71,8 @@ class MicroVM:
     config_file_path: Optional[Path] = None
     drives: List[Drive]
     init_timeout: float
+    guest_ip: Optional[str]
+    host_ip: Optional[str]
 
     _unix_socket: Server
 
@@ -97,14 +99,6 @@ def vsock_path(self):
         else:
             return f"{VSOCK_PATH}"
 
-    @property
-    def guest_ip(self):
-        return f"172.{self.vm_id // 256}.{self.vm_id % 256}.2"
-
-    @property
-    def host_ip(self):
-        return f"172.{self.vm_id // 256}.{self.vm_id % 256}.1"
-
     def __init__(
         self,
         vm_id: int,
@@ -112,6 +106,8 @@ def __init__(
         use_jailer: bool = True,
         jailer_bin_path: Optional[str] = None,
         init_timeout: float = 5.0,
+        guest_ip: Optional[str] = None,
+        host_ip: Optional[str] = None,
     ):
         self.vm_id = vm_id
         self.use_jailer = use_jailer
@@ -119,6 +115,8 @@ def __init__(
         self.jailer_bin_path = jailer_bin_path
         self.drives = []
         self.init_timeout = init_timeout
+        self.guest_ip = guest_ip
+        self.host_ip = host_ip
 
     def to_dict(self):
         return {
@@ -316,7 +314,7 @@ async def create_network_interface(self, interface: str = "eth0") -> str:
         self.network_tap = host_dev_name
 
         system(f"ip tuntap add {host_dev_name} mode tap")
-        system(f"ip addr add {self.host_ip}/24 dev {host_dev_name}")
+        system(f"ip addr add {self.host_ip} dev {host_dev_name}")
         system(f"ip link set {host_dev_name} up")
         system('sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"')
         # TODO: Don't fill iptables with duplicate rules; purge rules on delete
diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py
index 7f5b6652c..587b2bbed 100644
--- a/runtimes/aleph-alpine-3.13-python/init1.py
+++ b/runtimes/aleph-alpine-3.13-python/init1.py
@@ -123,11 +123,11 @@ def setup_network(
     system("ip addr add 127.0.0.1/8 dev lo brd + scope host")
     system("ip addr add ::1/128 dev lo")
     system("ip link set lo up")
-    system(f"ip addr add {ip}/24 dev eth0")
+    system(f"ip addr add {ip} dev eth0")
     system("ip link set eth0 up")
 
     if route:
-        system(f"ip route add default via {route} dev eth0")
+        system(f"ip route add default via {route.split('/')[0]} dev eth0")
         logger.debug("IP and route set")
     else:
         logger.warning("IP set with no network route")
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index fa18b6431..59e497099 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -72,17 +72,22 @@ class Settings(BaseSettings):
     REUSE_TIMEOUT: float = 60 * 60.0
     WATCH_FOR_MESSAGES = True
     WATCH_FOR_UPDATES = True
-    NETWORK_INTERFACE = "eth0"
-    DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf
-    DNS_NAMESERVERS: Optional[List[str]] = None
 
     API_SERVER = "https://official.aleph.cloud"
     USE_JAILER = True
     # System logs make boot ~2x slower
     PRINT_SYSTEM_LOGS = False
     DEBUG_ASYNCIO = False
+
     # Networking does not work inside Docker/Podman
     ALLOW_VM_NETWORKING = True
+    NETWORK_INTERFACE = "eth0"
+    IPV4_ADDRESS_POOL = "172.16.0.0/12"
+    IPV4_NETWORK_SIZE = 24
+
+    DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf
+    DNS_NAMESERVERS: Optional[List[str]] = None
+
     FIRECRACKER_PATH = "/opt/firecracker/firecracker"
     JAILER_PATH = "/opt/firecracker/jailer"
     LINUX_PATH = "/opt/firecracker/vmlinux.bin"
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 0a2398174..cfba99869 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -44,8 +44,10 @@ def get_unique_vm_id(self) -> int:
         This identifier is used to name the network interface and in the IPv4 range
         dedicated to the VM.
         """
+        _, network_range = settings.IPV4_ADDRESS_POOL.split("/")
+        available_bits = int(network_range) - settings.IPV4_NETWORK_SIZE
         self.counter += 1
-        if self.counter < 255**2:
+        if self.counter < 2**available_bits:
             # In common cases, use the counter itself as the vm_id. This makes it
             # easier to debug.
             return self.counter
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index 3c247af4d..08aad0082 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -3,7 +3,7 @@
 import logging
 from base64 import b32decode, b16encode
 from dataclasses import is_dataclass, asdict as dataclass_as_dict
-from typing import Any, Optional, Coroutine
+from typing import Any, Optional, Coroutine, Tuple, Iterable
 
 import aiodns
 
@@ -51,3 +51,53 @@ async def run_and_log_exception(coro: Coroutine):
 def create_task_log_exceptions(coro: Coroutine, *, name=None):
     """Ensure that exceptions running in coroutines are logged."""
     return asyncio.create_task(run_and_log_exception(coro), name=name)
+
+
+def ipstr_to_int(ip_string: str) -> Tuple[int, int]:
+    """Convert an IP address string with subnet mask to an integer
+    representation of the IP and the mask separately.
+    """
+    ip, mask = ip_string.split("/")
+    ip_int = sum(
+        int(octet) * 256**idx for idx, octet in enumerate(reversed(ip.split(".")))
+    )
+    return ip_int, int(mask)
+
+
+def int_to_ipstr(ip_int: int, mask: int) -> str:
+    """Converts an integer representation of an IP address and a subnetmask
+    and turns it into a string representation
+    """
+    ip_integers: Iterable[int] = (
+        (ip_int >> (8 * i)) & 0xFF for i in reversed(range(4))
+    )
+    ip_string: str = ".".join(str(i) for i in ip_integers)
+    return f"{ip_string}/{mask}"
+
+
+def get_ip_addresses(
+    vm_id: int, address_pool: str, ip_network_size: int
+) -> Tuple[str, str]:
+    """Calculates the host and guest ip from vm_id and returns it as their string representations with subnetmask"""
+    network_pool, pool_size = ipstr_to_int(address_pool)
+    pool_netmask = 0xFFFFFFFF << 32 - pool_size
+    network_part = vm_id << 32 - ip_network_size
+    network_part_mask = 2 ** (ip_network_size - pool_size) - 1 << 32 - ip_network_size
+    host = 1
+    guest = 2
+    hosts_mask = 2 ** (32 - ip_network_size) - 1
+
+    host_ip = (
+        (network_pool & pool_netmask)
+        | (network_part & network_part_mask)
+        | (host & hosts_mask)
+    )
+    guest_ip = (
+        (network_pool & pool_netmask)
+        | (network_part & network_part_mask)
+        | (guest & hosts_mask)
+    )
+
+    return int_to_ipstr(host_ip, ip_network_size), int_to_ipstr(
+        guest_ip, ip_network_size
+    )
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 50f5620d1..434bddeac 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -12,6 +12,8 @@
 
 import msgpack
 
+from ..utils import get_ip_addresses
+
 try:
     import psutil as psutil
 except ImportError:
@@ -211,6 +213,8 @@ class AlephFirecrackerVM:
     hardware_resources: MachineResources
     fvm: Optional[MicroVM] = None
     guest_api_process: Optional[Process] = None
+    host_ip: Optional[str] = None
+    guest_ip: Optional[str] = None
 
     def __init__(
         self,
@@ -260,12 +264,19 @@ def to_dict(self):
     async def setup(self):
         logger.debug("setup started")
         await setfacl()
+        host_ip, guest_ip = get_ip_addresses(
+            self.vm_id,
+            address_pool=settings.IPV4_ADDRESS_POOL,
+            ip_network_size=settings.IPV4_NETWORK_SIZE,
+        )
         fvm = MicroVM(
             vm_id=self.vm_id,
             firecracker_bin_path=settings.FIRECRACKER_PATH,
             use_jailer=settings.USE_JAILER,
             jailer_bin_path=settings.JAILER_PATH,
             init_timeout=settings.INIT_TIMEOUT,
+            host_ip=host_ip,
+            guest_ip=guest_ip,
         )
         fvm.prepare_jailer()
 

From df99175750e993a7fcb6e181135358789d18674f Mon Sep 17 00:00:00 2001
From: tomribbens 
Date: Mon, 21 Nov 2022 21:17:53 +0100
Subject: [PATCH 345/990] Refactor: Move from iptables to nft as iptables is
 deprecated (#250)

Refactor: Move from iptables to nft as iptables is deprecated

Since iptables is deprecated, a move to the replacement nftables was implemented in this change. This is in line with various network improvements that are ongoing as discussed in Issue #237. The use of nftables will facilitate some future enhancements, especially related to IPv6 support.

At the same time, this commit refactors all the networking and firewall code into its own package "network", as multiple packages were depending on making network changes. The new package is now an interface for all networking related tasks.

Co-authored-by: Hugo Herter 
---
 firecracker/microvm.py                     |  51 ---
 packaging/aleph-vm/DEBIAN/control          |   2 +-
 runtimes/aleph-alpine-3.13-python/init1.py |   4 +-
 vm_supervisor/README.md                    |   2 +-
 vm_supervisor/conf.py                      |   1 +
 vm_supervisor/metrics.py                   |   1 -
 vm_supervisor/models.py                    |   8 +-
 vm_supervisor/network/__init__.py          |   0
 vm_supervisor/network/firewall.py          | 401 +++++++++++++++++++++
 vm_supervisor/network/hostnetwork.py       |  63 ++++
 vm_supervisor/network/interfaces.py        |  51 +++
 vm_supervisor/network/ipaddresses.py       |  23 ++
 vm_supervisor/pool.py                      |  11 +-
 vm_supervisor/supervisor.py                |  16 +-
 vm_supervisor/utils.py                     |  52 +--
 vm_supervisor/vm/firecracker_microvm.py    |  32 +-
 16 files changed, 584 insertions(+), 134 deletions(-)
 create mode 100644 vm_supervisor/network/__init__.py
 create mode 100644 vm_supervisor/network/firewall.py
 create mode 100644 vm_supervisor/network/hostnetwork.py
 create mode 100644 vm_supervisor/network/interfaces.py
 create mode 100644 vm_supervisor/network/ipaddresses.py

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index ff3db5202..9508ec1a1 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -64,15 +64,11 @@ class MicroVM:
     firecracker_bin_path: str
     jailer_bin_path: Optional[str]
     proc: Optional[asyncio.subprocess.Process] = None
-    network_tap: Optional[str] = None
-    network_interface: Optional[str] = None
     stdout_task: Optional[Task] = None
     stderr_task: Optional[Task] = None
     config_file_path: Optional[Path] = None
     drives: List[Drive]
     init_timeout: float
-    guest_ip: Optional[str]
-    host_ip: Optional[str]
 
     _unix_socket: Server
 
@@ -106,8 +102,6 @@ def __init__(
         use_jailer: bool = True,
         jailer_bin_path: Optional[str] = None,
         init_timeout: float = 5.0,
-        guest_ip: Optional[str] = None,
-        host_ip: Optional[str] = None,
     ):
         self.vm_id = vm_id
         self.use_jailer = use_jailer
@@ -115,16 +109,12 @@ def __init__(
         self.jailer_bin_path = jailer_bin_path
         self.drives = []
         self.init_timeout = init_timeout
-        self.guest_ip = guest_ip
-        self.host_ip = host_ip
 
     def to_dict(self):
         return {
             "jailer_path": self.jailer_path,
             "socket_path": self.socket_path,
             "vsock_path": self.vsock_path,
-            "guest_ip": self.guest_ip,
-            "host_ip": self.host_ip,
             **self.__dict__,
         }
 
@@ -302,30 +292,6 @@ def enable_drive(self, drive_path: str, read_only: bool = True) -> Drive:
         self.drives.append(drive)
         return drive
 
-    async def create_network_interface(self, interface: str = "eth0") -> str:
-        logger.debug("Create network interface")
-
-        assert self.network_interface is None  # Only one is supported at the moment
-        assert self.network_tap is None
-
-        self.network_interface = interface
-
-        host_dev_name = f"vmtap{self.vm_id}"
-        self.network_tap = host_dev_name
-
-        system(f"ip tuntap add {host_dev_name} mode tap")
-        system(f"ip addr add {self.host_ip} dev {host_dev_name}")
-        system(f"ip link set {host_dev_name} up")
-        system('sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"')
-        # TODO: Don't fill iptables with duplicate rules; purge rules on delete
-        system(f"iptables -t nat -A POSTROUTING -o {interface} -j MASQUERADE")
-        system(
-            "iptables -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT"
-        )
-        system(f"iptables -A FORWARD -i {host_dev_name} -o {interface} -j ACCEPT")
-
-        return host_dev_name
-
     async def print_logs(self):
         while not self.proc:
             await asyncio.sleep(0.01)  # Todo: Use signal here
@@ -434,23 +400,6 @@ async def teardown(self):
         if self.stderr_task:
             self.stderr_task.cancel()
 
-        if self.network_tap:
-            await asyncio.sleep(
-                0.01
-            )  # Used to prevent `ioctl(TUNSETIFF): Device or resource busy`
-            logger.debug(f"Removing interface {self.network_tap}")
-            system(f"ip tuntap del {self.network_tap} mode tap")
-            logger.debug("Removing iptables rules")
-            system(
-                f"iptables -t nat -D POSTROUTING -o {self.network_interface} -j MASQUERADE"
-            )
-            system(
-                "iptables -D FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT"
-            )
-            system(
-                f"iptables -D FORWARD -i {self.network_tap} -o {self.network_interface} -j ACCEPT"
-            )
-
         if self._unix_socket:
             logger.debug("Closing unix socket")
             self._unix_socket.close()
diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index 1bbd7d5bb..0162c4a99 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema
 Section: aleph-im
 Priority: Extra
diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py
index 587b2bbed..d559f171e 100644
--- a/runtimes/aleph-alpine-3.13-python/init1.py
+++ b/runtimes/aleph-alpine-3.13-python/init1.py
@@ -127,8 +127,8 @@ def setup_network(
     system("ip link set eth0 up")
 
     if route:
-        system(f"ip route add default via {route.split('/')[0]} dev eth0")
-        logger.debug("IP and route set")
+        system(f"ip route add default via {route} dev eth0")
+        logger.debug(f"IP and route set: {ip} via {route}")
     else:
         logger.warning("IP set with no network route")
 
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index d125cb79d..aa3024fa3 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -57,7 +57,7 @@ when running the VM Supervisor.
 ```shell
 apt update
 apt install -y git python3 python3-aiohttp python3-msgpack python3-aiodns python3-sqlalchemy python3-setproctitle redis python3-aioredis \
- python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap
+ python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-nftables python3-jsonschema
 useradd jailman
 ```
 
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 59e497099..ea0709b2d 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -84,6 +84,7 @@ class Settings(BaseSettings):
     NETWORK_INTERFACE = "eth0"
     IPV4_ADDRESS_POOL = "172.16.0.0/12"
     IPV4_NETWORK_SIZE = 24
+    NFTABLES_CHAIN_PREFIX = "aleph"
 
     DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf
     DNS_NAMESERVERS: Optional[List[str]] = None
diff --git a/vm_supervisor/metrics.py b/vm_supervisor/metrics.py
index db9dbab4a..9789aa795 100644
--- a/vm_supervisor/metrics.py
+++ b/vm_supervisor/metrics.py
@@ -51,7 +51,6 @@ class ExecutionRecord(Base):
 
     vcpus = Column(Integer, nullable=False)
     memory = Column(Integer, nullable=False)
-    network_tap = Column(String, nullable=True)
 
     def __repr__(self):
         return f""
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index abbd23c8d..95f406141 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -15,6 +15,7 @@
 from .utils import dumps_for_json, create_task_log_exceptions
 from .vm import AlephFirecrackerVM
 from .vm.firecracker_microvm import AlephFirecrackerResources
+from .network.interfaces import TapInterface
 
 logger = logging.getLogger(__name__)
 
@@ -100,7 +101,9 @@ async def prepare(self):
         self.times.prepared_at = datetime.now()
         self.resources = resources
 
-    async def create(self, vm_id: int) -> AlephFirecrackerVM:
+    async def create(
+        self, vm_id: int, tap_interface: TapInterface
+    ) -> AlephFirecrackerVM:
         if not self.resources:
             raise ValueError("Execution resources must be configured first")
         self.times.starting_at = datetime.now()
@@ -110,6 +113,7 @@ async def create(self, vm_id: int) -> AlephFirecrackerVM:
             resources=self.resources,
             enable_networking=self.program.environment.internet,
             hardware_resources=self.program.resources,
+            tap_interface=tap_interface,
         )
         try:
             await vm.setup()
@@ -229,7 +233,6 @@ async def record_usage(self):
                     io_write_bytes=pid_info["process"]["io_counters"][3],
                     vcpus=self.vm.hardware_resources.vcpus,
                     memory=self.vm.hardware_resources.memory,
-                    network_tap=self.vm.fvm.network_tap,
                 )
             )
         else:
@@ -251,7 +254,6 @@ async def record_usage(self):
                     io_write_bytes=None,
                     vcpus=self.vm.hardware_resources.vcpus,
                     memory=self.vm.hardware_resources.memory,
-                    network_tap=self.vm.fvm.network_tap,
                 )
             )
 
diff --git a/vm_supervisor/network/__init__.py b/vm_supervisor/network/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/vm_supervisor/network/firewall.py b/vm_supervisor/network/firewall.py
new file mode 100644
index 000000000..5c0818b26
--- /dev/null
+++ b/vm_supervisor/network/firewall.py
@@ -0,0 +1,401 @@
+import json
+from typing import Dict, List
+import logging
+
+from nftables import Nftables
+from functools import lru_cache
+
+from ..conf import settings
+from .interfaces import TapInterface
+
+logger = logging.getLogger(__name__)
+
+
+@lru_cache()
+def get_customized_nftables() -> Nftables:
+    nft = Nftables()
+    nft.set_json_output(True)
+    nft.set_stateless_output(True)
+    nft.set_service_output(False)
+    nft.set_reversedns_output(False)
+    nft.set_numeric_proto_output(True)
+    return nft
+
+
+def execute_json_nft_commands(commands: List[Dict]) -> int:
+    """Executes a list of nftables commands, and returns the exit status"""
+    nft = get_customized_nftables()
+    commands_dict = {"nftables": commands}
+    try:
+        logger.debug("Validating nftables rules")
+        nft.json_validate(commands_dict)
+    except Exception as e:
+        logger.error(f"Failed to verify nftables rules: {e}")
+
+    logger.debug("Inserting nftables rules")
+    return_code, output, error = nft.json_cmd(commands_dict)
+    if return_code != 0:
+        logger.error(f"Failed to add nftables rules: {error}")
+
+    return return_code
+
+
+def get_existing_nftables_ruleset() -> Dict:
+    """Retrieves the full nftables ruleset and returns it"""
+    nft = get_customized_nftables()
+    return_code, output, error = nft.cmd("list ruleset")
+
+    if return_code != 0:
+        logger.error(f"Unable to get nftables ruleset: {error}")
+
+    nft_ruleset = json.loads(output)
+    return nft_ruleset
+
+
+def get_base_chains_for_hook(hook: str, family: str = "ip") -> List:
+    """Looks through the nftables ruleset and creates a list of
+    all chains that are base chains for the specified hook"""
+    nft_ruleset = get_existing_nftables_ruleset()
+    chains = []
+
+    for entry in nft_ruleset["nftables"]:
+        if (
+            not isinstance(entry, dict)
+            or "chain" not in entry
+            or "family" not in entry["chain"]
+            or entry["chain"]["family"] != family
+            or "hook" not in entry["chain"]
+            or entry["chain"]["hook"] != hook
+        ):
+            # Ignoring all entries that are not a base chain.
+            continue
+
+        chains.append(entry)
+
+    return chains
+
+
+def get_table_for_hook(hook: str, family: str = "ip") -> str:
+    chains = get_base_chains_for_hook(hook, family)
+    table = chains.pop()["chain"]["table"]
+    return table
+
+
+def check_if_table_exists(family: str, table: str) -> bool:
+    """Checks whether the specified table exists in the nftables ruleset"""
+    nft_ruleset = get_existing_nftables_ruleset()
+    for entry in nft_ruleset["nftables"]:
+        if (
+            isinstance(entry, dict)
+            and "table" in entry
+            and entry["family"] == family
+            and entry["name"] == table
+        ):
+            return True
+    return False
+
+
+def initialize_nftables() -> None:
+    """Creates basic chains and rules in the nftables ruleset to build on further.
+    Additionally, stores some information in the class for later use."""
+    commands: List[Dict] = []
+    base_chains: Dict[str, Dict[str, str]] = {
+        "postrouting": {},
+        "forward": {},
+    }
+    for hook in base_chains:
+        chains = get_base_chains_for_hook(hook)
+        if len(chains) == 0:
+            table = "nat" if hook == "postrouting" else "filter"
+            chain = "POSTROUTING" if hook == "postrouting" else "FORWARD"
+            prio = 100 if hook == "postrouting" else 0
+            if not check_if_table_exists("ip", table):
+                commands.append({"add": {"table": {"family": "ip", "name": table}}})
+            new_chain = {
+                "chain": {
+                    "family": "ip",
+                    "table": table,
+                    "name": chain,
+                    "type": table,
+                    "hook": hook,
+                    "prio": prio,
+                }
+            }
+            commands.append({"add": new_chain})
+            chains.append(new_chain)
+        elif len(chains) > 1:
+            raise NotImplementedError(
+                f"Multiple base chains for an nftables basechain are not supported: {hook}"
+            )
+        base_chains[hook] = chains.pop()["chain"]
+
+    add_chain(
+        "ip",
+        base_chains["postrouting"]["table"],
+        f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-nat",
+    )
+    commands.append(
+        {
+            "add": {
+                "rule": {
+                    "family": "ip",
+                    "table": base_chains["postrouting"]["table"],
+                    "chain": base_chains["postrouting"]["name"],
+                    "expr": [
+                        {
+                            "jump": {
+                                "target": f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-nat"
+                            }
+                        }
+                    ],
+                }
+            }
+        }
+    )
+
+    add_chain(
+        "ip",
+        base_chains["forward"]["table"],
+        f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-filter",
+    )
+    commands.append(
+        {
+            "add": {
+                "rule": {
+                    "family": "ip",
+                    "table": base_chains["forward"]["table"],
+                    "chain": base_chains["forward"]["name"],
+                    "expr": [
+                        {
+                            "jump": {
+                                "target": f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-filter"
+                            }
+                        }
+                    ],
+                }
+            }
+        }
+    )
+    commands.append(
+        {
+            "add": {
+                "rule": {
+                    "family": "ip",
+                    "table": base_chains["forward"]["table"],
+                    "chain": f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-filter",
+                    "expr": [
+                        {
+                            "match": {
+                                "op": "in",
+                                "left": {"ct": {"key": "state"}},
+                                "right": ["related", "established"],
+                            }
+                        },
+                        {"accept": None},
+                    ],
+                }
+            }
+        }
+    )
+
+    execute_json_nft_commands(commands)
+    return
+
+
+def teardown_nftables() -> None:
+    """Removes all of this project's related rules in the nftables ruleset."""
+    logger.debug("Tearing down nftables setup")
+    remove_chain(f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-nat")
+    remove_chain(f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-filter")
+    return
+
+
+def add_chain(family: str, table: str, name: str) -> int:
+    """Helper function to quickly create a new chain in the nftables ruleset
+    Returns the exit code from executing the nftables commands"""
+    commands = [
+        {
+            "add": {
+                "chain": {
+                    "family": family,
+                    "table": table,
+                    "name": name,
+                }
+            }
+        }
+    ]
+    return execute_json_nft_commands(commands)
+
+
+def remove_chain(name: str) -> int:
+    """Removes all rules that jump to the chain, and then removes the chain itself.
+    Returns the exit code from executing the nftables commands"""
+    nft_ruleset = get_existing_nftables_ruleset()
+    commands = []
+    remove_chain_commands = []
+
+    for entry in nft_ruleset["nftables"]:
+        if (
+            isinstance(entry, dict)
+            and "rule" in entry
+            and "expr" in entry["rule"]
+            and "jump" in entry["rule"]["expr"][0]
+            and entry["rule"]["expr"][0]["jump"]["target"] == name
+        ):
+            commands.append(
+                {
+                    "delete": {
+                        "rule": {
+                            "family": entry["rule"]["family"],
+                            "table": entry["rule"]["table"],
+                            "chain": entry["rule"]["chain"],
+                            "handle": entry["rule"]["handle"],
+                        }
+                    }
+                }
+            )
+        elif (
+            isinstance(entry, dict)
+            and "chain" in entry
+            and entry["chain"]["name"] == name
+        ):
+            remove_chain_commands.append(
+                {
+                    "delete": {
+                        "chain": {
+                            "family": entry["chain"]["family"],
+                            "table": entry["chain"]["table"],
+                            "name": entry["chain"]["name"],
+                        }
+                    }
+                }
+            )
+
+    commands += remove_chain_commands
+    return execute_json_nft_commands(commands)
+
+
+def add_postrouting_chain(name: str) -> int:
+    """Adds a chain and creates a rule from the base chain with the postrouting hook.
+    Returns the exit code from executing the nftables commands"""
+    table = get_table_for_hook("postrouting")
+    add_chain("ip", table, name)
+    command = [
+        {
+            "add": {
+                "rule": {
+                    "family": "ip",
+                    "table": table,
+                    "chain": f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-nat",
+                    "expr": [{"jump": {"target": name}}],
+                }
+            }
+        }
+    ]
+    return execute_json_nft_commands(command)
+
+
+def add_forward_chain(name: str) -> int:
+    """Adds a chain and creates a rule from the base chain with the forward hook.
+    Returns the exit code from executing the nftables commands"""
+    table = get_table_for_hook("forward")
+    add_chain("ip", table, name)
+    command = [
+        {
+            "add": {
+                "rule": {
+                    "family": "ip",
+                    "table": table,
+                    "chain": f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-filter",
+                    "expr": [{"jump": {"target": name}}],
+                }
+            }
+        }
+    ]
+    return execute_json_nft_commands(command)
+
+
+def add_masquerading_rule(vm_id: int, interface: TapInterface) -> int:
+    """Creates a rule for the VM with the specified id to allow outbound traffic to be masqueraded (NAT)
+    Returns the exit code from executing the nftables commands"""
+    table = get_table_for_hook("postrouting")
+    command = [
+        {
+            "add": {
+                "rule": {
+                    "family": "ip",
+                    "table": table,
+                    "chain": f"{settings.NFTABLES_CHAIN_PREFIX}-vm-nat-{vm_id}",
+                    "expr": [
+                        {
+                            "match": {
+                                "op": "==",
+                                "left": {"meta": {"key": "iifname"}},
+                                "right": interface.device_name,
+                            }
+                        },
+                        {
+                            "match": {
+                                "op": "==",
+                                "left": {"meta": {"key": "oifname"}},
+                                "right": settings.NETWORK_INTERFACE,
+                            }
+                        },
+                        {"masquerade": None},
+                    ],
+                }
+            }
+        }
+    ]
+
+    return execute_json_nft_commands(command)
+
+
+def add_forward_rule_to_external(vm_id: int, interface: TapInterface) -> int:
+    """Creates a rule for the VM with the specified id to allow outbound traffic
+    Returns the exit code from executing the nftables commands"""
+    table = get_table_for_hook("forward")
+    command = [
+        {
+            "add": {
+                "rule": {
+                    "family": "ip",
+                    "table": table,
+                    "chain": f"{settings.NFTABLES_CHAIN_PREFIX}-vm-filter-{vm_id}",
+                    "expr": [
+                        {
+                            "match": {
+                                "op": "==",
+                                "left": {"meta": {"key": "iifname"}},
+                                "right": interface.device_name,
+                            }
+                        },
+                        {
+                            "match": {
+                                "op": "==",
+                                "left": {"meta": {"key": "oifname"}},
+                                "right": settings.NETWORK_INTERFACE,
+                            }
+                        },
+                        {"accept": None},
+                    ],
+                }
+            }
+        }
+    ]
+
+    return execute_json_nft_commands(command)
+
+
+def setup_nftables_for_vm(vm_id: int, interface: TapInterface) -> None:
+    """Sets up chains for filter and nat purposes specific to this VM, and makes sure those chains are jumped to"""
+    add_postrouting_chain(f"{settings.NFTABLES_CHAIN_PREFIX}-vm-nat-{vm_id}")
+    add_forward_chain(f"{settings.NFTABLES_CHAIN_PREFIX}-vm-filter-{vm_id}")
+    add_masquerading_rule(vm_id, interface)
+    add_forward_rule_to_external(vm_id, interface)
+
+
+def teardown_nftables_for_vm(vm_id: int) -> None:
+    """Remove all nftables rules related to the specified VM"""
+    remove_chain(f"{settings.NFTABLES_CHAIN_PREFIX}-vm-nat-{vm_id}")
+    remove_chain(f"{settings.NFTABLES_CHAIN_PREFIX}-vm-filter-{vm_id}")
diff --git a/vm_supervisor/network/hostnetwork.py b/vm_supervisor/network/hostnetwork.py
new file mode 100644
index 000000000..a190b7565
--- /dev/null
+++ b/vm_supervisor/network/hostnetwork.py
@@ -0,0 +1,63 @@
+import logging
+
+from .firewall import initialize_nftables, teardown_nftables, setup_nftables_for_vm
+from .interfaces import TapInterface
+from .ipaddresses import IPv4NetworkWithInterfaces
+
+logger = logging.getLogger(__name__)
+
+
+def get_ipv4_forwarding_state() -> int:
+    """Reads the current ipv4 forwarding setting from the hosts, converts it to int and returns it"""
+    with open("/proc/sys/net/ipv4/ip_forward") as f:
+        return int(f.read())
+
+
+class Network:
+    ipv4_forward_state_before_setup: int
+    address_pool: IPv4NetworkWithInterfaces = IPv4NetworkWithInterfaces("172.16.0.0/12")
+    network_size: int
+    external_interface: str
+
+    def get_network_for_tap(self, vm_id: int) -> IPv4NetworkWithInterfaces:
+        subnets = list(self.address_pool.subnets(new_prefix=self.network_size))
+        return subnets[vm_id]
+
+    def enable_ipv4_forwarding(self) -> None:
+        """Saves the hosts IPv4 forwarding state, and if it was disabled, enables it"""
+        logger.debug(f"Enabling IPv4 forwarding")
+        self.ipv4_forward_state_before_setup = get_ipv4_forwarding_state()
+        if not self.ipv4_forward_state_before_setup:
+            with open("/proc/sys/net/ipv4/ip_forward", "w") as f:
+                f.write("1")
+
+    def reset_ipv4_forwarding_state(self) -> None:
+        """Returns the hosts IPv4 forwarding state how it was before we enabled it"""
+        logger.debug("Resetting IPv4 forwarding state to state before we enabled it")
+        if self.ipv4_forward_state_before_setup != get_ipv4_forwarding_state():
+            with open("/proc/sys/net/ipv4/ip_forward", "w") as f:
+                f.write(str(self.ipv4_forward_state_before_setup))
+
+    def __init__(self, vm_address_pool_range: str, vm_network_size: int, external_interface: str) -> None:
+        """Sets up the Network class with some information it needs so future function calls work as expected"""
+        self.address_pool = IPv4NetworkWithInterfaces(vm_address_pool_range)
+        if not self.address_pool.is_private:
+            logger.warning(
+                f"Using a network range that is not private: {self.address_pool}"
+            )
+        self.network_size = vm_network_size
+        self.external_interface = external_interface
+        self.enable_ipv4_forwarding()
+        initialize_nftables()
+
+    def teardown(self) -> None:
+        teardown_nftables()
+        self.reset_ipv4_forwarding_state()
+
+    async def create_tap(self, vm_id: int) -> TapInterface:
+        """ Create TAP interface to be used by VM
+        """
+        interface = TapInterface(f"vmtap{vm_id}", self.get_network_for_tap(vm_id))
+        await interface.create()
+        setup_nftables_for_vm(vm_id, interface)
+        return interface
diff --git a/vm_supervisor/network/interfaces.py b/vm_supervisor/network/interfaces.py
new file mode 100644
index 000000000..71b6b65c0
--- /dev/null
+++ b/vm_supervisor/network/interfaces.py
@@ -0,0 +1,51 @@
+import asyncio
+from ipaddress import IPv4Interface
+from subprocess import run
+import logging
+
+from .ipaddresses import IPv4NetworkWithInterfaces
+
+logger = logging.getLogger(__name__)
+
+
+class TapInterface:
+    device_name: str
+    ip_network: IPv4NetworkWithInterfaces
+
+    def __init__(
+        self, device_name: str, ip_network: IPv4NetworkWithInterfaces
+    ):
+        self.device_name: str = device_name
+        self.ip_network: IPv4NetworkWithInterfaces = ip_network
+
+    @property
+    def guest_ip(self) -> IPv4Interface:
+        return self.ip_network[2]
+
+    @property
+    def host_ip(self) -> IPv4Interface:
+        return self.ip_network[1]
+
+    async def create(self):
+        logger.debug("Create network interface")
+
+        run(["/usr/bin/ip", "tuntap", "add", self.device_name, "mode", "tap"])
+        run(
+            [
+                "/usr/bin/ip",
+                "addr",
+                "add",
+                str(self.host_ip.with_prefixlen),
+                "dev",
+                self.device_name,
+            ]
+        )
+        run(["/usr/bin/ip", "link", "set", self.device_name, "up"])
+        logger.debug(f"Network interface created: {self.device_name}")
+
+    async def delete(self) -> None:
+        """Asks the firewall to teardown any rules for the VM with id provided.
+        Then removes the interface from the host."""
+        logger.debug(f"Removing interface {self.device_name}")
+        await asyncio.sleep(0.1)  # Avoids Device/Resource busy bug
+        run(["ip", "tuntap", "del", self.device_name, "mode", "tap"])
diff --git a/vm_supervisor/network/ipaddresses.py b/vm_supervisor/network/ipaddresses.py
new file mode 100644
index 000000000..1d007c57a
--- /dev/null
+++ b/vm_supervisor/network/ipaddresses.py
@@ -0,0 +1,23 @@
+from ipaddress import IPv4Network, IPv4Interface
+from typing import Iterable
+
+
+class IPv4NetworkWithInterfaces(IPv4Network):
+    def hosts(self) -> Iterable[IPv4Interface]:
+        network = int(self.network_address)
+        broadcast = int(self.broadcast_address)
+        for x in range(network + 1, broadcast):
+            yield IPv4Interface((x, self.prefixlen))
+
+    def __getitem__(self, n) -> IPv4Interface:
+        network = int(self.network_address)
+        broadcast = int(self.broadcast_address)
+        if n >= 0:
+            if network + n > broadcast:
+                raise IndexError("address out of range")
+            return IPv4Interface((network + n, self.prefixlen))
+        else:
+            n += 1
+            if broadcast + n < network:
+                raise IndexError("address out of range")
+            return IPv4Interface((broadcast + n, self.prefixlen))
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index cfba99869..cddb73c52 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -6,6 +6,7 @@
 
 from .conf import settings
 from .models import VmHash, VmExecution
+from vm_supervisor.network.hostnetwork import Network
 
 logger = logging.getLogger(__name__)
 
@@ -22,10 +23,16 @@ class VmPool:
     counter: int  # Used to provide distinct ids to network interfaces
     executions: Dict[VmHash, VmExecution]
     message_cache: Dict[str, ProgramMessage] = {}
+    network: Optional[Network]
 
     def __init__(self):
         self.counter = settings.START_ID_INDEX
         self.executions = {}
+        self.network = Network(
+            vm_address_pool_range=settings.IPV4_ADDRESS_POOL,
+            vm_network_size=settings.IPV4_NETWORK_SIZE,
+            external_interface=settings.NETWORK_INTERFACE,
+        ) if settings.ALLOW_VM_NETWORKING else None
 
     async def create_a_vm(
         self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent
@@ -35,7 +42,9 @@ async def create_a_vm(
         self.executions[vm_hash] = execution
         await execution.prepare()
         vm_id = self.get_unique_vm_id()
-        await execution.create(vm_id=vm_id)
+
+        tap_interface = await self.network.create_tap(vm_id)
+        await execution.create(vm_id=vm_id, tap_interface=tap_interface)
         return execution
 
     def get_unique_vm_id(self) -> int:
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 83259c61a..84e42186f 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -84,9 +84,13 @@ def run():
     engine = metrics.setup_engine()
     metrics.create_tables(engine)
 
-    if settings.WATCH_FOR_MESSAGES:
-        app.on_startup.append(start_watch_for_messages_task)
-        app.on_cleanup.append(stop_watch_for_messages_task)
-        app.on_cleanup.append(stop_all_vms)
-
-    web.run_app(app, host=settings.SUPERVISOR_HOST, port=settings.SUPERVISOR_PORT)
+    try:
+        if settings.WATCH_FOR_MESSAGES:
+            app.on_startup.append(start_watch_for_messages_task)
+            app.on_cleanup.append(stop_watch_for_messages_task)
+            app.on_cleanup.append(stop_all_vms)
+
+        web.run_app(app, host=settings.SUPERVISOR_HOST, port=settings.SUPERVISOR_PORT)
+    finally:
+        if settings.ALLOW_VM_NETWORKING:
+            pool.network.teardown()
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index 08aad0082..3c247af4d 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -3,7 +3,7 @@
 import logging
 from base64 import b32decode, b16encode
 from dataclasses import is_dataclass, asdict as dataclass_as_dict
-from typing import Any, Optional, Coroutine, Tuple, Iterable
+from typing import Any, Optional, Coroutine
 
 import aiodns
 
@@ -51,53 +51,3 @@ async def run_and_log_exception(coro: Coroutine):
 def create_task_log_exceptions(coro: Coroutine, *, name=None):
     """Ensure that exceptions running in coroutines are logged."""
     return asyncio.create_task(run_and_log_exception(coro), name=name)
-
-
-def ipstr_to_int(ip_string: str) -> Tuple[int, int]:
-    """Convert an IP address string with subnet mask to an integer
-    representation of the IP and the mask separately.
-    """
-    ip, mask = ip_string.split("/")
-    ip_int = sum(
-        int(octet) * 256**idx for idx, octet in enumerate(reversed(ip.split(".")))
-    )
-    return ip_int, int(mask)
-
-
-def int_to_ipstr(ip_int: int, mask: int) -> str:
-    """Converts an integer representation of an IP address and a subnetmask
-    and turns it into a string representation
-    """
-    ip_integers: Iterable[int] = (
-        (ip_int >> (8 * i)) & 0xFF for i in reversed(range(4))
-    )
-    ip_string: str = ".".join(str(i) for i in ip_integers)
-    return f"{ip_string}/{mask}"
-
-
-def get_ip_addresses(
-    vm_id: int, address_pool: str, ip_network_size: int
-) -> Tuple[str, str]:
-    """Calculates the host and guest ip from vm_id and returns it as their string representations with subnetmask"""
-    network_pool, pool_size = ipstr_to_int(address_pool)
-    pool_netmask = 0xFFFFFFFF << 32 - pool_size
-    network_part = vm_id << 32 - ip_network_size
-    network_part_mask = 2 ** (ip_network_size - pool_size) - 1 << 32 - ip_network_size
-    host = 1
-    guest = 2
-    hosts_mask = 2 ** (32 - ip_network_size) - 1
-
-    host_ip = (
-        (network_pool & pool_netmask)
-        | (network_part & network_part_mask)
-        | (host & hosts_mask)
-    )
-    guest_ip = (
-        (network_pool & pool_netmask)
-        | (network_part & network_part_mask)
-        | (guest & hosts_mask)
-    )
-
-    return int_to_ipstr(host_ip, ip_network_size), int_to_ipstr(
-        guest_ip, ip_network_size
-    )
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 434bddeac..cfbb0afaf 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -12,8 +12,6 @@
 
 import msgpack
 
-from ..utils import get_ip_addresses
-
 try:
     import psutil as psutil
 except ImportError:
@@ -34,6 +32,8 @@
 from guest_api.__main__ import run_guest_api
 from ..conf import settings
 from ..storage import get_code_path, get_runtime_path, get_data_path, get_volume_path
+from ..network.interfaces import TapInterface
+from ..network.firewall import teardown_nftables_for_vm
 
 logger = logging.getLogger(__name__)
 set_start_method("spawn")
@@ -213,8 +213,7 @@ class AlephFirecrackerVM:
     hardware_resources: MachineResources
     fvm: Optional[MicroVM] = None
     guest_api_process: Optional[Process] = None
-    host_ip: Optional[str] = None
-    guest_ip: Optional[str] = None
+    tap_interface: Optional[TapInterface] = None
 
     def __init__(
         self,
@@ -224,6 +223,7 @@ def __init__(
         enable_networking: bool = False,
         enable_console: Optional[bool] = None,
         hardware_resources: MachineResources = MachineResources(),
+        tap_interface: Optional[TapInterface] = None,
     ):
         self.vm_id = vm_id
         self.vm_hash = vm_hash
@@ -233,6 +233,7 @@ def __init__(
             enable_console = settings.PRINT_SYSTEM_LOGS
         self.enable_console = enable_console
         self.hardware_resources = hardware_resources
+        self.tap_interface = tap_interface
 
     def to_dict(self):
         if self.fvm.proc and psutil:
@@ -264,19 +265,13 @@ def to_dict(self):
     async def setup(self):
         logger.debug("setup started")
         await setfacl()
-        host_ip, guest_ip = get_ip_addresses(
-            self.vm_id,
-            address_pool=settings.IPV4_ADDRESS_POOL,
-            ip_network_size=settings.IPV4_NETWORK_SIZE,
-        )
+
         fvm = MicroVM(
             vm_id=self.vm_id,
             firecracker_bin_path=settings.FIRECRACKER_PATH,
             use_jailer=settings.USE_JAILER,
             jailer_bin_path=settings.JAILER_PATH,
             init_timeout=settings.INIT_TIMEOUT,
-            host_ip=host_ip,
-            guest_ip=guest_ip,
         )
         fvm.prepare_jailer()
 
@@ -311,10 +306,7 @@ async def setup(self):
             vsock=Vsock(),
             network_interfaces=[
                 NetworkInterface(
-                    iface_id="eth0",
-                    host_dev_name=await fvm.create_network_interface(
-                        interface=settings.NETWORK_INTERFACE
-                    ),
+                    iface_id="eth0", host_dev_name=self.tap_interface.device_name
                 )
             ]
             if self.enable_networking
@@ -329,6 +321,8 @@ async def setup(self):
             self.fvm = fvm
         except Exception:
             await fvm.teardown()
+            teardown_nftables_for_vm(self.vm_id)
+            await self.tap_interface.delete()
             raise
 
     async def start(self):
@@ -395,8 +389,10 @@ async def configure(self):
 
         reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
         config = ConfigurationPayload(
-            ip=self.fvm.guest_ip if self.enable_networking else None,
-            route=self.fvm.host_ip if self.enable_networking else None,
+            ip=self.tap_interface.guest_ip.with_prefixlen
+            if self.enable_networking
+            else None,
+            route=str(self.tap_interface.host_ip) if self.enable_networking else None,
             dns_servers=settings.DNS_NAMESERVERS,
             code=code,
             encoding=self.resources.code_encoding,
@@ -439,6 +435,8 @@ async def stop_guest_api(self):
     async def teardown(self):
         if self.fvm:
             await self.fvm.teardown()
+            teardown_nftables_for_vm(self.vm_id)
+            await self.tap_interface.delete()
         await self.stop_guest_api()
 
     async def run_code(

From c0bac790ce2f3a2a992a1d5900f17608056b3aad Mon Sep 17 00:00:00 2001
From: Bonjour Internet 
Date: Mon, 21 Nov 2022 17:52:00 +0100
Subject: [PATCH 346/990] doc: Add docstrings for /raise & /crash

---
 examples/example_fastapi/main.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index 652e9853a..4ad5f108f 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -149,11 +149,13 @@ class CustomError(Exception):
 
 @app.get("/raise")
 def raise_error():
+    """Raises an error to check that the init handles it properly without crashing"""
     raise CustomError("Whoops")
 
 
 @app.get("/crash")
 def crash():
+    """Crash the entire VM in order to check that the supervisor can handle it"""
     sys.exit(1)
 
 

From a1fcab6a542070ce8dec936a5d077b5386838ab5 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 5 Dec 2022 22:35:32 +0100
Subject: [PATCH 347/990] Fix: Curl did not error when test failed.

Curl requires the `--fail` argument in order to return a code different than zero and raise and issue in the action.
---
 .github/workflows/test-on-droplet.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-on-droplet.yml b/.github/workflows/test-on-droplet.yml
index 0d4e6a656..9588d09f8 100644
--- a/.github/workflows/test-on-droplet.yml
+++ b/.github/workflows/test-on-droplet.yml
@@ -65,8 +65,8 @@ jobs:
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           
           sleep 3
-          curl --retry 5 "http://${DROPLET_IPV4}:4020/about/usage/system"
-          curl --retry 5 "http://${DROPLET_IPV4}:4020/status/check/fastapi"
+          curl --retry 5 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
+          curl --retry 5 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
 
       - name: Cleanup
         if: always()

From bc27cec1c3cdeb7cbabeaec4cbd00290c57473c9 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 31 Jan 2023 01:00:01 +0100
Subject: [PATCH 348/990] Fix: Network route should not contain IP mask

This was breaking the `ip route add` command in the initialisation of VMs.
---
 .github/workflows/test-on-droplet.yml      |  1 +
 runtimes/aleph-alpine-3.13-python/init1.py |  7 ++++++-
 vm_supervisor/vm/firecracker_microvm.py    | 12 ++++++++----
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test-on-droplet.yml b/.github/workflows/test-on-droplet.yml
index 9588d09f8..e518a9984 100644
--- a/.github/workflows/test-on-droplet.yml
+++ b/.github/workflows/test-on-droplet.yml
@@ -72,3 +72,4 @@ jobs:
         if: always()
         run: |
           doctl compute droplet delete -f aleph-vm-ci
+
diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py
index d559f171e..45ff61f9d 100644
--- a/runtimes/aleph-alpine-3.13-python/init1.py
+++ b/runtimes/aleph-alpine-3.13-python/init1.py
@@ -123,7 +123,12 @@ def setup_network(
     system("ip addr add 127.0.0.1/8 dev lo brd + scope host")
     system("ip addr add ::1/128 dev lo")
     system("ip link set lo up")
-    system(f"ip addr add {ip} dev eth0")
+    if "/" in ip:
+        # Forward compatibility with future supervisors that pass the mask with the IP.
+        system(f"ip addr add {ip} dev eth0")
+    else:
+        logger.warning("Not passing the mask with the IP is deprecated and will be unsupported")
+        system(f"ip addr add {ip}/24 dev eth0")
     system("ip link set eth0 up")
 
     if route:
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index cfbb0afaf..053ddf949 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -388,11 +388,15 @@ async def configure(self):
             ]
 
         reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
+
+        # The ip and route should not contain the network mask in order to maintain
+        # compatibility with the existing runtimes.
+        ip = self.tap_interface.guest_ip.with_prefixlen.split('/', 1)[0]
+        route = str(self.tap_interface.host_ip).split('/', 1)[0]
+
         config = ConfigurationPayload(
-            ip=self.tap_interface.guest_ip.with_prefixlen
-            if self.enable_networking
-            else None,
-            route=str(self.tap_interface.host_ip) if self.enable_networking else None,
+            ip=ip if self.enable_networking else None,
+            route=route if self.enable_networking else None,
             dns_servers=settings.DNS_NAMESERVERS,
             code=code,
             encoding=self.resources.code_encoding,

From 37a9bc16cb3a6622870f4d86b30d46799e926f3e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 31 Jan 2023 12:01:31 +0100
Subject: [PATCH 349/990] Fix: Required directories were not created.

---
 vm_supervisor/conf.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index ea0709b2d..63eee8944 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -186,6 +186,9 @@ def setup(self):
         os.makedirs(self.CODE_CACHE, exist_ok=True)
         os.makedirs(self.RUNTIME_CACHE, exist_ok=True)
         os.makedirs(self.DATA_CACHE, exist_ok=True)
+        os.makedirs(self.EXECUTION_ROOT, exist_ok=True)
+        os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True)
+        os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True)
 
         if self.DNS_NAMESERVERS is None and self.DNS_RESOLUTION:
             if self.DNS_RESOLUTION == DnsResolver.resolv_conf:

From cb2547df514fe40a44b03395e69f0f5ffde4348f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 1 Feb 2023 20:18:12 +0100
Subject: [PATCH 350/990] Fix: Droplet was slow, took long time

---
 .github/workflows/test-on-droplet.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-on-droplet.yml b/.github/workflows/test-on-droplet.yml
index e518a9984..bb2c5965a 100644
--- a/.github/workflows/test-on-droplet.yml
+++ b/.github/workflows/test-on-droplet.yml
@@ -31,9 +31,14 @@ jobs:
 
       - name: Create the Droplet
         run: |
-          doctl compute droplet create --image debian-11-x64 \
-          --size s-1vcpu-1gb --region ams3 aleph-vm-ci \
-          --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5
+          doctl compute droplet create \
+          --image debian-11-x64 \
+          --size c-2 \
+          --region fra1 \
+          --vpc-uuid 8c422d04-5dfa-4eca-add7-1e41b5f60d39 \
+          --enable-ipv6 \
+          --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
+          aleph-vm-ci
 
       - name: Build Debian Package
         run: |

From f6038d75e99f23246d2d3c0925cadafd834a65b4 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 1 Feb 2023 20:55:51 +0100
Subject: [PATCH 351/990] Fix: CI tests with curl had not timeout

---
 .github/workflows/test-on-droplet.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-on-droplet.yml b/.github/workflows/test-on-droplet.yml
index bb2c5965a..315459a22 100644
--- a/.github/workflows/test-on-droplet.yml
+++ b/.github/workflows/test-on-droplet.yml
@@ -70,8 +70,8 @@ jobs:
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           
           sleep 3
-          curl --retry 5 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
-          curl --retry 5 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
+          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
+          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
 
       - name: Cleanup
         if: always()

From c8a2e43c64bbce9df8436c98218f23cf0a0609d1 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 31 Jan 2023 01:03:16 +0100
Subject: [PATCH 352/990] Fix: Docker containers could not run in runtime

Solution: Mount cgroups, configure iptables and install dependencies
---
 runtimes/aleph-alpine-3.13-python/init0.sh           | 9 +++++++++
 runtimes/aleph-debian-11-python/create_disk_image.sh | 8 +++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/runtimes/aleph-alpine-3.13-python/init0.sh b/runtimes/aleph-alpine-3.13-python/init0.sh
index 914debaa8..8eb1b62bf 100644
--- a/runtimes/aleph-alpine-3.13-python/init0.sh
+++ b/runtimes/aleph-alpine-3.13-python/init0.sh
@@ -29,6 +29,15 @@ mount -t tmpfs run /run -o mode=0755,nosuid,nodev
 mount -t devpts devpts /dev/pts -o mode=0620,gid=5,nosuid,noexec
 mount -t tmpfs shm /dev/shm -omode=1777,nosuid,nodev
 
+# Required by Docker
+cgroupfs-mount
+update-alternatives --set iptables /usr/sbin/iptables-legacy
+update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+
+# Enable the following to force the storage driver used by Docker.
+# See https://docs.docker.com/storage/storagedriver/select-storage-driver/
+#echo '{\n"storage-driver": "overlay2"\n}\n' > /etc/docker/daemon.json
+
 # List block devices
 lsblk
 
diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
index 7fbb6209a..c556b1950 100755
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-11-python/create_disk_image.sh
@@ -23,7 +23,13 @@ apt-get install -y --no-install-recommends --no-install-suggests \
   python3-pip python3-cytoolz python3-pydantic \
   iproute2 unzip \
   nodejs npm \
-  build-essential python3-dev
+  build-essential python3-dev \
+  \
+  docker.io \
+  cgroupfs-mount \
+  nftables \
+  \
+  iputils-ping curl
 
 pip3 install 'fastapi~=0.71.0'
 

From 3e4e194029d43642089d75990d769ae37f39b4f8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 31 Jan 2023 01:50:54 +0100
Subject: [PATCH 353/990] Fix: Some types were not coherent.

---
 runtimes/aleph-alpine-3.13-python/init1.py | 9 +++++----
 vm_connector/conf.py                       | 4 ++--
 vm_supervisor/models.py                    | 4 ++--
 vm_supervisor/resources.py                 | 4 ++--
 vm_supervisor/views.py                     | 2 +-
 vm_supervisor/vm/firecracker_microvm.py    | 2 +-
 6 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py
index 45ff61f9d..c9aa906e0 100644
--- a/runtimes/aleph-alpine-3.13-python/init1.py
+++ b/runtimes/aleph-alpine-3.13-python/init1.py
@@ -30,7 +30,7 @@
 
 logger.debug("Imports finished")
 
-ASGIApplication = NewType("AsgiApplication", Any)
+ASGIApplication = NewType("ASGIApplication", Any)
 
 
 class Encoding(str, Enum):
@@ -171,6 +171,7 @@ def setup_code_asgi(
     sys.path.append("/opt/packages")
 
     logger.debug("Extracting code")
+    app: ASGIApplication
     if encoding == Encoding.squashfs:
         sys.path.append("/opt/code")
         module_name, app_name = entrypoint.split(":", 1)
@@ -178,7 +179,7 @@ def setup_code_asgi(
         module = __import__(module_name)
         for level in module_name.split(".")[1:]:
             module = getattr(module, level)
-        app: ASGIApplication = getattr(module, app_name)
+        app = getattr(module, app_name)
     elif encoding == Encoding.zip:
         # Unzip in /opt and import the entrypoint from there
         if not os.path.exists("/opt/archive.zip"):
@@ -191,12 +192,12 @@ def setup_code_asgi(
         module = __import__(module_name)
         for level in module_name.split(".")[1:]:
             module = getattr(module, level)
-        app: ASGIApplication = getattr(module, app_name)
+        app = getattr(module, app_name)
     elif encoding == Encoding.plain:
         # Execute the code and extract the entrypoint
         locals: Dict[str, Any] = {}
         exec(code, globals(), locals)
-        app: ASGIApplication = locals[entrypoint]
+        app = locals[entrypoint]
     else:
         raise ValueError(f"Unknown encoding '{encoding}'")
     return app
diff --git a/vm_connector/conf.py b/vm_connector/conf.py
index 7c7558bf2..58b5862a3 100644
--- a/vm_connector/conf.py
+++ b/vm_connector/conf.py
@@ -9,8 +9,8 @@
 
 
 class ConnectorSettings(BaseSettings):
-    API_SERVER: Url = "https://official.aleph.cloud"
-    IPFS_SERVER: Url = "https://ipfs.aleph.im/ipfs"
+    API_SERVER: Url = Url("https://official.aleph.cloud")
+    IPFS_SERVER: Url = Url("https://ipfs.aleph.im/ipfs")
     OFFLINE_TEST_MODE: bool = False
 
     def update(self, **kwargs):
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 95f406141..d986e0bce 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -130,7 +130,7 @@ async def create(
     def stop_after_timeout(self, timeout: float = 5.0) -> Optional[Task]:
         if self.persistent:
             logger.debug("VM marked as long running. Ignoring timeout.")
-            return
+            return None
 
         if self.expire_task:
             logger.debug("VM already has a timeout. Extending it.")
@@ -257,7 +257,7 @@ async def record_usage(self):
                 )
             )
 
-    async def run_code(self, scope: dict = None) -> bytes:
+    async def run_code(self, scope: Optional[dict] = None) -> bytes:
         if not self.vm:
             raise ValueError("The VM has not been created yet")
         self.concurrent_runs += 1
diff --git a/vm_supervisor/resources.py b/vm_supervisor/resources.py
index 1f9c06050..1ef237bfa 100644
--- a/vm_supervisor/resources.py
+++ b/vm_supervisor/resources.py
@@ -105,8 +105,8 @@ async def about_system_usage(request: web.Request):
             available_kB=psutil.virtual_memory().available / 1000,
         ),
         disk=DiskUsage(
-            total_kB=psutil.disk_usage(settings.PERSISTENT_VOLUMES_DIR).total // 1000,
-            available_kB=psutil.disk_usage(settings.PERSISTENT_VOLUMES_DIR).free
+            total_kB=psutil.disk_usage(str(settings.PERSISTENT_VOLUMES_DIR)).total // 1000,
+            available_kB=psutil.disk_usage(str(settings.PERSISTENT_VOLUMES_DIR)).free
             // 1000,
         ),
         period=UsagePeriod(
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index e894687d1..229ca7716 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -175,7 +175,7 @@ async def status_check_version(request: web.Request):
 
 def authenticate_api_request(request: web.Request) -> bool:
     """Authenticate an API request to update the VM allocations."""
-    signature: bytes = request.headers.get("X-Auth-Signature").encode()
+    signature: bytes = request.headers.get("X-Auth-Signature", '').encode()
 
     if not signature:
         raise web.HTTPUnauthorized(text="Authentication token is missing")
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 053ddf949..248669cbe 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -445,7 +445,7 @@ async def teardown(self):
 
     async def run_code(
         self,
-        scope: dict = None,
+        scope: Optional[dict] = None,
     ):
         if not self.fvm:
             raise ValueError("MicroVM must be created first")

From a6675cacf008b7c8bd1d6279892076a48b0dd797 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 13 Feb 2023 13:51:25 +0100
Subject: [PATCH 354/990] Fix: Command `systemd-resolve` does not exist in
 Ubuntu 22.04

THe command `resolvectl` however appears to be present in both Ubuntu 20.04 and Ubuntu 22.04.
---
 vm_supervisor/conf.py | 43 ++++++++++++++-----------------------------
 1 file changed, 14 insertions(+), 29 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 63eee8944..f9f1dc82e 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -27,34 +27,19 @@ def etc_resolv_conf_dns_servers():
                 yield ip[0]
 
 
-def systemd_resolved_dns_servers(interface):
-    # Example output format from systemd-resolve --status {interface}:
-    # Link 2 (enp7s0)
-    #       Current Scopes: DNS
-    # DefaultRoute setting: yes
-    #        LLMNR setting: yes
-    # MulticastDNS setting: no
-    #   DNSOverTLS setting: no
-    #       DNSSEC setting: no
-    #     DNSSEC supported: no
-    #   Current DNS Server: 213.133.100.100
-    #          DNS Servers: 213.133.100.100
-    #                       213.133.98.98
-    #                       213.133.99.99
-    #                       2a01:4f8:0:1::add:9898
-    #                       2a01:4f8:0:1::add:1010
-    #                       2a01:4f8:0:1::add:9999
-    output = check_output(["/usr/bin/systemd-resolve", "--status", interface])
-    nameserver_line = False
-    for line in output.split(b"\n"):
-        if b"DNS Servers" in line:
-            nameserver_line = True
-            _, ip = line.decode().split(":", 1)
-            yield ip.strip()
-        elif nameserver_line:
-            ip = line.decode().strip()
-            if ip:
-                yield ip
+def resolvectl_dns_servers(interface):
+    """
+    On Ubuntu 22.04, DNS servers can be queries using `resolvectl dns`.
+     The command `systemd-resolve` used in Ubuntu 20.04 is not found.
+
+    Example output for `resolvectl dns -i eth0`:
+    Link 2 (eth0): 67.207.67.3 67.207.67.2
+
+    """
+    output: bytes = check_output(["/usr/bin/resolvectl", "dns", "-i", interface])
+    link, servers = output.split(b":")
+    for server in servers.split(b" "):
+        yield server.decode().strip()
 
 
 class Settings(BaseSettings):
@@ -196,7 +181,7 @@ def setup(self):
 
             elif self.DNS_RESOLUTION == DnsResolver.resolvectl:
                 self.DNS_NAMESERVERS = list(
-                    systemd_resolved_dns_servers(interface=self.NETWORK_INTERFACE)
+                    resolvectl_dns_servers(interface=self.NETWORK_INTERFACE)
                 )
             else:
                 assert "This should never happen"

From 0d9d98d77d00cd9086b38738a174385467f27e32 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 13 Feb 2023 13:59:02 +0100
Subject: [PATCH 355/990] CI: Build packages and test droplet with Ubuntu 22.04

---
 .github/workflows/build-deb-package.yml       |  5 ++
 ...plet.yml => test-on-droplet-debian-11.yml} |  4 +-
 .../test-on-droplet-ubuntu-22.04.yml          | 81 +++++++++++++++++++
 packaging/Makefile                            | 24 +++++-
 packaging/ubuntu-22.04.dockerfile             | 18 +++++
 5 files changed, 128 insertions(+), 4 deletions(-)
 rename .github/workflows/{test-on-droplet.yml => test-on-droplet-debian-11.yml} (97%)
 create mode 100644 .github/workflows/test-on-droplet-ubuntu-22.04.yml
 create mode 100644 packaging/ubuntu-22.04.dockerfile

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 000b51f6e..663cb878e 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -19,6 +19,7 @@ jobs:
       - run: |
           cd packaging && make all-podman-debian-11 && cd ..
           cd packaging && make all-podman-ubuntu-2004 && cd ..
+          cd packaging && make all-podman-ubuntu-2204 && cd ..
           ls packaging/target
           
       - uses: actions/upload-artifact@v2
@@ -31,6 +32,10 @@ jobs:
           name: aleph-vm.ubuntu-20.04.deb
           path: packaging/target/aleph-vm.ubuntu-20.04.deb
 
+      - uses: actions/upload-artifact@v2
+        with:
+          name: aleph-vm.ubuntu-22.04.deb
+          path: packaging/target/aleph-vm.ubuntu-22.04.deb
 
   build_rootfs:
     name: "Build runtime aleph-debian-11-python"
diff --git a/.github/workflows/test-on-droplet.yml b/.github/workflows/test-on-droplet-debian-11.yml
similarity index 97%
rename from .github/workflows/test-on-droplet.yml
rename to .github/workflows/test-on-droplet-debian-11.yml
index 315459a22..cff18933d 100644
--- a/.github/workflows/test-on-droplet.yml
+++ b/.github/workflows/test-on-droplet-debian-11.yml
@@ -3,8 +3,8 @@ on:
   push
 
 jobs:
-  build_deb:
-    name: "Run in DigitalOcean Droplet"
+  run_debian_11:
+    name: "Run in DigitalOcean Droplet with Debian 11"
     runs-on: ubuntu-latest
     concurrency: droplet-aleph-vm
 
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
new file mode 100644
index 000000000..4516eb1a8
--- /dev/null
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -0,0 +1,81 @@
+name: "Run tests on DigitalOcean Droplet"
+on:
+  push
+
+jobs:
+  run_ubuntu_22_04:
+    name: "Run in DigitalOcean Droplet with Ubuntu 22.04"
+    runs-on: ubuntu-latest
+    concurrency: droplet-aleph-vm
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          # Fetch the whole history for all tags and branches (required for aleph.__version__)
+          fetch-depth: 0
+
+      - name: Install doctl
+        uses: digitalocean/action-doctl@v2
+        with:
+          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+
+      - name: Setup SSH private key
+        run: |
+          mkdir ~/.ssh
+          echo $DIGITALOCEAN_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519
+          chmod 0700 ~/.ssh
+          chmod 0600 ~/.ssh/id_ed25519
+        env:
+          DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }}
+
+      - name: Create the Droplet
+        run: |
+          doctl compute droplet create \
+          --image ubuntu-22-04-x64 \
+          --size c-2 \
+          --region fra1 \
+          --vpc-uuid 8c422d04-5dfa-4eca-add7-1e41b5f60d39 \
+          --enable-ipv6 \
+          --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
+          aleph-vm-ci
+
+      - name: Build Ubuntu Package
+        run: |
+          cd packaging && make all-podman-ubuntu-2204 && cd ..
+          ls packaging/target
+
+      - name: Wait for the system to setup and boot
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
+
+      - name: Install Aleph-VM on the Droplet
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
+          
+          ssh root@${DROPLET_IPV4} "apt-get update"
+          ssh root@${DROPLET_IPV4} "apt-get upgrade -y"
+          ssh root@${DROPLET_IPV4} "apt-get install -y docker.io apparmor-profiles"
+          ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
+          
+          scp packaging/target/aleph-vm.ubuntu-22.04.deb root@${DROPLET_IPV4}:/opt
+          ssh root@${DROPLET_IPV4} "apt install -y /opt/aleph-vm.ubuntu-22.04.deb"
+          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
+          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_DNS_RESOLUTION=resolvectl >> /etc/aleph-vm/supervisor.env"
+          ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
+
+      - name: Test Aleph-VM on the Droplet
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          
+          sleep 3
+          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
+          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
+
+      - name: Cleanup
+        if: always()
+        run: |
+          doctl compute droplet delete -f aleph-vm-ci
+
diff --git a/packaging/Makefile b/packaging/Makefile
index 87563ed20..0ac1b744b 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -16,7 +16,7 @@ debian-package-code:
 	cp ../examples/message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.2.2'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.3.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux
@@ -78,6 +78,17 @@ all-podman-ubuntu-2004: version
 	file target/aleph-vm.deb
 	mv target/aleph-vm.deb target/aleph-vm.ubuntu-20.04.deb
 
+all-podman-ubuntu-2204: version
+	cd .. && podman build -t localhost/aleph-vm-packaging-ubuntu-2204:latest -f ./packaging/ubuntu-22.04.dockerfile .
+	mkdir -p ./target
+	podman run --rm -ti \
+		-w /opt/packaging \
+		-v ./target:/opt/packaging/target \
+		localhost/aleph-vm-packaging-ubuntu-2204:latest \
+		make
+	file target/aleph-vm.deb
+	mv target/aleph-vm.deb target/aleph-vm.ubuntu-22.04.deb
+
 # extract Python requirements from Debian 11 container
 requirements-debian-11: all-podman-debian-11
 	podman run --rm -ti \
@@ -88,7 +99,7 @@ requirements-debian-11: all-podman-debian-11
 		bash -c "/opt/extract_requirements.sh /mnt/requirements-debian-11.txt"
 
 # extract Python requirements from Ubuntu 20.04 container
-requirements-ubuntu-2004: all-podman-ubuntu-2004
+requirements-ubuntu-20.04: all-podman-ubuntu-2004
 	podman run --rm -ti \
 		-v ./target/aleph-vm.ubuntu-20.04.deb:/opt/packaging/target/aleph-vm.deb:ro \
 		-v ./extract_requirements.sh:/opt/extract_requirements.sh:ro \
@@ -96,6 +107,15 @@ requirements-ubuntu-2004: all-podman-ubuntu-2004
 		ubuntu:focal \
 		bash -c "/opt/extract_requirements.sh /mnt/requirements-ubuntu-20.04.txt"
 
+# extract Python requirements from Ubuntu 22.04 container
+requirements-ubuntu-22.04: all-podman-ubuntu-2204
+	podman run --rm -ti \
+		-v ./target/aleph-vm.ubuntu-22.04.deb:/opt/packaging/target/aleph-vm.deb:ro \
+		-v ./extract_requirements.sh:/opt/extract_requirements.sh:ro \
+		-v ./requirements-ubuntu-22.04.txt:/mnt/requirements-ubuntu-22.04.txt \
+		ubuntu:jammy \
+		bash -c "/opt/extract_requirements.sh /mnt/requirements-ubuntu-22.04.txt"
+
 # run on host in order to sign with GPG
 repository-bullseye:
 	cd ./repositories/bullseye && reprepro -Vb . includedeb bullseye ../../target/aleph-vm.debian-11.deb && cd ..
diff --git a/packaging/ubuntu-22.04.dockerfile b/packaging/ubuntu-22.04.dockerfile
new file mode 100644
index 000000000..8c42c8637
--- /dev/null
+++ b/packaging/ubuntu-22.04.dockerfile
@@ -0,0 +1,18 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && apt-get -y upgrade && apt-get install -y \
+    make \
+    git \
+    curl \
+    sudo \
+    python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt
+COPY ../vm_supervisor ./vm_supervisor
+COPY ../guest_api ./guest_api
+COPY ../firecracker ./firecracker
+COPY ../packaging ./packaging
+COPY ../kernels ./kernels
+
+COPY ../examples/ ./examples

From 4e49a1878a0b5c9e0de4de0533694227faea3485 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 13 Feb 2023 14:52:36 +0100
Subject: [PATCH 356/990] Fix: Apt required interaction in workflows

---
 .github/workflows/test-on-droplet-debian-11.yml    | 8 ++++----
 .github/workflows/test-on-droplet-ubuntu-22.04.yml | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test-on-droplet-debian-11.yml b/.github/workflows/test-on-droplet-debian-11.yml
index cff18933d..3f6d46089 100644
--- a/.github/workflows/test-on-droplet-debian-11.yml
+++ b/.github/workflows/test-on-droplet-debian-11.yml
@@ -55,13 +55,13 @@ jobs:
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
           
-          ssh root@${DROPLET_IPV4} "apt-get update"
-          ssh root@${DROPLET_IPV4} "apt-get upgrade -y"
-          ssh root@${DROPLET_IPV4} "apt-get install -y docker.io apparmor-profiles"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
           
           scp packaging/target/aleph-vm.debian-11.deb root@${DROPLET_IPV4}:/opt
-          ssh root@${DROPLET_IPV4} "apt install -y /opt/aleph-vm.debian-11.deb"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.debian-11.deb"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
 
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index 4516eb1a8..4fd67b3fe 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -55,13 +55,13 @@ jobs:
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
           
-          ssh root@${DROPLET_IPV4} "apt-get update"
-          ssh root@${DROPLET_IPV4} "apt-get upgrade -y"
-          ssh root@${DROPLET_IPV4} "apt-get install -y docker.io apparmor-profiles"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
           
           scp packaging/target/aleph-vm.ubuntu-22.04.deb root@${DROPLET_IPV4}:/opt
-          ssh root@${DROPLET_IPV4} "apt install -y /opt/aleph-vm.ubuntu-22.04.deb"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.ubuntu-22.04.deb"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_DNS_RESOLUTION=resolvectl >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"

From 9b95fc181ca61f2eef34074b79fae89db0cdc2dd Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 13 Feb 2023 15:09:05 +0100
Subject: [PATCH 357/990] CI: Many jobs were sequential

---
 .github/workflows/build-deb-package.yml       | 38 +++++++++++++++++--
 .../workflows/test-on-droplet-debian-11.yml   | 12 +++---
 .../test-on-droplet-ubuntu-22.04.yml          | 12 +++---
 3 files changed, 47 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 663cb878e..96f6af021 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -3,7 +3,7 @@ on:
   push
 
 jobs:
-  build_deb:
+  build_deb_debian_11:
     name: "Build Debian Package"
     runs-on: ubuntu-latest
 
@@ -18,8 +18,6 @@ jobs:
 
       - run: |
           cd packaging && make all-podman-debian-11 && cd ..
-          cd packaging && make all-podman-ubuntu-2004 && cd ..
-          cd packaging && make all-podman-ubuntu-2204 && cd ..
           ls packaging/target
           
       - uses: actions/upload-artifact@v2
@@ -27,11 +25,45 @@ jobs:
           name: aleph-vm.debian-11.deb
           path: packaging/target/aleph-vm.debian-11.deb
 
+  build_deb_ubuntu_20_04:
+    name: "Build Debian Package"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Unshallow
+        run: |
+          git fetch --prune --unshallow
+          git describe --tags
+
+      - run: |
+          cd packaging && make all-podman-ubuntu-2004 && cd ..
+          ls packaging/target
+
       - uses: actions/upload-artifact@v2
         with:
           name: aleph-vm.ubuntu-20.04.deb
           path: packaging/target/aleph-vm.ubuntu-20.04.deb
 
+  build_deb_ubuntu_22_04:
+    name: "Build Debian Package"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Unshallow
+        run: |
+          git fetch --prune --unshallow
+          git describe --tags
+
+      - run: |
+          cd packaging && make all-podman-ubuntu-2204 && cd ..
+          ls packaging/target
+
       - uses: actions/upload-artifact@v2
         with:
           name: aleph-vm.ubuntu-22.04.deb
diff --git a/.github/workflows/test-on-droplet-debian-11.yml b/.github/workflows/test-on-droplet-debian-11.yml
index 3f6d46089..5ed2000a0 100644
--- a/.github/workflows/test-on-droplet-debian-11.yml
+++ b/.github/workflows/test-on-droplet-debian-11.yml
@@ -6,7 +6,7 @@ jobs:
   run_debian_11:
     name: "Run in DigitalOcean Droplet with Debian 11"
     runs-on: ubuntu-latest
-    concurrency: droplet-aleph-vm
+    concurrency: droplet-aleph-vm-debian-11
 
     steps:
       - name: Checkout repository
@@ -38,7 +38,7 @@ jobs:
           --vpc-uuid 8c422d04-5dfa-4eca-add7-1e41b5f60d39 \
           --enable-ipv6 \
           --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
-          aleph-vm-ci
+          aleph-vm-ci-debian-11
 
       - name: Build Debian Package
         run: |
@@ -47,12 +47,12 @@ jobs:
 
       - name: Wait for the system to setup and boot
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
 
       - name: Install Aleph-VM on the Droplet
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
           
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
@@ -67,7 +67,7 @@ jobs:
 
       - name: Test Aleph-VM on the Droplet
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           
           sleep 3
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
@@ -76,5 +76,5 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          doctl compute droplet delete -f aleph-vm-ci
+          doctl compute droplet delete -f aleph-vm-ci-debian-11
 
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index 4fd67b3fe..bc2291897 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -6,7 +6,7 @@ jobs:
   run_ubuntu_22_04:
     name: "Run in DigitalOcean Droplet with Ubuntu 22.04"
     runs-on: ubuntu-latest
-    concurrency: droplet-aleph-vm
+    concurrency: droplet-aleph-vm-ubuntu-22-04
 
     steps:
       - name: Checkout repository
@@ -38,7 +38,7 @@ jobs:
           --vpc-uuid 8c422d04-5dfa-4eca-add7-1e41b5f60d39 \
           --enable-ipv6 \
           --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
-          aleph-vm-ci
+          aleph-vm-ci-ubuntu-22-04
 
       - name: Build Ubuntu Package
         run: |
@@ -47,12 +47,12 @@ jobs:
 
       - name: Wait for the system to setup and boot
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-ubuntu-22-04 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
 
       - name: Install Aleph-VM on the Droplet
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-ubuntu-22-04 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
           
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
@@ -68,7 +68,7 @@ jobs:
 
       - name: Test Aleph-VM on the Droplet
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-ubuntu-22-04 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           
           sleep 3
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
@@ -77,5 +77,5 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          doctl compute droplet delete -f aleph-vm-ci
+          doctl compute droplet delete -f aleph-vm-ci-ubuntu-22-04
 

From 76df379dc2ec8685df14ec4453700db86715ee60 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 13 Feb 2023 16:56:21 +0100
Subject: [PATCH 358/990] Fix: Supervisor started without /dev/kvm

Problem: The supervisor could start while the virtualization device /dev/kvm was absent from the system.

Solution: Add a check in the initialization of the configuration for /dev/kvm to be present.
---
 vm_supervisor/conf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index f9f1dc82e..b89d4231b 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -142,6 +142,7 @@ def update(self, **kwargs):
                 raise ValueError(f"Unknown setting '{key}'")
 
     def check(self):
+        assert Path("/dev/kvm").exists(), "KVM not found on `/dev/kvm`."
         assert isfile(self.FIRECRACKER_PATH), f"File not found {self.FIRECRACKER_PATH}"
         assert isfile(self.JAILER_PATH), f"File not found {self.JAILER_PATH}"
         assert isfile(self.LINUX_PATH), f"File not found {self.LINUX_PATH}"

From 673f9d5fb3d83f8c80dd51375f7fdff0cf5531c8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 20 Feb 2023 13:40:40 +0100
Subject: [PATCH 359/990] Fix: CI Ubuntu droplet fails due to apt already
 running.

The droplet appears to automatically run upon installation.

Solution: Wait for apt to finish working and release its lock before calling it.
---
 .github/workflows/test-on-droplet-ubuntu-22.04.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index bc2291897..7c9157d97 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -55,6 +55,10 @@ jobs:
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-ubuntu-22-04 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
           
+          # Ubuntu droplets run upgrades at boot
+          sleep 15
+          until ! ssh root@${DROPLET_IPV4} "lslocks --json | grep /var/lib/dpkg/lock" > /dev/null; do sleep 1; echo "Waiting for dpkg lock..."; done
+          
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"

From 6b22387c0c5e23f7fe640e1c19f593e027796967 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 20 Feb 2023 16:04:59 +0100
Subject: [PATCH 360/990] Fix: Network configuration was little documented

And there was no validation on network prefix sizes.
---
 vm_supervisor/conf.py | 15 +++++++++++++--
 vm_supervisor/pool.py | 16 ++++++++++------
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index b89d4231b..6c7dff802 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -67,8 +67,14 @@ class Settings(BaseSettings):
     # Networking does not work inside Docker/Podman
     ALLOW_VM_NETWORKING = True
     NETWORK_INTERFACE = "eth0"
-    IPV4_ADDRESS_POOL = "172.16.0.0/12"
-    IPV4_NETWORK_SIZE = 24
+    IPV4_ADDRESS_POOL = Field(
+        default="172.16.0.0/12",
+        description="IPv4 address range used to provide networks to VMs.",
+    )
+    IPV4_NETWORK_PREFIX_LENGTH = Field(
+        default=24,
+        description="Individual VM network prefix length in bits",
+    )
     NFTABLES_CHAIN_PREFIX = "aleph"
 
     DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf
@@ -154,6 +160,11 @@ def check(self):
                 f"/sys/class/net/{self.NETWORK_INTERFACE}"
             ), f"Network interface {self.NETWORK_INTERFACE} does not exist"
 
+            _, ipv4_pool_length = settings.IPV4_ADDRESS_POOL.split("/")
+            assert (
+                int(ipv4_pool_length) <= settings.IPV4_NETWORK_PREFIX_LENGTH
+            ), "The IPv4 address pool prefix must be shorter than an individual VM network prefix"
+
         if self.FAKE_DATA_PROGRAM:
             assert isdir(
                 self.FAKE_DATA_PROGRAM
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index cddb73c52..2c69cd264 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -28,11 +28,15 @@ class VmPool:
     def __init__(self):
         self.counter = settings.START_ID_INDEX
         self.executions = {}
-        self.network = Network(
-            vm_address_pool_range=settings.IPV4_ADDRESS_POOL,
-            vm_network_size=settings.IPV4_NETWORK_SIZE,
-            external_interface=settings.NETWORK_INTERFACE,
-        ) if settings.ALLOW_VM_NETWORKING else None
+        self.network = (
+            Network(
+                vm_address_pool_range=settings.IPV4_ADDRESS_POOL,
+                vm_network_size=settings.IPV4_NETWORK_PREFIX_LENGTH,
+                external_interface=settings.NETWORK_INTERFACE,
+            )
+            if settings.ALLOW_VM_NETWORKING
+            else None
+        )
 
     async def create_a_vm(
         self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent
@@ -54,7 +58,7 @@ def get_unique_vm_id(self) -> int:
         dedicated to the VM.
         """
         _, network_range = settings.IPV4_ADDRESS_POOL.split("/")
-        available_bits = int(network_range) - settings.IPV4_NETWORK_SIZE
+        available_bits = int(network_range) - settings.IPV4_NETWORK_PREFIX_LENGTH
         self.counter += 1
         if self.counter < 2**available_bits:
             # In common cases, use the counter itself as the vm_id. This makes it

From 16f57f8f8768a51eaed67a0279ce1fc1ef2d5df2 Mon Sep 17 00:00:00 2001
From: aliel 
Date: Wed, 22 Feb 2023 21:35:07 +0100
Subject: [PATCH 361/990] Fix: dict.get does not take an argument named

---
 vm_connector/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm_connector/main.py b/vm_connector/main.py
index 7467ecde9..4b505abb5 100644
--- a/vm_connector/main.py
+++ b/vm_connector/main.py
@@ -110,7 +110,7 @@ async def download_data(ref: str) -> Union[StreamingResponse, Response]:
     if not msg:
         return Response(status_code=404, content="Hash not found")
 
-    media_type = msg["content"].get("mime_type", default="application/octet-stream")
+    media_type = msg["content"].get("mime_type", "application/octet-stream")
 
     data_hash = msg["content"]["item_hash"]
     if msg["content"]["item_type"] == "ipfs":

From 05c2bb61e2815babe3db97cb3d55c080247abe02 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 7 Mar 2023 18:01:25 +0100
Subject: [PATCH 362/990] Fix: GitHub actions failed on Ubuntu due to Grub

Errors were encountered while processing:
  grub-efi-amd64-signed
needrestart is being skipped since dpkg has failed
---
 .github/workflows/build-deb-package.yml   | 4 ++++
 .github/workflows/test-build-examples.yml | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 96f6af021..159cb87da 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -77,6 +77,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        if: startsWith(matrix.os, 'ubuntu-')
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
       - run: |
           sudo apt update
           sudo apt install -y debootstrap
diff --git a/.github/workflows/test-build-examples.yml b/.github/workflows/test-build-examples.yml
index 974b56b2c..2dad83f7d 100644
--- a/.github/workflows/test-build-examples.yml
+++ b/.github/workflows/test-build-examples.yml
@@ -13,6 +13,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        if: startsWith(matrix.os, 'ubuntu-')
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
       - run: |
           sudo apt-get -y update
           sudo apt-get -y upgrade

From 2640562ed0e6ab6483bbe581ea6bfd85985342dc Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 7 Mar 2023 18:20:32 +0100
Subject: [PATCH 363/990] Fix: GitHub actions/checkout@v2 was deprecated

---
 .github/workflows/build-deb-package.yml            | 11 +++++------
 .github/workflows/codeql-analysis.yml              |  2 +-
 .github/workflows/test-build-examples.yml          |  3 +--
 .github/workflows/test-on-droplet-debian-11.yml    |  2 +-
 .github/workflows/test-on-droplet-ubuntu-22.04.yml |  2 +-
 5 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 159cb87da..7e9e950f1 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -9,7 +9,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Unshallow
         run: |
@@ -31,7 +31,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Unshallow
         run: |
@@ -53,7 +53,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Unshallow
         run: |
@@ -75,10 +75,9 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
-        if: startsWith(matrix.os, 'ubuntu-')
         run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
 
       - run: |
@@ -98,7 +97,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - run: |
           docker build -t aleph-vm-build-squashfs -f examples/volumes/Dockerfile examples/volumes
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 9bda95264..28c490c57 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -39,7 +39,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
diff --git a/.github/workflows/test-build-examples.yml b/.github/workflows/test-build-examples.yml
index 2dad83f7d..1884bdc0e 100644
--- a/.github/workflows/test-build-examples.yml
+++ b/.github/workflows/test-build-examples.yml
@@ -11,10 +11,9 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
-        if: startsWith(matrix.os, 'ubuntu-')
         run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
 
       - run: |
diff --git a/.github/workflows/test-on-droplet-debian-11.yml b/.github/workflows/test-on-droplet-debian-11.yml
index 5ed2000a0..9fc7f8240 100644
--- a/.github/workflows/test-on-droplet-debian-11.yml
+++ b/.github/workflows/test-on-droplet-debian-11.yml
@@ -10,7 +10,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Fetch the whole history for all tags and branches (required for aleph.__version__)
           fetch-depth: 0
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index 7c9157d97..589f108f7 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -10,7 +10,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Fetch the whole history for all tags and branches (required for aleph.__version__)
           fetch-depth: 0

From 3f9448c752e8bb43080028d6de30af8e3ae3d8ba Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Mon, 3 Apr 2023 15:37:06 +0200
Subject: [PATCH 364/990] Fix: VM connector does not start when run locally
 (#287)

* Fix: remove documentation of `use_latest`

Problem: the field does not appear in the VM connector API functions.

* Fix: FastAPI exception when launching the VM connector

Problem: launching the VM connector with `run_vm_connector.sh` yields
an exception because of issues with the return types of the API
endpoints.

Solution: remove `Response` and `StreamingResponse` type hints from
return values as they are not Pydantic models.
---
 vm_connector/main.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/vm_connector/main.py b/vm_connector/main.py
index 4b505abb5..324d3101b 100644
--- a/vm_connector/main.py
+++ b/vm_connector/main.py
@@ -64,44 +64,43 @@ async def stream_url_chunks(url):
 
 
 @app.get("/download/message/{ref}")
-async def download_message(ref: str) -> Union[Dict, Response]:
+async def download_message(ref: str) -> Dict:
     """
     Fetch on Aleph and return a VM function message, after checking its validity.
     Used by the VM Supervisor run the code.
 
     :param ref: item_hash of the code file
-    :param use_latest: should the last amend to the code be used
     :return: a file containing the code file
     """
 
     msg = await get_message(hash_=ref)
 
-    # TODO: Validate the validity of the message (signature, hashes)
+    # TODO: Validate the message (signature, hashes)
+    if not msg:
+        raise HTTPException(status_code=404, detail="Hash not found")
 
-    return msg or Response(status_code=404, content="Hash not found")
+    return msg
 
 
 @app.get("/download/code/{ref}")
-async def download_code(ref: str) -> Union[StreamingResponse, Response]:
+async def download_code(ref: str):
     """
     Fetch on Aleph and return a VM code file, after checking its validity.
     Used by the VM Supervisor to download function source code.
 
     :param ref: item_hash of the code file
-    :param use_latest: should the last amend to the code be used
     :return: a file containing the code file
     """
     return await download_data(ref=ref)
 
 
 @app.get("/download/data/{ref}")
-async def download_data(ref: str) -> Union[StreamingResponse, Response]:
+async def download_data(ref: str):
     """
     Fetch on Aleph and return a VM data file, after checking its validity.
     Used by the VM Supervisor to download state data.
 
     :param ref: item_hash of the data
-    :param use_latest: should the last amend to the data be used
     :return: a file containing the data
     """
 
@@ -122,13 +121,12 @@ async def download_data(ref: str) -> Union[StreamingResponse, Response]:
 
 
 @app.get("/download/runtime/{ref}")
-async def download_runtime(ref: str) -> Union[StreamingResponse, Response]:
+async def download_runtime(ref: str):
     """
     Fetch on Aleph and return a VM runtime, after checking its validity.
     Used by the VM Supervisor to download a runtime.
 
     :param ref: item_hash of the runtime
-    :param use_latest: should the last amend to the runtime be used
     :return: a file containing the runtime
     """
     return await download_data(ref=ref)

From 9432f1ae333570ad31c827c6d79c744e4d6ee9c6 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Mon, 3 Apr 2023 15:38:27 +0200
Subject: [PATCH 365/990] Fix: add nftables package to VM supervisor Dockerfile
 (#288)

Problem: the VM supervisor Docker image does not work because of
a missing dependency.

Solution: add `python3-nftables` to the Dockerfile.
---
 docker/vm_supervisor-dev.dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 8fcad9484..aeecf7881 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -4,7 +4,7 @@ FROM debian:bullseye
 
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     sudo acl curl squashfs-tools git \
-    python3 python3-aiohttp python3-msgpack python3-pip python3-aiodns python3-aioredis  \
+    python3 python3-aiohttp python3-msgpack python3-pip python3-aiodns python3-aioredis python3-nftables \
     python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging python3-cpuinfo \
     && rm -rf /var/lib/apt/lists/*
 
@@ -47,4 +47,4 @@ COPY ./runtimes /opt/aleph-vm/runtimes
 
 WORKDIR /opt/aleph-vm
 
-CMD "bash"
\ No newline at end of file
+CMD "bash"

From 9225edaeec92aebb75f9f5967ee80ab5fbcabdae Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Mon, 3 Apr 2023 18:49:37 +0200
Subject: [PATCH 366/990] Fix: Firecracker version in README is outdated

---
 vm_supervisor/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index aa3024fa3..b94fa5431 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -65,7 +65,7 @@ useradd jailman
 from the [Firecracker project releases](https://github.com/firecracker-microvm/firecracker/releases):
 ```shell
 mkdir /opt/firecracker
-curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.0.0/firecracker-v1.0.0-x86_64.tgz | tar -xz --directory /opt/firecracker
+curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.1.1/firecracker-v1.1.1-x86_64.tgz | tar -xz --directory /opt/firecracker
 
 # Link binaries on version-agnostic paths:
 ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker

From 105092e789e0bd4eb2a768a9fa3898f5534a7fba Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 25 Apr 2023 14:39:46 +0200
Subject: [PATCH 367/990] Doc: Info about filesystem paths was missing

---
 doc/INSTALL-Debian-11.md    | 21 +++++++++++++++++++--
 doc/INSTALL-Ubuntu-20.04.md | 21 +++++++++++++++++++--
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index 4c3d80b32..9c2114896 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -20,7 +20,7 @@ In order to run an official Aleph.im Compute Resource Node (CRN), you will also
 
 You will need a public domain name with access to add TXT and wildcard records.
 
-> 🛈 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
+> 💡 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
 
 ## 2. Installation
 
@@ -47,11 +47,15 @@ Reboot if required (new kernel, ...).
 
 Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
 
+#### Hostname
+
 You will want to insert your domain name in the form of:
 ```
 ALEPH_VM_DOMAIN_NAME=vm.example.org
 ```
 
+#### Network configuration
+
 On some systems, the default network interface is not `eth0` and you will want to configure the default interface
 by adding:
 ```
@@ -66,7 +70,20 @@ instead, uncomment and add the following setting:
 #ALEPH_VM_DNS_RESOLUTION=resolvctl
 ```
 
-> 🛈 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
+> 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
+
+#### Volumes and partitions
+
+Two directories are used to store data from the network:
+- `/var/lib/aleph/vm` contains all the execution and persistent data.
+- `/var/cache/aleph/vm` contains data downloaded from the network.
+
+These two directories must be stored on the same partition.
+That partition must meet the minimum requirements specified for a CRN.
+
+> 💡 This is required due to the software using hard links to optimize performance and disk usage.
+
+#### Applying changes
 
 Finally, restart the service:
 ```shell
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
index cfcfc8110..a5359c357 100644
--- a/doc/INSTALL-Ubuntu-20.04.md
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -20,7 +20,7 @@ In order to run an official Aleph.im Compute Resource Node (CRN), you will also
 
 You will need a public domain name with access to add TXT and wildcard records.
 
-> 🛈 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
+> 💡 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
 
 ## 2. Installation
 
@@ -45,6 +45,8 @@ Reboot if required (new kernel, ...).
 
 ### Configuration
 
+#### Hostname
+
 Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
 
 You will want to insert your domain name in the form of:
@@ -52,13 +54,15 @@ You will want to insert your domain name in the form of:
 ALEPH_VM_DOMAIN_NAME=vm.example.org
 ```
 
+#### Network configuration
+
 Ubuntu 20.04 by default uses [systemd-resolved](https://manpages.ubuntu.com/manpages/focal/man8/systemd-resolved.service.8.html)
 for DNS resolution. The following setting configures the VM Supervisor to use it instead of reading the default `/etc/resolv.conf`.
 ```
 ALEPH_VM_DNS_RESOLUTION=resolvectl
 ```
 
-> 🛈 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
+> 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
 
 On some systems, the default network interface is not `eth0` and you will want to configure the default interface
 by adding:
@@ -67,6 +71,19 @@ ALEPH_VM_NETWORK_INTERFACE=enp0s1
 ```
 (don't forget to replace `enp0s1` with the name of your default network interface).
 
+#### Volumes and partitions
+
+Two directories are used to store data from the network:
+- `/var/lib/aleph/vm` contains all the execution and persistent data.
+- `/var/cache/aleph/vm` contains data downloaded from the network.
+
+These two directories must be stored on the same partition.
+That partition must meet the minimum requirements specified for a CRN.
+
+> 💡 This is required due to the software using hard links to optimize performance and disk usage.
+
+#### Applying changes
+
 Finally, restart the service:
 ```shell
 sudo systemctl restart aleph-vm-supervisor

From 101c29d23cacb4e0b95cc1dc8098ad59c688b562 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 2 May 2023 18:38:20 +0200
Subject: [PATCH 368/990] Fix: MongoDB '_id' cannot be fetched from 0.5.0

This removes manipulation of this obsolete field.
---
 vm_supervisor/tasks.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py
index 5821363fb..300aa85c2 100644
--- a/vm_supervisor/tasks.py
+++ b/vm_supervisor/tasks.py
@@ -45,8 +45,6 @@ async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]:
                 if msg.type == aiohttp.WSMsgType.TEXT:
                     try:
                         data = json.loads(msg.data)
-                        # Patch data format to match HTTP GET format
-                        data["_id"] = {"$oid": data["_id"]}
                     except json.JSONDecodeError:
                         logger.error(
                             f"Invalid JSON from websocket subscription {msg.data}",

From 032f53b6957052e95b60212ead191d330d100817 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Fri, 5 May 2023 17:52:52 +0200
Subject: [PATCH 369/990] Fix: link to latest version in install guides

The install guides mentioned 0.2.4 instead of 0.2.5.
---
 doc/INSTALL-Debian-11.md    | 2 +-
 doc/INSTALL-Ubuntu-20.04.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index 9c2114896..23f146eb7 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.4/aleph-vm.debian-11.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.5/aleph-vm.debian-11.deb
 apt install /opt/aleph-vm.debian-11.deb
 ```
 
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
index a5359c357..239645171 100644
--- a/doc/INSTALL-Ubuntu-20.04.md
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.4/aleph-vm.ubuntu-20.04.deb
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.5/aleph-vm.ubuntu-20.04.deb
 sudo apt install /opt/aleph-vm.ubuntu-20.04.deb
 ```
 

From 1e624b4ca71af9661465acdcc5a2b45ab2158eec Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 7 Mar 2023 21:58:56 +0100
Subject: [PATCH 370/990] Fix: get_event_loop() was called without a running
 event loop

This behaviour is deprecated and should not be used anymore.
---
 vm_supervisor/__main__.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index d150c3d83..d2ce8809e 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -249,13 +249,8 @@ def main():
 
     settings.check()
 
-    loop = asyncio.get_event_loop()
-
-    if args.debug_asyncio:
-        loop.set_debug(True)
-
     if args.benchmark > 0:
-        loop.run_until_complete(benchmark(runs=args.benchmark))
+        asyncio.run(benchmark(runs=args.benchmark), debug=args.debug_asyncio)
         print("Finished")
     elif args.do_not_run:
         logger.info("Option --do-not-run, exiting")

From 8f78de96ac57d096f94a7e8b9e1c6d1c65fc4458 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Mon, 8 May 2023 23:17:00 +0200
Subject: [PATCH 371/990] Doc: update doc and type hints for
 resolvectl_dns_servers

---
 vm_supervisor/conf.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 6c7dff802..d18219dc3 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -5,7 +5,7 @@
 from os.path import isfile, join, exists, abspath, isdir
 from pathlib import Path
 from subprocess import check_output
-from typing import NewType, Optional, List, Dict, Any
+from typing import NewType, Optional, List, Dict, Any, Iterable
 
 from pydantic import BaseSettings, Field
 
@@ -27,14 +27,16 @@ def etc_resolv_conf_dns_servers():
                 yield ip[0]
 
 
-def resolvectl_dns_servers(interface):
+def resolvectl_dns_servers(interface: str) -> Iterable[str]:
     """
-    On Ubuntu 22.04, DNS servers can be queries using `resolvectl dns`.
-     The command `systemd-resolve` used in Ubuntu 20.04 is not found.
+    Use resolvectl to list available DNS servers (IPv4 and IPv6).
 
-    Example output for `resolvectl dns -i eth0`:
-    Link 2 (eth0): 67.207.67.3 67.207.67.2
+    Note: we used to use systemd-resolve for Ubuntu 20.04 and Debian.
+    This command is not available anymore on Ubuntu 22.04 and is actually a symlink
+    to resolvectl.
 
+    Example output for `resolvectl dns -i eth0`:
+    Link 2 (eth0): 67.207.67.3 67.207.67.2 2a02:2788:fff0:5::140
     """
     output: bytes = check_output(["/usr/bin/resolvectl", "dns", "-i", interface])
     link, servers = output.split(b":")

From 20c4859a08ec57c59f8b4d68746ed1add9c900b7 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Mon, 8 May 2023 23:18:41 +0200
Subject: [PATCH 372/990] Fix: use text mode for calls to resolvectl

This simplifies the function and avoids decoding bytes.
---
 vm_supervisor/conf.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index d18219dc3..3caaa6f62 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -38,10 +38,10 @@ def resolvectl_dns_servers(interface: str) -> Iterable[str]:
     Example output for `resolvectl dns -i eth0`:
     Link 2 (eth0): 67.207.67.3 67.207.67.2 2a02:2788:fff0:5::140
     """
-    output: bytes = check_output(["/usr/bin/resolvectl", "dns", "-i", interface])
-    link, servers = output.split(b":")
-    for server in servers.split(b" "):
-        yield server.decode().strip()
+    output = check_output(["/usr/bin/resolvectl", "dns", "-i", interface], text=True)
+    link, servers = output.split(":")
+    for server in servers.split(" "):
+        yield server.strip()
 
 
 class Settings(BaseSettings):

From dbad02b551173ed26023c3a453db114c214f327f Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Mon, 8 May 2023 23:20:39 +0200
Subject: [PATCH 373/990] Fix: support IPv6 DNS addresses in
 resolvectl_dns_servers

---
 vm_supervisor/conf.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 3caaa6f62..0d85becca 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -39,7 +39,8 @@ def resolvectl_dns_servers(interface: str) -> Iterable[str]:
     Link 2 (eth0): 67.207.67.3 67.207.67.2 2a02:2788:fff0:5::140
     """
     output = check_output(["/usr/bin/resolvectl", "dns", "-i", interface], text=True)
-    link, servers = output.split(":")
+    # Split on the first colon only to support IPv6 addresses.
+    link, servers = output.split(":", maxsplit=1)
     for server in servers.split(" "):
         yield server.strip()
 

From ce0b97bdb6035d008698535f003cef31e950e51f Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Mon, 8 May 2023 23:33:53 +0200
Subject: [PATCH 374/990] Fix: avoid empty entries in resolvectl_dns_servers

The output may contain newlines in addition to spaces.
---
 vm_supervisor/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 0d85becca..8e10b8c32 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -41,7 +41,7 @@ def resolvectl_dns_servers(interface: str) -> Iterable[str]:
     output = check_output(["/usr/bin/resolvectl", "dns", "-i", interface], text=True)
     # Split on the first colon only to support IPv6 addresses.
     link, servers = output.split(":", maxsplit=1)
-    for server in servers.split(" "):
+    for server in servers.split():
         yield server.strip()
 
 

From a4e15f2ae859a9a41480f9bd52ae5c137cadd98a Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Mon, 8 May 2023 23:36:14 +0200
Subject: [PATCH 375/990] Fix: filter out IPv6 DNS servers when using
 resolvectl

Problem: VMs do not support IPv6 (yet). Specifying an IPv6 DNS
server in resolv.conf.
Solution: filter them out.
---
 .../supervisor/test_resolvectl_dns_servers.py | 34 +++++++++++++++++++
 vm_supervisor/conf.py                         | 14 +++++++-
 2 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100644 tests/supervisor/test_resolvectl_dns_servers.py

diff --git a/tests/supervisor/test_resolvectl_dns_servers.py b/tests/supervisor/test_resolvectl_dns_servers.py
new file mode 100644
index 000000000..0889ba23c
--- /dev/null
+++ b/tests/supervisor/test_resolvectl_dns_servers.py
@@ -0,0 +1,34 @@
+# Avoid failures linked to nftables when initializing the global VmPool object
+import os
+os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
+
+from vm_supervisor.conf import resolvectl_dns_servers, resolvectl_dns_servers_ipv4
+
+
+def test_resolvectl(mocker):
+    with mocker.patch(
+        "vm_supervisor.conf.check_output",
+        return_value="Link 2 (eth0): 109.88.203.3 62.197.111.140\n",
+    ):
+        servers = {"109.88.203.3", "62.197.111.140"}
+
+        dns_servers = set(resolvectl_dns_servers("eth0"))
+        assert dns_servers == servers
+
+        dns_servers_ipv4 = set(resolvectl_dns_servers_ipv4("eth0"))
+        assert dns_servers_ipv4 == servers
+
+
+def test_resolvectl_ipv6(mocker):
+    with mocker.patch(
+        "vm_supervisor.conf.check_output",
+        return_value="Link 2 (eth0): 109.88.203.3 62.197.111.140 2a02:2788:fff0:7::3\n        2a02:2788:fff0:5::140\n",
+    ):
+        ipv4_servers = {"109.88.203.3", "62.197.111.140"}
+        ipv6_servers = {"2a02:2788:fff0:7::3", "2a02:2788:fff0:5::140"}
+
+        dns_servers = set(resolvectl_dns_servers("eth0"))
+        assert dns_servers == ipv4_servers | ipv6_servers
+
+        dns_servers_ipv4 = set(resolvectl_dns_servers_ipv4("eth0"))
+        assert dns_servers_ipv4 == ipv4_servers
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 8e10b8c32..0930cf2b9 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -1,3 +1,4 @@
+import ipaddress
 import logging
 import os
 import re
@@ -45,6 +46,17 @@ def resolvectl_dns_servers(interface: str) -> Iterable[str]:
         yield server.strip()
 
 
+def resolvectl_dns_servers_ipv4(interface: str) -> Iterable[str]:
+    """
+    Use resolvectl to list available IPv4 DNS servers.
+    VMs only support IPv4 networking for now, we must exclude IPv6 DNS from their config.
+    """
+    for server in resolvectl_dns_servers(interface):
+        ip_addr = ipaddress.ip_address(server)
+        if isinstance(ip_addr, ipaddress.IPv4Address):
+            yield server
+
+
 class Settings(BaseSettings):
     SUPERVISOR_HOST = "127.0.0.1"
     SUPERVISOR_PORT: int = 4020
@@ -196,7 +208,7 @@ def setup(self):
 
             elif self.DNS_RESOLUTION == DnsResolver.resolvectl:
                 self.DNS_NAMESERVERS = list(
-                    resolvectl_dns_servers(interface=self.NETWORK_INTERFACE)
+                    resolvectl_dns_servers_ipv4(interface=self.NETWORK_INTERFACE)
                 )
             else:
                 assert "This should never happen"

From 0c87558d3151f72c84a22b37bcb8b9b0e1548274 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 10 May 2023 11:52:30 +0200
Subject: [PATCH 376/990] Chore: Update GitHub Actions version

This removes a deprecation warning in CI.
---
 .github/workflows/build-deb-package.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 7e9e950f1..efc6516aa 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -19,8 +19,8 @@ jobs:
       - run: |
           cd packaging && make all-podman-debian-11 && cd ..
           ls packaging/target
-          
-      - uses: actions/upload-artifact@v2
+
+      - uses: actions/upload-artifact@v3
         with:
           name: aleph-vm.debian-11.deb
           path: packaging/target/aleph-vm.debian-11.deb
@@ -42,7 +42,7 @@ jobs:
           cd packaging && make all-podman-ubuntu-2004 && cd ..
           ls packaging/target
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: aleph-vm.ubuntu-20.04.deb
           path: packaging/target/aleph-vm.ubuntu-20.04.deb
@@ -64,7 +64,7 @@ jobs:
           cd packaging && make all-podman-ubuntu-2204 && cd ..
           ls packaging/target
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: aleph-vm.ubuntu-22.04.deb
           path: packaging/target/aleph-vm.ubuntu-22.04.deb
@@ -85,7 +85,7 @@ jobs:
           sudo apt install -y debootstrap
           cd runtimes/aleph-debian-11-python && sudo ./create_disk_image.sh && cd ../..
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: aleph-debian-11-python.squashfs
           path: runtimes/aleph-debian-11-python/rootfs.squashfs
@@ -103,7 +103,7 @@ jobs:
           docker build -t aleph-vm-build-squashfs -f examples/volumes/Dockerfile examples/volumes
           docker run --rm -v "$(pwd)":/mnt aleph-vm-build-squashfs
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: example-volume-venv.squashfs
           path: volume-venv.squashfs

From dfacf72b4bffaf5122299c9546c2aa4693c9e763 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 10 May 2023 14:23:16 +0200
Subject: [PATCH 377/990] Fix: CI failed due to apt lock being held

The error message was:
```
Could not get lock /var/lib/apt/lists/lock. It is held by process 1399 (apt-get)
E: Unable to lock directory /var/lib/apt/lists/
```
---
 .github/workflows/test-on-droplet-debian-11.yml    | 8 ++++----
 .github/workflows/test-on-droplet-ubuntu-22.04.yml | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test-on-droplet-debian-11.yml b/.github/workflows/test-on-droplet-debian-11.yml
index 9fc7f8240..2917bd4f2 100644
--- a/.github/workflows/test-on-droplet-debian-11.yml
+++ b/.github/workflows/test-on-droplet-debian-11.yml
@@ -55,13 +55,13 @@ jobs:
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
           
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 update"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 upgrade -y"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y docker.io apparmor-profiles"
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
           
           scp packaging/target/aleph-vm.debian-11.deb root@${DROPLET_IPV4}:/opt
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.debian-11.deb"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt -o DPkg::Lock::Timeout=60 install -y /opt/aleph-vm.debian-11.deb"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
 
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index 589f108f7..4f2ad156f 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -59,13 +59,13 @@ jobs:
           sleep 15
           until ! ssh root@${DROPLET_IPV4} "lslocks --json | grep /var/lib/dpkg/lock" > /dev/null; do sleep 1; echo "Waiting for dpkg lock..."; done
           
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 update"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 upgrade -y"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y docker.io apparmor-profiles"
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
           
           scp packaging/target/aleph-vm.ubuntu-22.04.deb root@${DROPLET_IPV4}:/opt
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.ubuntu-22.04.deb"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt -o DPkg::Lock::Timeout=60 install -y /opt/aleph-vm.ubuntu-22.04.deb"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_DNS_RESOLUTION=resolvectl >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"

From 2e81e0c551257e3bfa982b43cf873c69dbebcf19 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 10 May 2023 13:44:08 +0200
Subject: [PATCH 378/990] Cleanup: Format with `black` and `isort`

---
 vm_supervisor/__init__.py               | 36 +++++++++++++------------
 vm_supervisor/__main__.py               | 10 +++----
 vm_supervisor/conf.py                   |  4 +--
 vm_supervisor/messages.py               |  4 +--
 vm_supervisor/metrics.py                |  3 +--
 vm_supervisor/models.py                 |  8 +++---
 vm_supervisor/network/firewall.py       |  4 +--
 vm_supervisor/network/hostnetwork.py    |  9 ++++---
 vm_supervisor/network/interfaces.py     |  6 ++---
 vm_supervisor/network/ipaddresses.py    |  2 +-
 vm_supervisor/pool.py                   |  7 ++---
 vm_supervisor/reactor.py                |  2 +-
 vm_supervisor/resources.py              |  6 ++---
 vm_supervisor/run.py                    |  7 ++---
 vm_supervisor/status.py                 |  4 +--
 vm_supervisor/storage.py                |  2 +-
 vm_supervisor/supervisor.py             | 10 +++----
 vm_supervisor/tasks.py                  |  4 +--
 vm_supervisor/utils.py                  |  7 ++---
 vm_supervisor/version.py                |  2 +-
 vm_supervisor/views.py                  | 11 ++++----
 vm_supervisor/vm/firecracker_microvm.py | 21 ++++++++-------
 22 files changed, 87 insertions(+), 82 deletions(-)

diff --git a/vm_supervisor/__init__.py b/vm_supervisor/__init__.py
index ae44191e8..aa99dfdaf 100644
--- a/vm_supervisor/__init__.py
+++ b/vm_supervisor/__init__.py
@@ -1,20 +1,22 @@
-from . import conf
-from . import messages
-from . import metrics
-from . import models
-from . import pool
-from . import pubsub
-from . import reactor
-from . import resources
-from . import run
-from . import status
-from . import storage
-from . import supervisor
-from . import tasks
-from . import utils
-from . import version
-from . import views
-from . import vm
+from . import (
+    conf,
+    messages,
+    metrics,
+    models,
+    pool,
+    pubsub,
+    reactor,
+    resources,
+    run,
+    status,
+    storage,
+    supervisor,
+    tasks,
+    utils,
+    version,
+    views,
+    vm,
+)
 
 __version__ = version.__version__
 
diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index d2ce8809e..175a21d65 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -4,20 +4,20 @@
 import sys
 import time
 from statistics import mean
-from typing import List, Tuple, Dict, Callable
+from typing import Callable, Dict, List, Tuple
 
-from aiohttp.web import Response, Request
+from aiohttp.web import Request, Response
 
 try:
     import sentry_sdk
 except ImportError:
     sentry_sdk = None
 
-from .pubsub import PubSub
-from . import supervisor, metrics
+from . import metrics, supervisor
 from .conf import settings
 from .models import VmHash
-from .run import run_code_on_request, run_code_on_event
+from .pubsub import PubSub
+from .run import run_code_on_event, run_code_on_request
 
 logger = logging.getLogger(__name__)
 
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 0930cf2b9..3230eacdd 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -3,10 +3,10 @@
 import os
 import re
 from enum import Enum
-from os.path import isfile, join, exists, abspath, isdir
+from os.path import abspath, exists, isdir, isfile, join
 from pathlib import Path
 from subprocess import check_output
-from typing import NewType, Optional, List, Dict, Any, Iterable
+from typing import Any, Dict, Iterable, List, NewType, Optional
 
 from pydantic import BaseSettings, Field
 
diff --git a/vm_supervisor/messages.py b/vm_supervisor/messages.py
index 7b78d070d..6c2bc8fa6 100644
--- a/vm_supervisor/messages.py
+++ b/vm_supervisor/messages.py
@@ -3,11 +3,11 @@
 from typing import Tuple
 
 from aiohttp import ClientConnectorError, ClientResponseError
-from aiohttp.web_exceptions import HTTPServiceUnavailable, HTTPNotFound
+from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable
 from aleph_message.models import ProgramMessage
 
 from .models import VmHash
-from .storage import get_message, get_latest_amend
+from .storage import get_latest_amend, get_message
 
 
 async def try_get_message(ref: str) -> ProgramMessage:
diff --git a/vm_supervisor/metrics.py b/vm_supervisor/metrics.py
index 9789aa795..d75c5d851 100644
--- a/vm_supervisor/metrics.py
+++ b/vm_supervisor/metrics.py
@@ -4,8 +4,7 @@
 from typing import Iterable, Optional
 from uuid import UUID
 
-from sqlalchemy import Column, Integer, String, Float, DateTime
-from sqlalchemy import create_engine
+from sqlalchemy import Column, DateTime, Float, Integer, String, create_engine
 from sqlalchemy.engine import Engine
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index d986e0bce..bebaf710e 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -5,17 +5,17 @@
 from asyncio import Task
 from dataclasses import dataclass
 from datetime import datetime
-from typing import NewType, Optional, Dict
+from typing import Dict, NewType, Optional
 
 from aleph_message.models import ProgramContent
 
 from .conf import settings
-from .metrics import save_record, save_execution_data, ExecutionRecord
+from .metrics import ExecutionRecord, save_execution_data, save_record
+from .network.interfaces import TapInterface
 from .pubsub import PubSub
-from .utils import dumps_for_json, create_task_log_exceptions
+from .utils import create_task_log_exceptions, dumps_for_json
 from .vm import AlephFirecrackerVM
 from .vm.firecracker_microvm import AlephFirecrackerResources
-from .network.interfaces import TapInterface
 
 logger = logging.getLogger(__name__)
 
diff --git a/vm_supervisor/network/firewall.py b/vm_supervisor/network/firewall.py
index 5c0818b26..454cd5aff 100644
--- a/vm_supervisor/network/firewall.py
+++ b/vm_supervisor/network/firewall.py
@@ -1,9 +1,9 @@
 import json
-from typing import Dict, List
 import logging
+from functools import lru_cache
+from typing import Dict, List
 
 from nftables import Nftables
-from functools import lru_cache
 
 from ..conf import settings
 from .interfaces import TapInterface
diff --git a/vm_supervisor/network/hostnetwork.py b/vm_supervisor/network/hostnetwork.py
index a190b7565..fee59abb7 100644
--- a/vm_supervisor/network/hostnetwork.py
+++ b/vm_supervisor/network/hostnetwork.py
@@ -1,6 +1,6 @@
 import logging
 
-from .firewall import initialize_nftables, teardown_nftables, setup_nftables_for_vm
+from .firewall import initialize_nftables, setup_nftables_for_vm, teardown_nftables
 from .interfaces import TapInterface
 from .ipaddresses import IPv4NetworkWithInterfaces
 
@@ -38,7 +38,9 @@ def reset_ipv4_forwarding_state(self) -> None:
             with open("/proc/sys/net/ipv4/ip_forward", "w") as f:
                 f.write(str(self.ipv4_forward_state_before_setup))
 
-    def __init__(self, vm_address_pool_range: str, vm_network_size: int, external_interface: str) -> None:
+    def __init__(
+        self, vm_address_pool_range: str, vm_network_size: int, external_interface: str
+    ) -> None:
         """Sets up the Network class with some information it needs so future function calls work as expected"""
         self.address_pool = IPv4NetworkWithInterfaces(vm_address_pool_range)
         if not self.address_pool.is_private:
@@ -55,8 +57,7 @@ def teardown(self) -> None:
         self.reset_ipv4_forwarding_state()
 
     async def create_tap(self, vm_id: int) -> TapInterface:
-        """ Create TAP interface to be used by VM
-        """
+        """Create TAP interface to be used by VM"""
         interface = TapInterface(f"vmtap{vm_id}", self.get_network_for_tap(vm_id))
         await interface.create()
         setup_nftables_for_vm(vm_id, interface)
diff --git a/vm_supervisor/network/interfaces.py b/vm_supervisor/network/interfaces.py
index 71b6b65c0..271e44f08 100644
--- a/vm_supervisor/network/interfaces.py
+++ b/vm_supervisor/network/interfaces.py
@@ -1,7 +1,7 @@
 import asyncio
+import logging
 from ipaddress import IPv4Interface
 from subprocess import run
-import logging
 
 from .ipaddresses import IPv4NetworkWithInterfaces
 
@@ -12,9 +12,7 @@ class TapInterface:
     device_name: str
     ip_network: IPv4NetworkWithInterfaces
 
-    def __init__(
-        self, device_name: str, ip_network: IPv4NetworkWithInterfaces
-    ):
+    def __init__(self, device_name: str, ip_network: IPv4NetworkWithInterfaces):
         self.device_name: str = device_name
         self.ip_network: IPv4NetworkWithInterfaces = ip_network
 
diff --git a/vm_supervisor/network/ipaddresses.py b/vm_supervisor/network/ipaddresses.py
index 1d007c57a..c445129d6 100644
--- a/vm_supervisor/network/ipaddresses.py
+++ b/vm_supervisor/network/ipaddresses.py
@@ -1,4 +1,4 @@
-from ipaddress import IPv4Network, IPv4Interface
+from ipaddress import IPv4Interface, IPv4Network
 from typing import Iterable
 
 
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 2c69cd264..2f889955b 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -1,13 +1,14 @@
 import asyncio
 import logging
-from typing import Dict, Optional, Iterable
+from typing import Dict, Iterable, Optional
 
 from aleph_message.models import ProgramContent, ProgramMessage
 
-from .conf import settings
-from .models import VmHash, VmExecution
 from vm_supervisor.network.hostnetwork import Network
 
+from .conf import settings
+from .models import VmExecution, VmHash
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/vm_supervisor/reactor.py b/vm_supervisor/reactor.py
index b4e499bc4..e87ecd1e8 100644
--- a/vm_supervisor/reactor.py
+++ b/vm_supervisor/reactor.py
@@ -1,5 +1,5 @@
 import logging
-from typing import List, Coroutine
+from typing import Coroutine, List
 
 from aleph_message.models import Message, ProgramMessage
 from aleph_message.models.program import Subscription
diff --git a/vm_supervisor/resources.py b/vm_supervisor/resources.py
index 1ef237bfa..27f86e088 100644
--- a/vm_supervisor/resources.py
+++ b/vm_supervisor/resources.py
@@ -1,7 +1,6 @@
 from datetime import datetime, timezone
 from functools import lru_cache
-from typing import Set, Optional
-from typing import Tuple
+from typing import Optional, Set, Tuple
 
 import cpuinfo
 import psutil
@@ -105,7 +104,8 @@ async def about_system_usage(request: web.Request):
             available_kB=psutil.virtual_memory().available / 1000,
         ),
         disk=DiskUsage(
-            total_kB=psutil.disk_usage(str(settings.PERSISTENT_VOLUMES_DIR)).total // 1000,
+            total_kB=psutil.disk_usage(str(settings.PERSISTENT_VOLUMES_DIR)).total
+            // 1000,
             available_kB=psutil.disk_usage(str(settings.PERSISTENT_VOLUMES_DIR)).free
             // 1000,
         ),
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 2df96cf64..2e5ca0ff2 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -1,6 +1,6 @@
 import asyncio
 import logging
-from typing import Dict, Any, Optional
+from typing import Any, Dict, Optional
 
 import msgpack
 from aiohttp import web
@@ -8,15 +8,16 @@
 from msgpack import UnpackValueError
 
 from firecracker.microvm import MicroVMFailedInit
+
 from .conf import settings
 from .messages import load_updated_message
-from .models import VmHash, VmExecution
+from .models import VmExecution, VmHash
 from .pool import VmPool
 from .pubsub import PubSub
 from .vm.firecracker_microvm import (
+    FileTooLargeError,
     ResourceDownloadError,
     VmSetupError,
-    FileTooLargeError,
 )
 
 logger = logging.getLogger(__name__)
diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py
index d4b4a5de9..37156f81e 100644
--- a/vm_supervisor/status.py
+++ b/vm_supervisor/status.py
@@ -3,9 +3,9 @@
 in a deployed supervisor.
 """
 import logging
-from typing import Dict, Any, List
+from typing import Any, Dict, List
 
-from aiohttp import ClientSession, ClientResponseError
+from aiohttp import ClientResponseError, ClientSession
 
 from .conf import settings
 
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 2945b249a..35c5c80b7 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -19,8 +19,8 @@
 from aleph_message.models import ProgramMessage
 from aleph_message.models.program import (
     Encoding,
-    MachineVolume,
     ImmutableVolume,
+    MachineVolume,
     PersistentVolume,
     VolumePersistence,
 )
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 84e42186f..3d9095bdd 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -18,13 +18,13 @@
 from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task
 from .version import __version__
 from .views import (
-    run_code_from_path,
-    run_code_from_hostname,
-    about_login,
-    about_executions,
     about_config,
-    status_check_fastapi,
     about_execution_records,
+    about_executions,
+    about_login,
+    run_code_from_hostname,
+    run_code_from_path,
+    status_check_fastapi,
     status_check_version,
     update_allocations,
 )
diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py
index 300aa85c2..bdcc8bc76 100644
--- a/vm_supervisor/tasks.py
+++ b/vm_supervisor/tasks.py
@@ -8,10 +8,10 @@
 import aiohttp
 import pydantic
 from aiohttp import web
-from yarl import URL
-
 from aleph_message import Message
 from aleph_message.models import BaseMessage, ProgramMessage
+from yarl import URL
+
 from .conf import settings
 from .messages import load_updated_message
 from .models import VmHash
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index 3c247af4d..219d97289 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -1,9 +1,10 @@
 import asyncio
 import json
 import logging
-from base64 import b32decode, b16encode
-from dataclasses import is_dataclass, asdict as dataclass_as_dict
-from typing import Any, Optional, Coroutine
+from base64 import b16encode, b32decode
+from dataclasses import asdict as dataclass_as_dict
+from dataclasses import is_dataclass
+from typing import Any, Coroutine, Optional
 
 import aiodns
 
diff --git a/vm_supervisor/version.py b/vm_supervisor/version.py
index c64544bb1..280d0acc1 100644
--- a/vm_supervisor/version.py
+++ b/vm_supervisor/version.py
@@ -1,5 +1,5 @@
 import logging
-from subprocess import check_output, CalledProcessError
+from subprocess import CalledProcessError, check_output
 from typing import Optional
 
 logger = logging.getLogger(__name__)
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index 229ca7716..4b21dd2f9 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -9,18 +9,19 @@
 import aiohttp
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPNotFound
-from packaging.version import Version, InvalidVersion
 from pydantic import ValidationError
 
+from packaging.version import InvalidVersion, Version
+
 from . import status
-from .version import __version__
 from .conf import settings
 from .metrics import get_execution_records
 from .models import VmHash
 from .pubsub import PubSub
 from .resources import Allocation
-from .run import run_code_on_request, pool, start_persistent_vm
-from .utils import b32_to_b16, get_ref_from_dns, dumps_for_json
+from .run import pool, run_code_on_request, start_persistent_vm
+from .utils import b32_to_b16, dumps_for_json, get_ref_from_dns
+from .version import __version__
 
 logger = logging.getLogger(__name__)
 
@@ -175,7 +176,7 @@ async def status_check_version(request: web.Request):
 
 def authenticate_api_request(request: web.Request) -> bool:
     """Authenticate an API request to update the VM allocations."""
-    signature: bytes = request.headers.get("X-Auth-Signature", '').encode()
+    signature: bytes = request.headers.get("X-Auth-Signature", "").encode()
 
     if not signature:
         raise web.HTTPUnauthorized(text="Authentication token is missing")
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 248669cbe..2c8506476 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -6,9 +6,9 @@
 from dataclasses import dataclass, field
 from enum import Enum
 from multiprocessing import Process, set_start_method
-from os.path import isfile, exists
-from typing import Optional, Dict, List
+from os.path import exists, isfile
 from pathlib import Path
+from typing import Dict, List, Optional
 
 import msgpack
 
@@ -17,23 +17,24 @@
 except ImportError:
     psutil = None
 from aiohttp import ClientResponseError
-
 from aleph_message.models import ProgramContent
-from aleph_message.models.program import MachineResources, Encoding
+from aleph_message.models.program import Encoding, MachineResources
+
 from firecracker.config import (
     BootSource,
     Drive,
-    MachineConfig,
     FirecrackerConfig,
-    Vsock,
+    MachineConfig,
     NetworkInterface,
+    Vsock,
 )
 from firecracker.microvm import MicroVM, setfacl
 from guest_api.__main__ import run_guest_api
+
 from ..conf import settings
-from ..storage import get_code_path, get_runtime_path, get_data_path, get_volume_path
-from ..network.interfaces import TapInterface
 from ..network.firewall import teardown_nftables_for_vm
+from ..network.interfaces import TapInterface
+from ..storage import get_code_path, get_data_path, get_runtime_path, get_volume_path
 
 logger = logging.getLogger(__name__)
 set_start_method("spawn")
@@ -391,8 +392,8 @@ async def configure(self):
 
         # The ip and route should not contain the network mask in order to maintain
         # compatibility with the existing runtimes.
-        ip = self.tap_interface.guest_ip.with_prefixlen.split('/', 1)[0]
-        route = str(self.tap_interface.host_ip).split('/', 1)[0]
+        ip = self.tap_interface.guest_ip.with_prefixlen.split("/", 1)[0]
+        route = str(self.tap_interface.host_ip).split("/", 1)[0]
 
         config = ConfigurationPayload(
             ip=ip if self.enable_networking else None,

From dfe653e54ee6553ccaa999cb6dc7d7ce63f64b2c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 10 May 2023 16:24:36 +0200
Subject: [PATCH 379/990]  Internal: DB migration system (#304)

Problem: some updates require to modify the DB models.
Solution: add a migration system based on alembic (the SQLAlchemy
migration system).

Developers can now create migrations by running
`alembic revision -m "" --autogenerate`.

The migrations are automatically run at startup time.

Co-authored-by: Olivier Desenfans 
---
 docker/vm_supervisor-dev.dockerfile           |   4 +-
 packaging/aleph-vm/DEBIAN/control             |   2 +-
 packaging/requirements-debian-11.txt          |  17 ++-
 packaging/requirements-ubuntu-20.04.txt       |  18 ++-
 packaging/requirements-ubuntu-22.04.txt       |  35 ++++++
 vm_supervisor/__main__.py                     |  37 ++++++-
 vm_supervisor/alembic.ini                     | 103 ++++++++++++++++++
 vm_supervisor/conf.py                         |   4 +
 vm_supervisor/metrics.py                      |   4 +-
 vm_supervisor/migrations/__init__.py          |   0
 vm_supervisor/migrations/env.py               |  70 ++++++++++++
 vm_supervisor/migrations/script.py.mako       |  24 ++++
 .../0001_bbb12a12372e_execution_records.py    |  46 ++++++++
 13 files changed, 353 insertions(+), 11 deletions(-)
 create mode 100644 packaging/requirements-ubuntu-22.04.txt
 create mode 100644 vm_supervisor/alembic.ini
 create mode 100644 vm_supervisor/migrations/__init__.py
 create mode 100644 vm_supervisor/migrations/env.py
 create mode 100644 vm_supervisor/migrations/script.py.mako
 create mode 100644 vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index aeecf7881..77b00c1a7 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -4,8 +4,8 @@ FROM debian:bullseye
 
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     sudo acl curl squashfs-tools git \
-    python3 python3-aiohttp python3-msgpack python3-pip python3-aiodns python3-aioredis python3-nftables \
-    python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging python3-cpuinfo \
+    python3 python3-aiohttp python3-alembic python3-msgpack python3-pip python3-aiodns python3-aioredis\
+    python3-nftables python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging python3-cpuinfo \
     && rm -rf /var/lib/apt/lists/*
 
 RUN useradd jailman
diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index 0162c4a99..0760e76a0 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema
 Section: aleph-im
 Priority: Extra
diff --git a/packaging/requirements-debian-11.txt b/packaging/requirements-debian-11.txt
index e6774d4f8..d708640db 100644
--- a/packaging/requirements-debian-11.txt
+++ b/packaging/requirements-debian-11.txt
@@ -1,17 +1,32 @@
 aiodns==2.0.0
 aiohttp==3.7.4
 aioredis==1.3.1
+alembic==1.4.3
 async-timeout==3.0.1
 attrs==20.3.0
 chardet==4.0.0
 hiredis==1.0.1
 idna==2.10
+importlib-metadata==1.6.0
+jsonschema==3.2.0
+Mako==1.1.3
+MarkupSafe==1.1.1
+more-itertools==4.2.0
 msgpack==1.0.0
 multidict==5.1.0
-psutil==5.6.6
+git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
+packaging==20.9
+psutil==5.8.0
+py-cpuinfo==5.0.0
 pycares==3.1.1
+pyparsing==2.4.7
+pyrsistent==0.15.5
+python-dateutil==2.8.1
+python-editor==1.0.3
 redis==3.5.3
 setproctitle==1.2.1
+six==1.16.0
 SQLAlchemy==1.3.22
 typing-extensions==3.7.4.3
 yarl==1.6.3
+zipp==1.0.0
diff --git a/packaging/requirements-ubuntu-20.04.txt b/packaging/requirements-ubuntu-20.04.txt
index 82f134c03..1175ab784 100644
--- a/packaging/requirements-ubuntu-20.04.txt
+++ b/packaging/requirements-ubuntu-20.04.txt
@@ -1,18 +1,32 @@
 aiodns==2.0.0
-aiohttp==3.7.4
+aiohttp==3.6.2
 aioredis==1.3.1
+alembic==1.1.0
 async-timeout==3.0.1
 attrs==19.3.0
 chardet==3.0.4
 dbus-python==1.2.16
 hiredis==1.0.0
 idna==2.8
+importlib-metadata==1.5.0
+jsonschema==3.2.0
+Mako==1.1.0
+MarkupSafe==1.1.0
+more-itertools==4.2.0
 msgpack==0.6.2
 multidict==4.7.3
-psutil==5.6.6
+git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
+packaging==20.3
+psutil==5.5.1
+py-cpuinfo==5.0.0
 pycares==3.1.1
 PyGObject==3.36.0
+pyparsing==2.4.6
+pyrsistent==0.15.5
+python-dateutil==2.7.3
 redis==3.3.11
 setproctitle==1.1.10
+six==1.14.0
 SQLAlchemy==1.3.12
 yarl==1.4.2
+zipp==1.0.0
diff --git a/packaging/requirements-ubuntu-22.04.txt b/packaging/requirements-ubuntu-22.04.txt
new file mode 100644
index 000000000..580dc68ef
--- /dev/null
+++ b/packaging/requirements-ubuntu-22.04.txt
@@ -0,0 +1,35 @@
+aiodns==3.0.0
+aiohttp==3.8.1
+aioredis==1.3.1
+aiosignal==1.2.0
+alembic==1.7.6
+async-timeout==4.0.1
+attrs==21.2.0
+charset-normalizer==2.0.6
+dbus-python==1.2.18
+frozenlist==1.2.0
+greenlet==1.1.2
+hiredis==1.0.1
+idna==3.3
+importlib-metadata==4.6.4
+jsonschema==3.2.0
+Mako==1.1.3
+MarkupSafe==2.0.1
+more-itertools==8.10.0
+msgpack==1.0.3
+multidict==5.1.0
+git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
+packaging==21.3
+psutil==5.9.0
+py-cpuinfo==5.0.0
+pycares==4.1.2
+PyGObject==3.42.1
+pyparsing==2.4.7
+pyrsistent==0.18.1
+redis==3.5.3
+setproctitle==1.2.2
+six==1.16.0
+SQLAlchemy==1.4.31
+typing-extensions==3.10.0.2
+yarl==1.7.2
+zipp==1.0.0
diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index 175a21d65..acf34ae24 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -1,8 +1,11 @@
 import argparse
 import asyncio
+import contextlib
 import logging
+import os
 import sys
 import time
+from pathlib import Path
 from statistics import mean
 from typing import Callable, Dict, List, Tuple
 
@@ -13,11 +16,13 @@
 except ImportError:
     sentry_sdk = None
 
-from . import metrics, supervisor
-from .conf import settings
+from . import supervisor, metrics
+from .conf import settings, make_db_url
 from .models import VmHash
 from .pubsub import PubSub
-from .run import run_code_on_event, run_code_on_request
+from .run import run_code_on_request, run_code_on_event
+import alembic.config
+import alembic.command
 
 logger = logging.getLogger(__name__)
 
@@ -203,6 +208,28 @@ async def fake_read() -> bytes:
     print("Event result", result)
 
 
+@contextlib.contextmanager
+def change_dir(directory: Path):
+    current_directory = Path.cwd()
+    try:
+        os.chdir(directory)
+        yield
+    finally:
+        os.chdir(current_directory)
+
+
+def run_db_migrations():
+    project_dir = Path(__file__).parent
+
+    db_url = make_db_url()
+    alembic_cfg = alembic.config.Config("alembic.ini")
+    alembic_cfg.attributes["configure_logger"] = False
+    logging.getLogger("alembic").setLevel(logging.CRITICAL)
+
+    with change_dir(project_dir):
+        alembic.command.upgrade(alembic_cfg, "head", tag=db_url)
+
+
 def main():
     args = parse_args(sys.argv[1:])
 
@@ -249,6 +276,10 @@ def main():
 
     settings.check()
 
+    logger.debug("Initialising the DB...")
+    run_db_migrations()
+    logger.debug("DB up to date.")
+
     if args.benchmark > 0:
         asyncio.run(benchmark(runs=args.benchmark), debug=args.debug_asyncio)
         print("Finished")
diff --git a/vm_supervisor/alembic.ini b/vm_supervisor/alembic.ini
new file mode 100644
index 000000000..1cf7e2b20
--- /dev/null
+++ b/vm_supervisor/alembic.ini
@@ -0,0 +1,103 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+;script_location = vm_supervisor/migrations
+script_location = migrations
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 3230eacdd..7022510ba 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -236,5 +236,9 @@ class Config:
         env_file = ".env"
 
 
+def make_db_url():
+    return f"sqlite:///{settings.EXECUTION_DATABASE}"
+
+
 # Settings singleton
 settings = Settings()
diff --git a/vm_supervisor/metrics.py b/vm_supervisor/metrics.py
index d75c5d851..4a1f5d4a4 100644
--- a/vm_supervisor/metrics.py
+++ b/vm_supervisor/metrics.py
@@ -9,7 +9,7 @@
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 
-from .conf import settings
+from .conf import make_db_url, settings
 
 Session: sessionmaker
 
@@ -20,7 +20,7 @@
 
 def setup_engine():
     global Session
-    engine = create_engine(f"sqlite:///{settings.EXECUTION_DATABASE}", echo=True)
+    engine = create_engine(make_db_url(), echo=True)
     Session = sessionmaker(bind=engine)
     return engine
 
diff --git a/vm_supervisor/migrations/__init__.py b/vm_supervisor/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/vm_supervisor/migrations/env.py b/vm_supervisor/migrations/env.py
new file mode 100644
index 000000000..9db9eb318
--- /dev/null
+++ b/vm_supervisor/migrations/env.py
@@ -0,0 +1,70 @@
+from logging.config import fileConfig
+
+from alembic import context
+from vm_supervisor.conf import make_db_url
+from sqlalchemy import create_engine
+
+# # this is the Alembic Config object, which provides
+# # access to the values within the .ini file in use.
+# config = context.config
+#
+# # Interpret the config file for Python logging.
+# # This line sets up loggers basically.
+# if config.config_file_name is not None:
+#     fileConfig(config.config_file_name)
+
+# Auto-generate migrations
+from vm_supervisor.metrics import Base
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = make_db_url()
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = create_engine(make_db_url())
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()
diff --git a/vm_supervisor/migrations/script.py.mako b/vm_supervisor/migrations/script.py.mako
new file mode 100644
index 000000000..55df2863d
--- /dev/null
+++ b/vm_supervisor/migrations/script.py.mako
@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}
diff --git a/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py b/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py
new file mode 100644
index 000000000..1ed84fdae
--- /dev/null
+++ b/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py
@@ -0,0 +1,46 @@
+"""execution records
+
+Revision ID: bbb12a12372e
+Revises: 
+Create Date: 2022-09-28 18:52:16.431200
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "bbb12a12372e"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "records",
+        sa.Column("uuid", sa.String(), nullable=False),
+        sa.Column("vm_hash", sa.String(), nullable=False),
+        sa.Column("time_defined", sa.DateTime(), nullable=False),
+        sa.Column("time_prepared", sa.DateTime(), nullable=True),
+        sa.Column("time_started", sa.DateTime(), nullable=True),
+        sa.Column("time_stopping", sa.DateTime(), nullable=True),
+        sa.Column("cpu_time_user", sa.Float(), nullable=True),
+        sa.Column("cpu_time_system", sa.Float(), nullable=True),
+        sa.Column("io_read_count", sa.Integer(), nullable=True),
+        sa.Column("io_write_count", sa.Integer(), nullable=True),
+        sa.Column("io_read_bytes", sa.Integer(), nullable=True),
+        sa.Column("io_write_bytes", sa.Integer(), nullable=True),
+        sa.Column("vcpus", sa.Integer(), nullable=False),
+        sa.Column("memory", sa.Integer(), nullable=False),
+        sa.Column("network_tap", sa.String(), nullable=True),
+        sa.PrimaryKeyConstraint("uuid"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("records")
+    # ### end Alembic commands ###

From 705a2bf23acc515042df09a4e6ea64c91ecd9fa1 Mon Sep 17 00:00:00 2001
From: aliel 
Date: Wed, 15 Mar 2023 21:22:52 +0100
Subject: [PATCH 380/990] Fix: VM hostnames could cause the VM to crash #123

---
 runtimes/aleph-alpine-3.13-python/init1.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py
index c9aa906e0..47f435f62 100644
--- a/runtimes/aleph-alpine-3.13-python/init1.py
+++ b/runtimes/aleph-alpine-3.13-python/init1.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python3 -OO
-
+import base64
 import logging
 
 logging.basicConfig(
@@ -451,7 +451,12 @@ def receive_config(client) -> ConfigurationPayload:
 
 
 def setup_system(config: ConfigurationPayload):
-    setup_hostname(config.vm_hash)
+    # Linux host names are limited to 63 characters. We therefore use the base32 representation
+    # of the item_hash instead of its common base16 representation.
+    item_hash_binary: bytes = base64.b16decode(config.vm_hash.encode().upper())
+    hostname = base64.b32encode(item_hash_binary).decode().strip('=').lower()
+    setup_hostname(hostname)
+
     setup_variables(config.variables)
     setup_volumes(config.volumes)
     setup_network(config.ip, config.route, config.dns_servers)

From 95a204f85b6c8fa790968e79c7017f9c5b2a0663 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 12 May 2023 16:09:49 +0200
Subject: [PATCH 381/990] Fix: Command apt was locked in CI on Ubuntu 22.04

---
 .github/workflows/test-on-droplet-debian-11.yml    |  8 ++++----
 .github/workflows/test-on-droplet-ubuntu-22.04.yml | 10 +++++-----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test-on-droplet-debian-11.yml b/.github/workflows/test-on-droplet-debian-11.yml
index 2917bd4f2..9fc7f8240 100644
--- a/.github/workflows/test-on-droplet-debian-11.yml
+++ b/.github/workflows/test-on-droplet-debian-11.yml
@@ -55,13 +55,13 @@ jobs:
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
           
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 update"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 upgrade -y"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y docker.io apparmor-profiles"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
           
           scp packaging/target/aleph-vm.debian-11.deb root@${DROPLET_IPV4}:/opt
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt -o DPkg::Lock::Timeout=60 install -y /opt/aleph-vm.debian-11.deb"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.debian-11.deb"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
 
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index 4f2ad156f..e6a0ce835 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -56,16 +56,16 @@ jobs:
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
           
           # Ubuntu droplets run upgrades at boot
-          sleep 15
+          sleep 30
           until ! ssh root@${DROPLET_IPV4} "lslocks --json | grep /var/lib/dpkg/lock" > /dev/null; do sleep 1; echo "Waiting for dpkg lock..."; done
           
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 update"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 upgrade -y"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y docker.io apparmor-profiles"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
           
           scp packaging/target/aleph-vm.ubuntu-22.04.deb root@${DROPLET_IPV4}:/opt
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt -o DPkg::Lock::Timeout=60 install -y /opt/aleph-vm.ubuntu-22.04.deb"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.ubuntu-22.04.deb"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_DNS_RESOLUTION=resolvectl >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"

From 95f4dff6821397abcb3020cd8274e1d7318db88b Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Fri, 12 May 2023 17:43:29 +0200
Subject: [PATCH 382/990] Fix: support running migrations on existing CRNs

Problem: the migration script introduced with alembic assumes
that the records table does not exist yet. This makes it impossible
to run the new version on existing CRNs as the table exists,
making the migration statement fail.

The network_tap column was also deleted in a previous commit
after 0.2.5, but the libsqlite3 version shipped with Ubuntu 20.04
does not support `ALTER TABLE ... DROP COLUMN` statements.

Solution: check if the table exists in the migration script.
---
 vm_supervisor/metrics.py                      |  1 +
 .../0001_bbb12a12372e_execution_records.py    | 60 ++++++++++++-------
 vm_supervisor/models.py                       |  1 +
 3 files changed, 40 insertions(+), 22 deletions(-)

diff --git a/vm_supervisor/metrics.py b/vm_supervisor/metrics.py
index 4a1f5d4a4..3d397ff6c 100644
--- a/vm_supervisor/metrics.py
+++ b/vm_supervisor/metrics.py
@@ -50,6 +50,7 @@ class ExecutionRecord(Base):
 
     vcpus = Column(Integer, nullable=False)
     memory = Column(Integer, nullable=False)
+    network_tap = Column(String, nullable=True)
 
     def __repr__(self):
         return f""
diff --git a/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py b/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py
index 1ed84fdae..b210e8cf9 100644
--- a/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py
+++ b/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py
@@ -5,11 +5,15 @@
 Create Date: 2022-09-28 18:52:16.431200
 
 """
-from alembic import op
 import sqlalchemy as sa
-
+from alembic import op
 
 # revision identifiers, used by Alembic.
+from sqlalchemy import create_engine
+from sqlalchemy.engine import reflection
+
+from vm_supervisor.conf import make_db_url
+
 revision = "bbb12a12372e"
 down_revision = None
 branch_labels = None
@@ -17,26 +21,38 @@
 
 
 def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table(
-        "records",
-        sa.Column("uuid", sa.String(), nullable=False),
-        sa.Column("vm_hash", sa.String(), nullable=False),
-        sa.Column("time_defined", sa.DateTime(), nullable=False),
-        sa.Column("time_prepared", sa.DateTime(), nullable=True),
-        sa.Column("time_started", sa.DateTime(), nullable=True),
-        sa.Column("time_stopping", sa.DateTime(), nullable=True),
-        sa.Column("cpu_time_user", sa.Float(), nullable=True),
-        sa.Column("cpu_time_system", sa.Float(), nullable=True),
-        sa.Column("io_read_count", sa.Integer(), nullable=True),
-        sa.Column("io_write_count", sa.Integer(), nullable=True),
-        sa.Column("io_read_bytes", sa.Integer(), nullable=True),
-        sa.Column("io_write_bytes", sa.Integer(), nullable=True),
-        sa.Column("vcpus", sa.Integer(), nullable=False),
-        sa.Column("memory", sa.Integer(), nullable=False),
-        sa.Column("network_tap", sa.String(), nullable=True),
-        sa.PrimaryKeyConstraint("uuid"),
-    )
+    engine = create_engine(make_db_url())
+    inspector = reflection.Inspector.from_engine(engine)
+
+    # The table already exists on most CRNs.
+    tables = inspector.get_table_names()
+    if "records" not in tables:
+        op.create_table(
+            "records",
+            sa.Column("uuid", sa.String(), nullable=False),
+            sa.Column("vm_hash", sa.String(), nullable=False),
+            sa.Column("time_defined", sa.DateTime(), nullable=False),
+            sa.Column("time_prepared", sa.DateTime(), nullable=True),
+            sa.Column("time_started", sa.DateTime(), nullable=True),
+            sa.Column("time_stopping", sa.DateTime(), nullable=True),
+            sa.Column("cpu_time_user", sa.Float(), nullable=True),
+            sa.Column("cpu_time_system", sa.Float(), nullable=True),
+            sa.Column("io_read_count", sa.Integer(), nullable=True),
+            sa.Column("io_write_count", sa.Integer(), nullable=True),
+            sa.Column("io_read_bytes", sa.Integer(), nullable=True),
+            sa.Column("io_write_bytes", sa.Integer(), nullable=True),
+            sa.Column("vcpus", sa.Integer(), nullable=False),
+            sa.Column("memory", sa.Integer(), nullable=False),
+            sa.Column("network_tap", sa.String(), nullable=True),
+            sa.PrimaryKeyConstraint("uuid"),
+        )
+
+    # Support intermediate versions that have the records table
+    # but without the network_tap column
+    records_columns = [column["name"] for column in inspector.get_columns("records")]
+    if "network_tap" not in records_columns:
+        op.add_column("records", sa.Column("network_tap", sa.String(), nullable=True))
+
     # ### end Alembic commands ###
 
 
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index bebaf710e..9e734471c 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -233,6 +233,7 @@ async def record_usage(self):
                     io_write_bytes=pid_info["process"]["io_counters"][3],
                     vcpus=self.vm.hardware_resources.vcpus,
                     memory=self.vm.hardware_resources.memory,
+                    network_tap=self.vm.tap_interface.device_name,
                 )
             )
         else:

From 3311ca05812895011ba9d6328e2a85c2b0d3cf87 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 7 Mar 2023 19:03:13 +0100
Subject: [PATCH 383/990] Fix: Network connectivity was difficult to debug

Solution: Add more checks on the network connectivity, in order to make it easier to analyze issues.
---
 examples/example_fastapi/main.py | 70 ++++++++++++++++++++++++++++++--
 vm_supervisor/status.py          | 30 ++++++++++++++
 vm_supervisor/views.py           |  3 ++
 3 files changed, 99 insertions(+), 4 deletions(-)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index 4ad5f108f..58f876b39 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -1,6 +1,8 @@
 import json
 import logging
 import os
+import socket
+import subprocess
 import sys
 from datetime import datetime
 from os import listdir
@@ -22,6 +24,7 @@
 
 logger.debug("import fastapi")
 from fastapi import FastAPI
+from fastapi.responses import PlainTextResponse
 logger.debug("imports done")
 
 http_app = FastAPI()
@@ -37,8 +40,18 @@ async def index():
         opt_venv = []
     return {
         "Example": "example_fastapi",
-        "endpoints": ["/environ", "/messages", "/internet", "/post_a_message",
-                      "/state/increment", "/wait-for/{delay}"],
+        "endpoints": [
+            "/environ",
+            "/messages",
+            "/dns",
+            "ip/address",
+            "/ip/4",
+            "/ip/6",
+            "/internet",
+            "/post_a_message",
+            "/state/increment",
+            "/wait-for/{delay}",
+        ],
         "files_in_volumes": {
             "/opt/venv": opt_venv,
         },
@@ -60,10 +73,59 @@ async def read_aleph_messages():
     return {"Messages": data}
 
 
+@app.get("/dns")
+async def resolve_dns_hostname():
+    """Check if DNS resolution is working.
+    """
+    info_inet, info_inet6 = socket.getaddrinfo("example.org", 80, proto=socket.IPPROTO_TCP)
+    ipv4 = info_inet[4][0]
+    ipv6 = info_inet6[4][0]
+    return {
+        "ipv4": ipv4,
+        "ipv6": ipv6,
+    }
+
+
+@app.get("/ip/address")
+async def ip_address():
+    """Fetch the ip addresses of the virtual machine."""
+    output = subprocess.check_output(["ip", "addr"], shell=False)
+    return PlainTextResponse(content=output)
+
+
+@app.get("/ip/4")
+async def connect_ipv4():
+    """Connect to the Quad9 VPN provider using their IPv4 address.
+    The webserver on that address returns a 404 error, so we accept that response code.
+    """
+    timeout = aiohttp.ClientTimeout(total=5)
+    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
+        async with session.get("https://9.9.9.9") as resp:
+            # We expect this endpoint to return a 404 error
+            if resp.status != 404:
+                resp.raise_for_status()
+            return {"result": True, "headers": resp.headers}
+
+
+@app.get("/ip/6")
+async def connect_ipv6():
+    """Connect to the Quad9 VPN provider using their IPv6 address.
+    The webserver on that address returns a 404 error, so we accept that response code.
+    """
+    timeout = aiohttp.ClientTimeout(total=5)
+    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
+        async with session.get("https://[2620:fe::fe]") as resp:
+            # We expect this endpoint to return a 404 error
+            if resp.status != 404:
+                resp.raise_for_status()
+            return {"result": True, "headers": resp.headers}
+
+
 @app.get("/internet")
 async def read_internet():
-    """Read data from the public Internet using aiohttp."""
-    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector()) as session:
+    """Connect the aleph.im official website to check Internet connectivity."""
+    timeout = aiohttp.ClientTimeout(total=5)
+    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
         async with session.get("https://aleph.im/") as resp:
             resp.raise_for_status()
             return {"result": resp.status, "headers": resp.headers}
diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py
index 37156f81e..02c462ef2 100644
--- a/vm_supervisor/status.py
+++ b/vm_supervisor/status.py
@@ -54,6 +54,36 @@ async def check_messages(session: ClientSession) -> bool:
         return False
 
 
+async def check_dns(session: ClientSession) -> bool:
+    try:
+        result: Dict = await get_json_from_vm(session, "/dns")
+        assert result["ipv4"]
+        assert result["ipv6"]
+        return True
+    except ClientResponseError:
+        return False
+
+
+async def check_ipv4(session: ClientSession) -> bool:
+    try:
+        result: Dict = await get_json_from_vm(session, "/ip/4")
+        assert result["result"] is True
+        assert "headers" in result
+        return True
+    except ClientResponseError:
+        return False
+
+
+async def check_ipv6(session: ClientSession) -> bool:
+    try:
+        result: Dict = await get_json_from_vm(session, "/ip/6")
+        assert result["result"] is True
+        assert "headers" in result
+        return True
+    except ClientResponseError:
+        return False
+
+
 async def check_internet(session: ClientSession) -> bool:
     try:
         result: Dict = await get_json_from_vm(session, "/internet")
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index 4b21dd2f9..fda5c7889 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -143,6 +143,9 @@ async def status_check_fastapi(request: web.Request):
             "index": await status.check_index(session),
             "environ": await status.check_environ(session),
             "messages": await status.check_messages(session),
+            "dns": await status.check_dns(session),
+            "ipv4": await status.check_ipv4(session),
+            # "ipv6": await status.check_ipv6(session),
             "internet": await status.check_internet(session),
             "cache": await status.check_cache(session),
             "persistent_storage": await status.check_persistent_storage(session),

From 2da7b3b4e6043c54c11fc5ce3d87ca8ed38bf364 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 12 May 2023 17:38:19 +0200
Subject: [PATCH 384/990] Reformat examples/example_fastapi/main.py with black

---
 examples/example_fastapi/main.py | 56 ++++++++++++++++++++------------
 1 file changed, 35 insertions(+), 21 deletions(-)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index 58f876b39..5f66f8624 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -16,15 +16,16 @@
 import aiohttp
 
 logger.debug("import aleph_client")
-from aleph_client.types import StorageEnum
-from aleph_client.asynchronous import get_messages, create_post
+from aleph_client.asynchronous import create_post, get_messages
 from aleph_client.chains.remote import RemoteAccount
-from aleph_client.vm.cache import VmCache
+from aleph_client.types import StorageEnum
 from aleph_client.vm.app import AlephApp
+from aleph_client.vm.cache import VmCache
 
 logger.debug("import fastapi")
 from fastapi import FastAPI
 from fastapi.responses import PlainTextResponse
+
 logger.debug("imports done")
 
 http_app = FastAPI()
@@ -75,9 +76,10 @@ async def read_aleph_messages():
 
 @app.get("/dns")
 async def resolve_dns_hostname():
-    """Check if DNS resolution is working.
-    """
-    info_inet, info_inet6 = socket.getaddrinfo("example.org", 80, proto=socket.IPPROTO_TCP)
+    """Check if DNS resolution is working."""
+    info_inet, info_inet6 = socket.getaddrinfo(
+        "example.org", 80, proto=socket.IPPROTO_TCP
+    )
     ipv4 = info_inet[4][0]
     ipv6 = info_inet6[4][0]
     return {
@@ -99,7 +101,9 @@ async def connect_ipv4():
     The webserver on that address returns a 404 error, so we accept that response code.
     """
     timeout = aiohttp.ClientTimeout(total=5)
-    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
+    async with aiohttp.ClientSession(
+        connector=aiohttp.TCPConnector(), timeout=timeout
+    ) as session:
         async with session.get("https://9.9.9.9") as resp:
             # We expect this endpoint to return a 404 error
             if resp.status != 404:
@@ -113,7 +117,9 @@ async def connect_ipv6():
     The webserver on that address returns a 404 error, so we accept that response code.
     """
     timeout = aiohttp.ClientTimeout(total=5)
-    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
+    async with aiohttp.ClientSession(
+        connector=aiohttp.TCPConnector(), timeout=timeout
+    ) as session:
         async with session.get("https://[2620:fe::fe]") as resp:
             # We expect this endpoint to return a 404 error
             if resp.status != 404:
@@ -125,7 +131,9 @@ async def connect_ipv6():
 async def read_internet():
     """Connect the aleph.im official website to check Internet connectivity."""
     timeout = aiohttp.ClientTimeout(total=5)
-    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
+    async with aiohttp.ClientSession(
+        connector=aiohttp.TCPConnector(), timeout=timeout
+    ) as session:
         async with session.get("https://aleph.im/") as resp:
             resp.raise_for_status()
             return {"result": resp.status, "headers": resp.headers}
@@ -136,7 +144,8 @@ async def post_a_message():
     """Post a message on the Aleph network"""
 
     account = await RemoteAccount.from_crypto_host(
-        host="http://localhost", unix_socket="/tmp/socat-socket")
+        host="http://localhost", unix_socket="/tmp/socat-socket"
+    )
 
     content = {
         "date": datetime.utcnow().isoformat(),
@@ -176,11 +185,13 @@ async def remove_from_cache(key: str):
     result = await cache.delete(key)
     return result == 1
 
+
 @app.get("/cache/keys")
-async def keys_from_cache(pattern: str = '*'):
+async def keys_from_cache(pattern: str = "*"):
     """List keys from the VM cache"""
     return await cache.keys(pattern)
 
+
 @app.get("/state/increment")
 async def increment():
     path = "/var/lib/example/storage.json"
@@ -190,7 +201,7 @@ async def increment():
         data["counter"] += 1
     except FileNotFoundError:
         data = {"counter": 0}
-    with open(path, 'w') as fd:
+    with open(path, "w") as fd:
         json.dump(data, fd)
     return data
 
@@ -221,18 +232,21 @@ def crash():
     sys.exit(1)
 
 
-filters = [{
-    # "sender": "0xB31B787AdA86c6067701d4C0A250c89C7f1f29A5",
-    "channel": "TEST"
-}]
+filters = [
+    {
+        # "sender": "0xB31B787AdA86c6067701d4C0A250c89C7f1f29A5",
+        "channel": "TEST"
+    }
+]
+
 
 @app.event(filters=filters)
 async def aleph_event(event):
     print("aleph_event", event)
     async with aiohttp.ClientSession(connector=aiohttp.TCPConnector()) as session:
-        async with session.get("https://official.aleph.cloud/api/v0/info/public.json") as resp:
-            print('RESP', resp)
+        async with session.get(
+            "https://official.aleph.cloud/api/v0/info/public.json"
+        ) as resp:
+            print("RESP", resp)
             resp.raise_for_status()
-    return {
-        "result": "Good"
-    }
+    return {"result": "Good"}

From 9a1a5b2fa8a6b6ae8a2997608bdf5175a3fb7abc Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 12 May 2023 16:22:56 +0200
Subject: [PATCH 385/990] Fix: Runtime and new example were not tested in CI

---
 .../workflows/test-new-runtime-examples.yml   | 103 ++++++++++++++++++
 1 file changed, 103 insertions(+)
 create mode 100644 .github/workflows/test-new-runtime-examples.yml

diff --git a/.github/workflows/test-new-runtime-examples.yml b/.github/workflows/test-new-runtime-examples.yml
new file mode 100644
index 000000000..a8bf34a4d
--- /dev/null
+++ b/.github/workflows/test-new-runtime-examples.yml
@@ -0,0 +1,103 @@
+name: "Test new runtime and examples"
+on:
+  push
+
+jobs:
+  run_debian_11:
+    name: "Run in DigitalOcean Droplet with Debian 11"
+    runs-on: ubuntu-latest
+    concurrency: droplet-aleph-vm-debian-11
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          # Fetch the whole history for all tags and branches (required for aleph.__version__)
+          fetch-depth: 0
+
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
+      - name: Install doctl
+        uses: digitalocean/action-doctl@v2
+        with:
+          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+
+      - name: Setup SSH private key
+        run: |
+          mkdir ~/.ssh
+          echo $DIGITALOCEAN_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519
+          chmod 0700 ~/.ssh
+          chmod 0600 ~/.ssh/id_ed25519
+        env:
+          DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }}
+
+      - name: Create the Droplet
+        run: |
+          doctl compute droplet create \
+          --image debian-11-x64 \
+          --size c-2 \
+          --region fra1 \
+          --vpc-uuid 8c422d04-5dfa-4eca-add7-1e41b5f60d39 \
+          --enable-ipv6 \
+          --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
+          aleph-vm-ci-debian-11
+
+      - name: Build custom runtime
+      - run: |
+          sudo apt update
+          sudo apt install -y debootstrap
+          cd runtimes/aleph-debian-11-python && sudo ./create_disk_image.sh && cd ../..
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: aleph-debian-11-python.squashfs
+          path: runtimes/aleph-debian-11-python/rootfs.squashfs
+
+      - name: Build Debian Package
+        run: |
+          cd packaging && make all-podman-debian-11 && cd ..
+          ls packaging/target
+
+      - name: Wait for the system to setup and boot
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
+
+      - name: Copy the runtime to the system
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
+          scp runtimes/aleph-debian-11-python/rootfs.squashfs root@${DROPLET_IPV4}:/opt
+
+      - name: Install Aleph-VM on the Droplet
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
+          
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 update"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 upgrade -y"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y docker.io apparmor-profiles"
+          ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
+          
+          scp packaging/target/aleph-vm.debian-11.deb root@${DROPLET_IPV4}:/opt
+          scp -pr ./examples root@${DROPLET_IPV4}:/opt/
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt -o DPkg::Lock::Timeout=60 install -y /opt/aleph-vm.debian-11.deb"
+          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
+          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_FAKE_DATA_PROGRAM=/opt/examples/example_fastapi >> /etc/aleph-vm/supervisor.env"
+          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_FAKE_DATA_RUNTIME=/opt/rootfs.squashfs >> /etc/aleph-vm/supervisor.env"
+          ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
+
+      - name: Test Aleph-VM on the Droplet
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          
+          sleep 3
+          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
+          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
+
+      - name: Cleanup
+        if: always()
+        run: |
+          doctl compute droplet delete -f aleph-vm-ci-debian-11
+

From 13af11a80784bc34646089922f9a3f6d4532e8d3 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 17 May 2023 03:06:40 +0200
Subject: [PATCH 386/990] Fix: Job names were inprecise

---
 .../workflows/test-new-runtime-examples.yml   | 21 +++++++++----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/test-new-runtime-examples.yml b/.github/workflows/test-new-runtime-examples.yml
index a8bf34a4d..de8c917a7 100644
--- a/.github/workflows/test-new-runtime-examples.yml
+++ b/.github/workflows/test-new-runtime-examples.yml
@@ -4,9 +4,9 @@ on:
 
 jobs:
   run_debian_11:
-    name: "Run in DigitalOcean Droplet with Debian 11"
+    name: "Test new runtime on Droplet with Debian 11"
     runs-on: ubuntu-latest
-    concurrency: droplet-aleph-vm-debian-11
+    concurrency: droplet-aleph-vm-runtime
 
     steps:
       - name: Checkout repository
@@ -41,9 +41,9 @@ jobs:
           --vpc-uuid 8c422d04-5dfa-4eca-add7-1e41b5f60d39 \
           --enable-ipv6 \
           --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
-          aleph-vm-ci-debian-11
-
-      - name: Build custom runtime
+          aleph-vm-ci-runtime
+      
+      - name: "Build custom runtime"
       - run: |
           sudo apt update
           sudo apt install -y debootstrap
@@ -61,18 +61,18 @@ jobs:
 
       - name: Wait for the system to setup and boot
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
 
       - name: Copy the runtime to the system
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
           scp runtimes/aleph-debian-11-python/rootfs.squashfs root@${DROPLET_IPV4}:/opt
 
       - name: Install Aleph-VM on the Droplet
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
           
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 update"
@@ -90,7 +90,7 @@ jobs:
 
       - name: Test Aleph-VM on the Droplet
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           
           sleep 3
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
@@ -99,5 +99,4 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          doctl compute droplet delete -f aleph-vm-ci-debian-11
-
+          doctl compute droplet delete -f aleph-vm-ci-runtime

From 8c0154e5b6a5b3ca697c7e7cd283de23f6823f54 Mon Sep 17 00:00:00 2001
From: aliel 
Date: Fri, 26 May 2023 16:54:56 +0200
Subject: [PATCH 387/990] Fix ip command path #313

---
 vm_supervisor/network/interfaces.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/vm_supervisor/network/interfaces.py b/vm_supervisor/network/interfaces.py
index 271e44f08..a121c6356 100644
--- a/vm_supervisor/network/interfaces.py
+++ b/vm_supervisor/network/interfaces.py
@@ -4,6 +4,7 @@
 from subprocess import run
 
 from .ipaddresses import IPv4NetworkWithInterfaces
+import shutil
 
 logger = logging.getLogger(__name__)
 
@@ -27,10 +28,11 @@ def host_ip(self) -> IPv4Interface:
     async def create(self):
         logger.debug("Create network interface")
 
-        run(["/usr/bin/ip", "tuntap", "add", self.device_name, "mode", "tap"])
+        ip_command = shutil.which("ip")
+        run([ip_command, "tuntap", "add", self.device_name, "mode", "tap"])
         run(
             [
-                "/usr/bin/ip",
+                ip_command,
                 "addr",
                 "add",
                 str(self.host_ip.with_prefixlen),
@@ -38,7 +40,7 @@ async def create(self):
                 self.device_name,
             ]
         )
-        run(["/usr/bin/ip", "link", "set", self.device_name, "up"])
+        run([ip_command, "link", "set", self.device_name, "up"])
         logger.debug(f"Network interface created: {self.device_name}")
 
     async def delete(self) -> None:

From 20efb02b1a5b49af28871538e2135b7fd6efd043 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 31 May 2023 16:08:09 +0200
Subject: [PATCH 388/990] Fix: Incompatibilities with aleph-message 0.4.0

---
 docker/vm_supervisor-dev.dockerfile     | 2 +-
 examples/volumes/Dockerfile             | 2 +-
 packaging/Makefile                      | 2 +-
 vm_supervisor/README.md                 | 2 +-
 vm_supervisor/reactor.py                | 6 +++---
 vm_supervisor/resources.py              | 2 +-
 vm_supervisor/storage.py                | 6 ++++--
 vm_supervisor/tasks.py                  | 4 ++--
 vm_supervisor/vm/firecracker_microvm.py | 3 ++-
 9 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 77b00c1a7..901bd8c4f 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -19,7 +19,7 @@ RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://s3.amazonaws.com/spec.ccf
 RUN ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker
 RUN ln /opt/firecracker/release-*/jailer-v* /opt/firecracker/jailer
 
-RUN pip3 install typing-extensions 'aleph-message==0.2.2'
+RUN pip3 install typing-extensions 'aleph-message==0.4.0a2'
 
 RUN mkdir -p /var/lib/aleph/vm/jailer
 
diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile
index 4ce6a6c10..d21bffb23 100644
--- a/examples/volumes/Dockerfile
+++ b/examples/volumes/Dockerfile
@@ -6,6 +6,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 RUN python3 -m venv /opt/venv
-RUN /opt/venv/bin/pip install 'aleph-message==0.2.2'
+RUN /opt/venv/bin/pip install 'aleph-message==0.4.0a2'
 
 CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs
diff --git a/packaging/Makefile b/packaging/Makefile
index 0ac1b744b..daa3bad6a 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -16,7 +16,7 @@ debian-package-code:
 	cp ../examples/message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.3.0'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.0a2'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index b94fa5431..846854591 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -87,7 +87,7 @@ is used to parse and validate Aleph messages.
 ```shell
 apt install -y --no-install-recommends --no-install-suggests python3-pip
 pip3 install pydantic[dotenv]
-pip3 install 'aleph-message==0.2.2'
+pip3 install 'aleph-message==0.4.0a2'
 ```
 
 ### 2.f. Create the jailer working directory:
diff --git a/vm_supervisor/reactor.py b/vm_supervisor/reactor.py
index e87ecd1e8..a8eb4d9f7 100644
--- a/vm_supervisor/reactor.py
+++ b/vm_supervisor/reactor.py
@@ -1,8 +1,8 @@
 import logging
 from typing import Coroutine, List
 
-from aleph_message.models import Message, ProgramMessage
-from aleph_message.models.program import Subscription
+from aleph_message.models import ProgramMessage, AlephMessage
+from aleph_message.models.execution.environment import Subscription
 
 from .pubsub import PubSub
 from .run import run_code_on_event
@@ -44,7 +44,7 @@ def __init__(self, pubsub: PubSub):
         self.pubsub = pubsub
         self.listeners = []
 
-    async def trigger(self, message: Message):
+    async def trigger(self, message: AlephMessage):
         coroutines: List[Coroutine] = []
 
         for listener in self.listeners:
diff --git a/vm_supervisor/resources.py b/vm_supervisor/resources.py
index 27f86e088..80341bb3c 100644
--- a/vm_supervisor/resources.py
+++ b/vm_supervisor/resources.py
@@ -5,7 +5,7 @@
 import cpuinfo
 import psutil
 from aiohttp import web
-from aleph_message.models.program import CpuProperties
+from aleph_message.models.execution.environment import CpuProperties
 from pydantic import BaseModel
 
 from .conf import settings
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 35c5c80b7..5869571ca 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -17,8 +17,10 @@
 
 import aiohttp
 from aleph_message.models import ProgramMessage
-from aleph_message.models.program import (
+from aleph_message.models.execution.base import (
     Encoding,
+)
+from aleph_message.models.execution.volume import (
     ImmutableVolume,
     MachineVolume,
     PersistentVolume,
@@ -92,7 +94,7 @@ async def get_message(ref: str) -> ProgramMessage:
             msg["item_hash"] = hashlib.sha256(
                 msg["item_content"].encode("utf-8")
             ).hexdigest()
-        return ProgramMessage(**msg)
+        return ProgramMessage.parse_obj(msg)
 
 
 async def get_code_path(ref: str) -> Path:
diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py
index bdcc8bc76..f2b78c99c 100644
--- a/vm_supervisor/tasks.py
+++ b/vm_supervisor/tasks.py
@@ -8,7 +8,7 @@
 import aiohttp
 import pydantic
 from aiohttp import web
-from aleph_message import Message
+from aleph_message import parse_message
 from aleph_message.models import BaseMessage, ProgramMessage
 from yarl import URL
 
@@ -63,7 +63,7 @@ async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]:
                         continue
 
                     try:
-                        yield Message(**data)
+                        yield parse_message(data)
                     except pydantic.error_wrappers.ValidationError as error:
                         logger.error(
                             f"Invalid Aleph message: \n  {error.json()}\n  {error.raw_errors}",
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 2c8506476..549aa570b 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -18,7 +18,8 @@
     psutil = None
 from aiohttp import ClientResponseError
 from aleph_message.models import ProgramContent
-from aleph_message.models.program import Encoding, MachineResources
+from aleph_message.models.execution.base import Encoding
+from aleph_message.models.execution.environment import MachineResources
 
 from firecracker.config import (
     BootSource,

From 29b820bb9ed9624104b087a2c93fda12c4dd43a5 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 31 May 2023 16:49:54 +0200
Subject: [PATCH 389/990] Fix: Remove deprecated runtime "alpine"

The "alpine" runtime was not maintained and can be classified as
deprecated.

This deletes the repository, moving the init scripts to the
"aleph-debian-11-python" directory.
---
 .../create_disk_image.sh                      |  90 ---
 runtimes/aleph-alpine-3.13-python/init0.sh    |  56 --
 runtimes/aleph-alpine-3.13-python/init1.py    | 566 -----------------
 .../aleph-alpine-3.13-python/update_inits.sh  |  16 -
 runtimes/aleph-debian-11-python/init0.sh      |  57 +-
 runtimes/aleph-debian-11-python/init1.py      | 567 +++++++++++++++++-
 6 files changed, 622 insertions(+), 730 deletions(-)
 delete mode 100644 runtimes/aleph-alpine-3.13-python/create_disk_image.sh
 delete mode 100644 runtimes/aleph-alpine-3.13-python/init0.sh
 delete mode 100644 runtimes/aleph-alpine-3.13-python/init1.py
 delete mode 100755 runtimes/aleph-alpine-3.13-python/update_inits.sh
 mode change 120000 => 100644 runtimes/aleph-debian-11-python/init0.sh
 mode change 120000 => 100644 runtimes/aleph-debian-11-python/init1.py

diff --git a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh b/runtimes/aleph-alpine-3.13-python/create_disk_image.sh
deleted file mode 100644
index 9144227f2..000000000
--- a/runtimes/aleph-alpine-3.13-python/create_disk_image.sh
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/bin/sh
-
-umount /mnt/rootfs
-
-set -euf
-
-curl -fsSL -o ./alpine-miniroot.tgz https://dl-cdn.alpinelinux.org/alpine/v3.13/releases/x86_64/alpine-minirootfs-3.13.5-x86_64.tar.gz
-
-dd if=/dev/zero of=./rootfs.ext4 bs=1M count=500
-mkfs.ext4 ./rootfs.ext4
-mkdir -p /mnt/rootfs
-mount ./rootfs.ext4 /mnt/rootfs
-tar --preserve-permissions --same-owner -xf  ./alpine-miniroot.tgz --directory /mnt/rootfs
-
-cat /etc/resolv.conf > /mnt/rootfs/etc/resolv.conf
-
-chroot /mnt/rootfs /bin/sh <=0.2.7' 'coincurve==15.0.0'
-
-# Compile all Python bytecode
-python3 -m compileall -f /usr/lib/python3.8/site-packages
-
-echo -e "toor\ntoor" | passwd root
-
-mkdir -p /overlay
-
-## Generate SSH host keys
-#ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
-#ssh-keygen -q -N "" -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key
-#ssh-keygen -q -N "" -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key
-#ssh-keygen -q -N "" -t ed25519 -f /etc/ssh/ssh_host_ed25519_key
-
-# Set up a login terminal on the serial console (ttyS0):
-ln -s agetty /etc/init.d/agetty.ttyS0
-echo ttyS0 > /etc/securetty
-EOT
-
-echo "PermitRootLogin yes" >> /mnt/rootfs/etc/ssh/sshd_config
-
-# Generate SSH host keys
-systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
-systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key
-systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key
-systemd-nspawn -D /mnt/rootfs/ ssh-keygen -q -N "" -t ed25519 -f /etc/ssh/ssh_host_ed25519_key
-
-cat < /mnt/rootfs/etc/inittab
-# /etc/inittab
-
-::sysinit:/sbin/init sysinit
-::sysinit:/sbin/init boot
-::wait:/sbin/init default
-
-# Set up a couple of getty's
-tty1::respawn:/sbin/getty 38400 tty1
-tty2::respawn:/sbin/getty 38400 tty2
-tty3::respawn:/sbin/getty 38400 tty3
-tty4::respawn:/sbin/getty 38400 tty4
-tty5::respawn:/sbin/getty 38400 tty5
-tty6::respawn:/sbin/getty 38400 tty6
-
-# Put a getty on the serial port
-ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100
-
-# Stuff to do for the 3-finger salute
-::ctrlaltdel:/sbin/reboot
-
-# Stuff to do before rebooting
-::shutdown:/sbin/init shutdown
-EOT
-
-# Custom init
-mv /mnt/rootfs/sbin/init /mnt/rootfs/sbin/init.copy
-cp ./init0.sh /mnt/rootfs/sbin/init
-cp ./init1.py /mnt/rootfs/root/init1.py
-chmod +x /mnt/rootfs/sbin/init
-chmod +x /mnt/rootfs/root/init1.py
-
-umount /mnt/rootfs
diff --git a/runtimes/aleph-alpine-3.13-python/init0.sh b/runtimes/aleph-alpine-3.13-python/init0.sh
deleted file mode 100644
index 8eb1b62bf..000000000
--- a/runtimes/aleph-alpine-3.13-python/init0.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/sh
-
-set -euf
-
-mount -t proc proc /proc -o nosuid,noexec,nodev
-
-log() {
-    echo "$(cat /proc/uptime | awk '{printf $1}')" '|S' "$@"
-}
-log "init0.sh is launching"
-
-# Switch root from read-only ext4 to to read-write overlay
-mkdir -p /overlay
-/bin/mount -t tmpfs -o noatime,mode=0755 tmpfs /overlay
-mkdir -p /overlay/root /overlay/work
-/bin/mount -o noatime,lowerdir=/,upperdir=/overlay/root,workdir=/overlay/work -t overlay "overlayfs:/overlay/root" /mnt
-mkdir -p /mnt/rom
-pivot_root /mnt /mnt/rom
-
-mount --move /rom/proc /proc
-mount --move /rom/dev /dev
-
-mkdir -p /dev/pts
-mkdir -p /dev/shm
-
-mount -t sysfs sys /sys -o nosuid,noexec,nodev
-mount -t tmpfs run /run -o mode=0755,nosuid,nodev
-#mount -t devtmpfs dev /dev -o mode=0755,nosuid
-mount -t devpts devpts /dev/pts -o mode=0620,gid=5,nosuid,noexec
-mount -t tmpfs shm /dev/shm -omode=1777,nosuid,nodev
-
-# Required by Docker
-cgroupfs-mount
-update-alternatives --set iptables /usr/sbin/iptables-legacy
-update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
-
-# Enable the following to force the storage driver used by Docker.
-# See https://docs.docker.com/storage/storagedriver/select-storage-driver/
-#echo '{\n"storage-driver": "overlay2"\n}\n' > /etc/docker/daemon.json
-
-# List block devices
-lsblk
-
-#cat /proc/sys/kernel/random/entropy_avail
-
-# TODO: Move in init1
-mkdir -p /run/sshd
-/usr/sbin/sshd &
-log "SSH UP"
-
-log "Setup socat"
-socat UNIX-LISTEN:/tmp/socat-socket,fork,reuseaddr VSOCK-CONNECT:2:53 &
-log "Socat ready"
-
-# Replace this script with the manager
-exec /root/init1.py
diff --git a/runtimes/aleph-alpine-3.13-python/init1.py b/runtimes/aleph-alpine-3.13-python/init1.py
deleted file mode 100644
index 47f435f62..000000000
--- a/runtimes/aleph-alpine-3.13-python/init1.py
+++ /dev/null
@@ -1,566 +0,0 @@
-#!/usr/bin/python3 -OO
-import base64
-import logging
-
-logging.basicConfig(
-    level=logging.DEBUG,
-    format="%(relativeCreated)4f |V %(levelname)s | %(message)s",
-)
-logger = logging.getLogger(__name__)
-
-logger.debug("Imports starting")
-
-import ctypes
-import asyncio
-import os
-import socket
-from enum import Enum
-import subprocess
-import sys
-import traceback
-from contextlib import redirect_stdout
-from dataclasses import dataclass, field
-from io import StringIO
-from os import system
-from shutil import make_archive
-from typing import Optional, Dict, Any, Tuple, List, NewType, Union, AsyncIterable
-
-import aiohttp
-import msgpack
-
-logger.debug("Imports finished")
-
-ASGIApplication = NewType("ASGIApplication", Any)
-
-
-class Encoding(str, Enum):
-    plain = "plain"
-    zip = "zip"
-    squashfs = "squashfs"
-
-
-class Interface(str, Enum):
-    asgi = "asgi"
-    executable = "executable"
-
-
-class ShutdownException(Exception):
-    pass
-
-
-@dataclass
-class Volume:
-    mount: str
-    device: str
-    read_only: bool
-
-
-@dataclass
-class ConfigurationPayload:
-    code: bytes
-    encoding: Encoding
-    entrypoint: str
-    input_data: bytes
-    interface: Interface
-    vm_hash: str
-    ip: Optional[str] = None
-    route: Optional[str] = None
-    dns_servers: List[str] = field(default_factory=list)
-    volumes: List[Volume] = field(default_factory=list)
-    variables: Optional[Dict[str, str]] = None
-
-
-@dataclass
-class RunCodePayload:
-    scope: Dict
-
-
-# Open a socket to receive instructions from the host
-s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
-s.bind((socket.VMADDR_CID_ANY, 52))
-s.listen()
-
-# Send the host that we are ready
-s0 = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
-s0.connect((2, 52))
-s0.close()
-
-# Configure aleph-client to use the guest API
-os.environ["ALEPH_API_HOST"] = "http://localhost"
-os.environ["ALEPH_API_UNIX_SOCKET"] = "/tmp/socat-socket"
-os.environ["ALEPH_REMOTE_CRYPTO_HOST"] = "http://localhost"
-os.environ["ALEPH_REMOTE_CRYPTO_UNIX_SOCKET"] = "/tmp/socat-socket"
-
-logger.debug("init1.py is launching")
-
-
-def setup_hostname(hostname: str):
-    os.environ["ALEPH_ADDRESS_TO_USE"] = hostname
-    system(f"hostname {hostname}")
-
-
-def setup_variables(variables: Optional[Dict[str, str]]):
-    if variables is None:
-        return
-    for key, value in variables.items():
-        os.environ[key] = value
-
-
-def setup_network(
-    ip: Optional[str], route: Optional[str], dns_servers: Optional[List[str]] = None
-):
-    """Setup the system with info from the host."""
-    dns_servers = dns_servers or []
-    if not os.path.exists("/sys/class/net/eth0"):
-        logger.info("No network interface eth0")
-        return
-
-    if not ip:
-        logger.info("No network IP")
-        return
-
-    logger.debug("Setting up networking")
-    system("ip addr add 127.0.0.1/8 dev lo brd + scope host")
-    system("ip addr add ::1/128 dev lo")
-    system("ip link set lo up")
-    if "/" in ip:
-        # Forward compatibility with future supervisors that pass the mask with the IP.
-        system(f"ip addr add {ip} dev eth0")
-    else:
-        logger.warning("Not passing the mask with the IP is deprecated and will be unsupported")
-        system(f"ip addr add {ip}/24 dev eth0")
-    system("ip link set eth0 up")
-
-    if route:
-        system(f"ip route add default via {route} dev eth0")
-        logger.debug(f"IP and route set: {ip} via {route}")
-    else:
-        logger.warning("IP set with no network route")
-
-    with open("/etc/resolv.conf", "wb") as resolvconf_fd:
-        for server in dns_servers:
-            resolvconf_fd.write(f"nameserver {server}\n".encode())
-
-
-def setup_input_data(input_data: bytes):
-    logger.debug("Extracting data")
-    if input_data:
-        # Unzip in /data
-        if not os.path.exists("/opt/input.zip"):
-            open("/opt/input.zip", "wb").write(input_data)
-            os.makedirs("/data", exist_ok=True)
-            os.system("unzip -q /opt/input.zip -d /data")
-
-
-def setup_volumes(volumes: List[Volume]):
-    for volume in volumes:
-        logger.debug(f"Mounting /dev/{volume.device} on {volume.mount}")
-        os.makedirs(volume.mount, exist_ok=True)
-        if volume.read_only:
-            system(f"mount -t squashfs -o ro /dev/{volume.device} {volume.mount}")
-        else:
-            system(f"mount -o rw /dev/{volume.device} {volume.mount}")
-
-    system("mount")
-
-
-def setup_code_asgi(
-    code: bytes, encoding: Encoding, entrypoint: str
-) -> ASGIApplication:
-    # Allow importing packages from /opt/packages
-    sys.path.append("/opt/packages")
-
-    logger.debug("Extracting code")
-    app: ASGIApplication
-    if encoding == Encoding.squashfs:
-        sys.path.append("/opt/code")
-        module_name, app_name = entrypoint.split(":", 1)
-        logger.debug("import module")
-        module = __import__(module_name)
-        for level in module_name.split(".")[1:]:
-            module = getattr(module, level)
-        app = getattr(module, app_name)
-    elif encoding == Encoding.zip:
-        # Unzip in /opt and import the entrypoint from there
-        if not os.path.exists("/opt/archive.zip"):
-            open("/opt/archive.zip", "wb").write(code)
-            logger.debug("Run unzip")
-            os.system("unzip -q /opt/archive.zip -d /opt")
-        sys.path.append("/opt")
-        module_name, app_name = entrypoint.split(":", 1)
-        logger.debug("import module")
-        module = __import__(module_name)
-        for level in module_name.split(".")[1:]:
-            module = getattr(module, level)
-        app = getattr(module, app_name)
-    elif encoding == Encoding.plain:
-        # Execute the code and extract the entrypoint
-        locals: Dict[str, Any] = {}
-        exec(code, globals(), locals)
-        app = locals[entrypoint]
-    else:
-        raise ValueError(f"Unknown encoding '{encoding}'")
-    return app
-
-
-def setup_code_executable(
-    code: bytes, encoding: Encoding, entrypoint: str
-) -> subprocess.Popen:
-    logger.debug("Extracting code")
-    if encoding == Encoding.squashfs:
-        path = f"/opt/code/{entrypoint}"
-        if not os.path.isfile(path):
-            os.system("find /opt/code/")
-            raise FileNotFoundError(f"No such file: {path}")
-        os.system(f"chmod +x {path}")
-    elif encoding == Encoding.zip:
-        open("/opt/archive.zip", "wb").write(code)
-        logger.debug("Run unzip")
-        os.makedirs("/opt/code", exist_ok=True)
-        os.system("unzip /opt/archive.zip -d /opt/code")
-        path = f"/opt/code/{entrypoint}"
-        if not os.path.isfile(path):
-            os.system("find /opt/code")
-            raise FileNotFoundError(f"No such file: {path}")
-        os.system(f"chmod +x {path}")
-    elif encoding == Encoding.plain:
-        os.makedirs("/opt/code", exist_ok=True)
-        path = f"/opt/code/executable {entrypoint}"
-        open(path, "wb").write(code)
-        os.system(f"chmod +x {path}")
-    else:
-        raise ValueError(f"Unknown encoding '{encoding}'. This should never happen.")
-
-    process = subprocess.Popen(path)
-    return process
-
-
-def setup_code(
-    code: bytes, encoding: Encoding, entrypoint: str, interface: Interface
-) -> Union[ASGIApplication, subprocess.Popen]:
-
-    if interface == Interface.asgi:
-        return setup_code_asgi(code=code, encoding=encoding, entrypoint=entrypoint)
-    elif interface == Interface.executable:
-        return setup_code_executable(
-            code=code, encoding=encoding, entrypoint=entrypoint
-        )
-    else:
-        raise ValueError("Invalid interface. This should never happen.")
-
-
-async def run_python_code_http(
-    application: ASGIApplication, scope: dict
-) -> Tuple[Dict, Dict, str, Optional[bytes]]:
-
-    logger.debug("Running code")
-    with StringIO() as buf, redirect_stdout(buf):
-        # Execute in the same process, saves ~20ms than a subprocess
-
-        # The body should not be part of the ASGI scope itself
-        body: bytes = scope.pop("body")
-
-        async def receive():
-            type_ = (
-                "http.request"
-                if scope["type"] in ("http", "websocket")
-                else "aleph.message"
-            )
-            return {"type": type_, "body": body, "more_body": False}
-
-        send_queue: asyncio.Queue = asyncio.Queue()
-
-        async def send(dico):
-            await send_queue.put(dico)
-
-        # TODO: Better error handling
-        logger.debug("Awaiting application...")
-        await application(scope, receive, send)
-
-        logger.debug("Waiting for headers")
-        headers: Dict
-        if scope["type"] == "http":
-            headers = await send_queue.get()
-        else:
-            headers = {}
-
-        logger.debug("Waiting for body")
-        body: Dict = await send_queue.get()
-
-        logger.debug("Waiting for buffer")
-        output = buf.getvalue()
-
-        logger.debug(f"Headers {headers}")
-        logger.debug(f"Body {body}")
-        logger.debug(f"Output {output}")
-
-    logger.debug("Getting output data")
-    output_data: bytes
-    if os.path.isdir("/data") and os.listdir("/data"):
-        make_archive("/opt/output", "zip", "/data")
-        with open("/opt/output.zip", "rb") as output_zipfile:
-            output_data = output_zipfile.read()
-    else:
-        output_data = b""
-
-    logger.debug("Returning result")
-    return headers, body, output, output_data
-
-
-async def make_request(session, scope):
-    async with session.request(
-        scope["method"],
-        url="http://localhost:8080{}".format(scope["path"]),
-        params=scope["query_string"],
-        headers=[(a.decode("utf-8"), b.decode("utf-8")) for a, b in scope["headers"]],
-        data=scope.get("body", None),
-    ) as resp:
-        headers = {
-            "headers": [
-                (a.encode("utf-8"), b.encode("utf-8")) for a, b in resp.headers.items()
-            ],
-            "status": resp.status,
-        }
-        body = {"body": await resp.content.read()}
-    return headers, body
-
-
-async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[bytes]]:
-    logger.debug("Calling localhost")
-
-    tries = 0
-    headers = None
-    body = None
-
-    timeout = aiohttp.ClientTimeout(total=5)
-    async with aiohttp.ClientSession(timeout=timeout) as session:
-        while not body:
-            try:
-                tries += 1
-                headers, body = await make_request(session, scope)
-            except aiohttp.ClientConnectorError:
-                if tries > 20:
-                    raise
-                await asyncio.sleep(0.05)
-
-    output = ""  # Process stdout is not captured per request
-    output_data = None
-    logger.debug("Returning result")
-    return headers, body, output, output_data
-
-
-async def process_instruction(
-    instruction: bytes,
-    interface: Interface,
-    application: Union[ASGIApplication, subprocess.Popen],
-) -> AsyncIterable[bytes]:
-
-    if instruction == b"halt":
-        logger.info("Received halt command")
-        system("sync")
-        logger.debug("Filesystems synced")
-        if isinstance(application, subprocess.Popen):
-            application.terminate()
-            logger.debug("Application terminated")
-            # application.communicate()
-        else:
-            # Close the cached session in aleph_client:
-            from aleph_client.asynchronous import get_fallback_session
-
-            session: aiohttp.ClientSession = get_fallback_session()
-            await session.close()
-            logger.debug("Aiohttp cached session closed")
-        yield b"STOP\n"
-        logger.debug("Supervisor informed of halt")
-        raise ShutdownException
-    elif instruction.startswith(b"!"):
-        # Execute shell commands in the form `!ls /`
-        msg = instruction[1:].decode()
-        try:
-            process_output = subprocess.check_output(
-                msg, stderr=subprocess.STDOUT, shell=True
-            )
-            yield process_output
-        except subprocess.CalledProcessError as error:
-            yield str(error).encode() + b"\n" + error.output
-    else:
-        # Python
-        logger.debug("msgpack.loads (")
-        msg_ = msgpack.loads(instruction, raw=False)
-        logger.debug("msgpack.loads )")
-        payload = RunCodePayload(**msg_)
-
-        output: Optional[str] = None
-        try:
-            headers: Dict
-            body: Dict
-            output_data: Optional[bytes]
-
-            if interface == Interface.asgi:
-                headers, body, output, output_data = await run_python_code_http(
-                    application=application, scope=payload.scope
-                )
-            elif interface == Interface.executable:
-                headers, body, output, output_data = await run_executable_http(
-                    scope=payload.scope
-                )
-            else:
-                raise ValueError("Unknown interface. This should never happen")
-
-            result = {
-                "headers": headers,
-                "body": body,
-                "output": output,
-                "output_data": output_data,
-            }
-            yield msgpack.dumps(result, use_bin_type=True)
-        except Exception as error:
-            yield msgpack.dumps(
-                {
-                    "error": str(error),
-                    "traceback": str(traceback.format_exc()),
-                    "output": output,
-                }
-            )
-
-
-def receive_data_length(client) -> int:
-    """Receive the length of the data to follow."""
-    buffer = b""
-    for _ in range(9):
-        byte = client.recv(1)
-        if byte == b"\n":
-            break
-        else:
-            buffer += byte
-    return int(buffer)
-
-
-def load_configuration(data: bytes) -> ConfigurationPayload:
-    msg_ = msgpack.loads(data, raw=False)
-    msg_["volumes"] = [Volume(**volume_dict) for volume_dict in msg_.get("volumes")]
-    return ConfigurationPayload(**msg_)
-
-
-def receive_config(client) -> ConfigurationPayload:
-    length = receive_data_length(client)
-    data = b""
-    while len(data) < length:
-        data += client.recv(1024 * 1024)
-    return load_configuration(data)
-
-
-def setup_system(config: ConfigurationPayload):
-    # Linux host names are limited to 63 characters. We therefore use the base32 representation
-    # of the item_hash instead of its common base16 representation.
-    item_hash_binary: bytes = base64.b16decode(config.vm_hash.encode().upper())
-    hostname = base64.b32encode(item_hash_binary).decode().strip('=').lower()
-    setup_hostname(hostname)
-
-    setup_variables(config.variables)
-    setup_volumes(config.volumes)
-    setup_network(config.ip, config.route, config.dns_servers)
-    setup_input_data(config.input_data)
-    logger.debug("Setup finished")
-
-
-def umount_volumes(volumes: List[Volume]):
-    "Umount user related filesystems"
-    system("sync")
-    for volume in volumes:
-        logger.debug(f"Umounting /dev/{volume.device} on {volume.mount}")
-        system(f"umount {volume.mount}")
-
-
-async def main():
-    client, addr = s.accept()
-
-    logger.debug("Receiving setup...")
-    config = receive_config(client)
-    setup_system(config)
-
-    try:
-        app: Union[ASGIApplication, subprocess.Popen] = setup_code(
-            config.code, config.encoding, config.entrypoint, config.interface
-        )
-        client.send(msgpack.dumps({"success": True}))
-    except Exception as error:
-        client.send(
-            msgpack.dumps(
-                {
-                    "success": False,
-                    "error": str(error),
-                    "traceback": str(traceback.format_exc()),
-                }
-            )
-        )
-        logger.exception("Program could not be started")
-        raise
-
-    class ServerReference:
-        "Reference used to close the server from within `handle_instruction"
-        server: asyncio.AbstractServer
-
-    server_reference = ServerReference()
-
-    async def handle_instruction(reader, writer):
-        data = await reader.read(1000_1000)  # Max 1 Mo
-
-        logger.debug("Init received msg")
-        if logger.level <= logging.DEBUG:
-            data_to_print = f"{data[:500]}..." if len(data) > 500 else data
-            logger.debug(f"<<<\n\n{data_to_print}\n\n>>>")
-
-        try:
-            async for result in process_instruction(
-                instruction=data, interface=config.interface, application=app
-            ):
-                writer.write(result)
-                await writer.drain()
-
-                logger.debug("Instruction processed")
-        except ShutdownException:
-            logger.info("Initiating shutdown")
-            writer.write(b"STOPZ\n")
-            await writer.drain()
-            logger.debug("Shutdown confirmed to supervisor")
-            server_reference.server.close()
-            logger.debug("Supervisor socket server closed")
-        finally:
-            writer.close()
-
-    server = await asyncio.start_server(handle_instruction, sock=s)
-    server_reference.server = server
-
-    addr = server.sockets[0].getsockname()
-    print(f"Serving on {addr}")
-
-    try:
-        async with server:
-            await server.serve_forever()
-    except asyncio.CancelledError:
-        logger.debug("Server was properly cancelled")
-    finally:
-        logger.warning("System shutdown")
-        server.close()
-        logger.debug("Server closed")
-        umount_volumes(config.volumes)
-        logger.debug("User volumes unmounted")
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.DEBUG)
-    asyncio.run(main())
-
-    logger.info("Unmounting system filesystems")
-    system("umount /dev/shm")
-    system("umount /dev/pts")
-    system("umount -a")
-
-    logger.info("Sending reboot syscall")
-    # Send reboot syscall, see man page
-    # https://man7.org/linux/man-pages/man2/reboot.2.html
-    libc = ctypes.CDLL(None)
-    libc.syscall(169, 0xFEE1DEAD, 672274793, 0x4321FEDC, None)
-    # The exit should not happen due to system halt.
-    sys.exit(0)
diff --git a/runtimes/aleph-alpine-3.13-python/update_inits.sh b/runtimes/aleph-alpine-3.13-python/update_inits.sh
deleted file mode 100755
index 0daa9b16a..000000000
--- a/runtimes/aleph-alpine-3.13-python/update_inits.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-
-umount /mnt/rootfs
-
-set -euf
-
-mount ./rootfs.ext4 /mnt/rootfs
-
-cp ./init0.sh /mnt/rootfs/sbin/init
-cp ./init1.py /mnt/rootfs/root/init1.py
-chmod +x /mnt/rootfs/sbin/init
-chmod +x /mnt/rootfs/root/init1.py
-
-umount /mnt/rootfs
-
-echo "OK"
diff --git a/runtimes/aleph-debian-11-python/init0.sh b/runtimes/aleph-debian-11-python/init0.sh
deleted file mode 120000
index 4315744b7..000000000
--- a/runtimes/aleph-debian-11-python/init0.sh
+++ /dev/null
@@ -1 +0,0 @@
-../aleph-alpine-3.13-python/init0.sh
\ No newline at end of file
diff --git a/runtimes/aleph-debian-11-python/init0.sh b/runtimes/aleph-debian-11-python/init0.sh
new file mode 100644
index 000000000..8eb1b62bf
--- /dev/null
+++ b/runtimes/aleph-debian-11-python/init0.sh
@@ -0,0 +1,56 @@
+#!/bin/sh
+
+set -euf
+
+mount -t proc proc /proc -o nosuid,noexec,nodev
+
+log() {
+    echo "$(cat /proc/uptime | awk '{printf $1}')" '|S' "$@"
+}
+log "init0.sh is launching"
+
+# Switch root from read-only ext4 to to read-write overlay
+mkdir -p /overlay
+/bin/mount -t tmpfs -o noatime,mode=0755 tmpfs /overlay
+mkdir -p /overlay/root /overlay/work
+/bin/mount -o noatime,lowerdir=/,upperdir=/overlay/root,workdir=/overlay/work -t overlay "overlayfs:/overlay/root" /mnt
+mkdir -p /mnt/rom
+pivot_root /mnt /mnt/rom
+
+mount --move /rom/proc /proc
+mount --move /rom/dev /dev
+
+mkdir -p /dev/pts
+mkdir -p /dev/shm
+
+mount -t sysfs sys /sys -o nosuid,noexec,nodev
+mount -t tmpfs run /run -o mode=0755,nosuid,nodev
+#mount -t devtmpfs dev /dev -o mode=0755,nosuid
+mount -t devpts devpts /dev/pts -o mode=0620,gid=5,nosuid,noexec
+mount -t tmpfs shm /dev/shm -omode=1777,nosuid,nodev
+
+# Required by Docker
+cgroupfs-mount
+update-alternatives --set iptables /usr/sbin/iptables-legacy
+update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+
+# Enable the following to force the storage driver used by Docker.
+# See https://docs.docker.com/storage/storagedriver/select-storage-driver/
+#echo '{\n"storage-driver": "overlay2"\n}\n' > /etc/docker/daemon.json
+
+# List block devices
+lsblk
+
+#cat /proc/sys/kernel/random/entropy_avail
+
+# TODO: Move in init1
+mkdir -p /run/sshd
+/usr/sbin/sshd &
+log "SSH UP"
+
+log "Setup socat"
+socat UNIX-LISTEN:/tmp/socat-socket,fork,reuseaddr VSOCK-CONNECT:2:53 &
+log "Socat ready"
+
+# Replace this script with the manager
+exec /root/init1.py
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
deleted file mode 120000
index 529895e71..000000000
--- a/runtimes/aleph-debian-11-python/init1.py
+++ /dev/null
@@ -1 +0,0 @@
-../aleph-alpine-3.13-python/init1.py
\ No newline at end of file
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
new file mode 100644
index 000000000..47f435f62
--- /dev/null
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -0,0 +1,566 @@
+#!/usr/bin/python3 -OO
+import base64
+import logging
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    format="%(relativeCreated)4f |V %(levelname)s | %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+logger.debug("Imports starting")
+
+import ctypes
+import asyncio
+import os
+import socket
+from enum import Enum
+import subprocess
+import sys
+import traceback
+from contextlib import redirect_stdout
+from dataclasses import dataclass, field
+from io import StringIO
+from os import system
+from shutil import make_archive
+from typing import Optional, Dict, Any, Tuple, List, NewType, Union, AsyncIterable
+
+import aiohttp
+import msgpack
+
+logger.debug("Imports finished")
+
+ASGIApplication = NewType("ASGIApplication", Any)
+
+
+class Encoding(str, Enum):
+    plain = "plain"
+    zip = "zip"
+    squashfs = "squashfs"
+
+
+class Interface(str, Enum):
+    asgi = "asgi"
+    executable = "executable"
+
+
+class ShutdownException(Exception):
+    pass
+
+
+@dataclass
+class Volume:
+    mount: str
+    device: str
+    read_only: bool
+
+
+@dataclass
+class ConfigurationPayload:
+    code: bytes
+    encoding: Encoding
+    entrypoint: str
+    input_data: bytes
+    interface: Interface
+    vm_hash: str
+    ip: Optional[str] = None
+    route: Optional[str] = None
+    dns_servers: List[str] = field(default_factory=list)
+    volumes: List[Volume] = field(default_factory=list)
+    variables: Optional[Dict[str, str]] = None
+
+
+@dataclass
+class RunCodePayload:
+    scope: Dict
+
+
+# Open a socket to receive instructions from the host
+s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
+s.bind((socket.VMADDR_CID_ANY, 52))
+s.listen()
+
+# Send the host that we are ready
+s0 = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
+s0.connect((2, 52))
+s0.close()
+
+# Configure aleph-client to use the guest API
+os.environ["ALEPH_API_HOST"] = "http://localhost"
+os.environ["ALEPH_API_UNIX_SOCKET"] = "/tmp/socat-socket"
+os.environ["ALEPH_REMOTE_CRYPTO_HOST"] = "http://localhost"
+os.environ["ALEPH_REMOTE_CRYPTO_UNIX_SOCKET"] = "/tmp/socat-socket"
+
+logger.debug("init1.py is launching")
+
+
+def setup_hostname(hostname: str):
+    os.environ["ALEPH_ADDRESS_TO_USE"] = hostname
+    system(f"hostname {hostname}")
+
+
+def setup_variables(variables: Optional[Dict[str, str]]):
+    if variables is None:
+        return
+    for key, value in variables.items():
+        os.environ[key] = value
+
+
+def setup_network(
+    ip: Optional[str], route: Optional[str], dns_servers: Optional[List[str]] = None
+):
+    """Setup the system with info from the host."""
+    dns_servers = dns_servers or []
+    if not os.path.exists("/sys/class/net/eth0"):
+        logger.info("No network interface eth0")
+        return
+
+    if not ip:
+        logger.info("No network IP")
+        return
+
+    logger.debug("Setting up networking")
+    system("ip addr add 127.0.0.1/8 dev lo brd + scope host")
+    system("ip addr add ::1/128 dev lo")
+    system("ip link set lo up")
+    if "/" in ip:
+        # Forward compatibility with future supervisors that pass the mask with the IP.
+        system(f"ip addr add {ip} dev eth0")
+    else:
+        logger.warning("Not passing the mask with the IP is deprecated and will be unsupported")
+        system(f"ip addr add {ip}/24 dev eth0")
+    system("ip link set eth0 up")
+
+    if route:
+        system(f"ip route add default via {route} dev eth0")
+        logger.debug(f"IP and route set: {ip} via {route}")
+    else:
+        logger.warning("IP set with no network route")
+
+    with open("/etc/resolv.conf", "wb") as resolvconf_fd:
+        for server in dns_servers:
+            resolvconf_fd.write(f"nameserver {server}\n".encode())
+
+
+def setup_input_data(input_data: bytes):
+    logger.debug("Extracting data")
+    if input_data:
+        # Unzip in /data
+        if not os.path.exists("/opt/input.zip"):
+            open("/opt/input.zip", "wb").write(input_data)
+            os.makedirs("/data", exist_ok=True)
+            os.system("unzip -q /opt/input.zip -d /data")
+
+
+def setup_volumes(volumes: List[Volume]):
+    for volume in volumes:
+        logger.debug(f"Mounting /dev/{volume.device} on {volume.mount}")
+        os.makedirs(volume.mount, exist_ok=True)
+        if volume.read_only:
+            system(f"mount -t squashfs -o ro /dev/{volume.device} {volume.mount}")
+        else:
+            system(f"mount -o rw /dev/{volume.device} {volume.mount}")
+
+    system("mount")
+
+
+def setup_code_asgi(
+    code: bytes, encoding: Encoding, entrypoint: str
+) -> ASGIApplication:
+    # Allow importing packages from /opt/packages
+    sys.path.append("/opt/packages")
+
+    logger.debug("Extracting code")
+    app: ASGIApplication
+    if encoding == Encoding.squashfs:
+        sys.path.append("/opt/code")
+        module_name, app_name = entrypoint.split(":", 1)
+        logger.debug("import module")
+        module = __import__(module_name)
+        for level in module_name.split(".")[1:]:
+            module = getattr(module, level)
+        app = getattr(module, app_name)
+    elif encoding == Encoding.zip:
+        # Unzip in /opt and import the entrypoint from there
+        if not os.path.exists("/opt/archive.zip"):
+            open("/opt/archive.zip", "wb").write(code)
+            logger.debug("Run unzip")
+            os.system("unzip -q /opt/archive.zip -d /opt")
+        sys.path.append("/opt")
+        module_name, app_name = entrypoint.split(":", 1)
+        logger.debug("import module")
+        module = __import__(module_name)
+        for level in module_name.split(".")[1:]:
+            module = getattr(module, level)
+        app = getattr(module, app_name)
+    elif encoding == Encoding.plain:
+        # Execute the code and extract the entrypoint
+        locals: Dict[str, Any] = {}
+        exec(code, globals(), locals)
+        app = locals[entrypoint]
+    else:
+        raise ValueError(f"Unknown encoding '{encoding}'")
+    return app
+
+
+def setup_code_executable(
+    code: bytes, encoding: Encoding, entrypoint: str
+) -> subprocess.Popen:
+    logger.debug("Extracting code")
+    if encoding == Encoding.squashfs:
+        path = f"/opt/code/{entrypoint}"
+        if not os.path.isfile(path):
+            os.system("find /opt/code/")
+            raise FileNotFoundError(f"No such file: {path}")
+        os.system(f"chmod +x {path}")
+    elif encoding == Encoding.zip:
+        open("/opt/archive.zip", "wb").write(code)
+        logger.debug("Run unzip")
+        os.makedirs("/opt/code", exist_ok=True)
+        os.system("unzip /opt/archive.zip -d /opt/code")
+        path = f"/opt/code/{entrypoint}"
+        if not os.path.isfile(path):
+            os.system("find /opt/code")
+            raise FileNotFoundError(f"No such file: {path}")
+        os.system(f"chmod +x {path}")
+    elif encoding == Encoding.plain:
+        os.makedirs("/opt/code", exist_ok=True)
+        path = f"/opt/code/executable {entrypoint}"
+        open(path, "wb").write(code)
+        os.system(f"chmod +x {path}")
+    else:
+        raise ValueError(f"Unknown encoding '{encoding}'. This should never happen.")
+
+    process = subprocess.Popen(path)
+    return process
+
+
+def setup_code(
+    code: bytes, encoding: Encoding, entrypoint: str, interface: Interface
+) -> Union[ASGIApplication, subprocess.Popen]:
+
+    if interface == Interface.asgi:
+        return setup_code_asgi(code=code, encoding=encoding, entrypoint=entrypoint)
+    elif interface == Interface.executable:
+        return setup_code_executable(
+            code=code, encoding=encoding, entrypoint=entrypoint
+        )
+    else:
+        raise ValueError("Invalid interface. This should never happen.")
+
+
+async def run_python_code_http(
+    application: ASGIApplication, scope: dict
+) -> Tuple[Dict, Dict, str, Optional[bytes]]:
+
+    logger.debug("Running code")
+    with StringIO() as buf, redirect_stdout(buf):
+        # Execute in the same process, saves ~20ms than a subprocess
+
+        # The body should not be part of the ASGI scope itself
+        body: bytes = scope.pop("body")
+
+        async def receive():
+            type_ = (
+                "http.request"
+                if scope["type"] in ("http", "websocket")
+                else "aleph.message"
+            )
+            return {"type": type_, "body": body, "more_body": False}
+
+        send_queue: asyncio.Queue = asyncio.Queue()
+
+        async def send(dico):
+            await send_queue.put(dico)
+
+        # TODO: Better error handling
+        logger.debug("Awaiting application...")
+        await application(scope, receive, send)
+
+        logger.debug("Waiting for headers")
+        headers: Dict
+        if scope["type"] == "http":
+            headers = await send_queue.get()
+        else:
+            headers = {}
+
+        logger.debug("Waiting for body")
+        body: Dict = await send_queue.get()
+
+        logger.debug("Waiting for buffer")
+        output = buf.getvalue()
+
+        logger.debug(f"Headers {headers}")
+        logger.debug(f"Body {body}")
+        logger.debug(f"Output {output}")
+
+    logger.debug("Getting output data")
+    output_data: bytes
+    if os.path.isdir("/data") and os.listdir("/data"):
+        make_archive("/opt/output", "zip", "/data")
+        with open("/opt/output.zip", "rb") as output_zipfile:
+            output_data = output_zipfile.read()
+    else:
+        output_data = b""
+
+    logger.debug("Returning result")
+    return headers, body, output, output_data
+
+
+async def make_request(session, scope):
+    async with session.request(
+        scope["method"],
+        url="http://localhost:8080{}".format(scope["path"]),
+        params=scope["query_string"],
+        headers=[(a.decode("utf-8"), b.decode("utf-8")) for a, b in scope["headers"]],
+        data=scope.get("body", None),
+    ) as resp:
+        headers = {
+            "headers": [
+                (a.encode("utf-8"), b.encode("utf-8")) for a, b in resp.headers.items()
+            ],
+            "status": resp.status,
+        }
+        body = {"body": await resp.content.read()}
+    return headers, body
+
+
+async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[bytes]]:
+    logger.debug("Calling localhost")
+
+    tries = 0
+    headers = None
+    body = None
+
+    timeout = aiohttp.ClientTimeout(total=5)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        while not body:
+            try:
+                tries += 1
+                headers, body = await make_request(session, scope)
+            except aiohttp.ClientConnectorError:
+                if tries > 20:
+                    raise
+                await asyncio.sleep(0.05)
+
+    output = ""  # Process stdout is not captured per request
+    output_data = None
+    logger.debug("Returning result")
+    return headers, body, output, output_data
+
+
+async def process_instruction(
+    instruction: bytes,
+    interface: Interface,
+    application: Union[ASGIApplication, subprocess.Popen],
+) -> AsyncIterable[bytes]:
+
+    if instruction == b"halt":
+        logger.info("Received halt command")
+        system("sync")
+        logger.debug("Filesystems synced")
+        if isinstance(application, subprocess.Popen):
+            application.terminate()
+            logger.debug("Application terminated")
+            # application.communicate()
+        else:
+            # Close the cached session in aleph_client:
+            from aleph_client.asynchronous import get_fallback_session
+
+            session: aiohttp.ClientSession = get_fallback_session()
+            await session.close()
+            logger.debug("Aiohttp cached session closed")
+        yield b"STOP\n"
+        logger.debug("Supervisor informed of halt")
+        raise ShutdownException
+    elif instruction.startswith(b"!"):
+        # Execute shell commands in the form `!ls /`
+        msg = instruction[1:].decode()
+        try:
+            process_output = subprocess.check_output(
+                msg, stderr=subprocess.STDOUT, shell=True
+            )
+            yield process_output
+        except subprocess.CalledProcessError as error:
+            yield str(error).encode() + b"\n" + error.output
+    else:
+        # Python
+        logger.debug("msgpack.loads (")
+        msg_ = msgpack.loads(instruction, raw=False)
+        logger.debug("msgpack.loads )")
+        payload = RunCodePayload(**msg_)
+
+        output: Optional[str] = None
+        try:
+            headers: Dict
+            body: Dict
+            output_data: Optional[bytes]
+
+            if interface == Interface.asgi:
+                headers, body, output, output_data = await run_python_code_http(
+                    application=application, scope=payload.scope
+                )
+            elif interface == Interface.executable:
+                headers, body, output, output_data = await run_executable_http(
+                    scope=payload.scope
+                )
+            else:
+                raise ValueError("Unknown interface. This should never happen")
+
+            result = {
+                "headers": headers,
+                "body": body,
+                "output": output,
+                "output_data": output_data,
+            }
+            yield msgpack.dumps(result, use_bin_type=True)
+        except Exception as error:
+            yield msgpack.dumps(
+                {
+                    "error": str(error),
+                    "traceback": str(traceback.format_exc()),
+                    "output": output,
+                }
+            )
+
+
+def receive_data_length(client) -> int:
+    """Receive the length of the data to follow."""
+    buffer = b""
+    for _ in range(9):
+        byte = client.recv(1)
+        if byte == b"\n":
+            break
+        else:
+            buffer += byte
+    return int(buffer)
+
+
+def load_configuration(data: bytes) -> ConfigurationPayload:
+    msg_ = msgpack.loads(data, raw=False)
+    msg_["volumes"] = [Volume(**volume_dict) for volume_dict in msg_.get("volumes")]
+    return ConfigurationPayload(**msg_)
+
+
+def receive_config(client) -> ConfigurationPayload:
+    length = receive_data_length(client)
+    data = b""
+    while len(data) < length:
+        data += client.recv(1024 * 1024)
+    return load_configuration(data)
+
+
+def setup_system(config: ConfigurationPayload):
+    # Linux host names are limited to 63 characters. We therefore use the base32 representation
+    # of the item_hash instead of its common base16 representation.
+    item_hash_binary: bytes = base64.b16decode(config.vm_hash.encode().upper())
+    hostname = base64.b32encode(item_hash_binary).decode().strip('=').lower()
+    setup_hostname(hostname)
+
+    setup_variables(config.variables)
+    setup_volumes(config.volumes)
+    setup_network(config.ip, config.route, config.dns_servers)
+    setup_input_data(config.input_data)
+    logger.debug("Setup finished")
+
+
+def umount_volumes(volumes: List[Volume]):
+    "Umount user related filesystems"
+    system("sync")
+    for volume in volumes:
+        logger.debug(f"Umounting /dev/{volume.device} on {volume.mount}")
+        system(f"umount {volume.mount}")
+
+
+async def main():
+    client, addr = s.accept()
+
+    logger.debug("Receiving setup...")
+    config = receive_config(client)
+    setup_system(config)
+
+    try:
+        app: Union[ASGIApplication, subprocess.Popen] = setup_code(
+            config.code, config.encoding, config.entrypoint, config.interface
+        )
+        client.send(msgpack.dumps({"success": True}))
+    except Exception as error:
+        client.send(
+            msgpack.dumps(
+                {
+                    "success": False,
+                    "error": str(error),
+                    "traceback": str(traceback.format_exc()),
+                }
+            )
+        )
+        logger.exception("Program could not be started")
+        raise
+
+    class ServerReference:
+        "Reference used to close the server from within `handle_instruction"
+        server: asyncio.AbstractServer
+
+    server_reference = ServerReference()
+
+    async def handle_instruction(reader, writer):
+        data = await reader.read(1000_1000)  # Max 1 Mo
+
+        logger.debug("Init received msg")
+        if logger.level <= logging.DEBUG:
+            data_to_print = f"{data[:500]}..." if len(data) > 500 else data
+            logger.debug(f"<<<\n\n{data_to_print}\n\n>>>")
+
+        try:
+            async for result in process_instruction(
+                instruction=data, interface=config.interface, application=app
+            ):
+                writer.write(result)
+                await writer.drain()
+
+                logger.debug("Instruction processed")
+        except ShutdownException:
+            logger.info("Initiating shutdown")
+            writer.write(b"STOPZ\n")
+            await writer.drain()
+            logger.debug("Shutdown confirmed to supervisor")
+            server_reference.server.close()
+            logger.debug("Supervisor socket server closed")
+        finally:
+            writer.close()
+
+    server = await asyncio.start_server(handle_instruction, sock=s)
+    server_reference.server = server
+
+    addr = server.sockets[0].getsockname()
+    print(f"Serving on {addr}")
+
+    try:
+        async with server:
+            await server.serve_forever()
+    except asyncio.CancelledError:
+        logger.debug("Server was properly cancelled")
+    finally:
+        logger.warning("System shutdown")
+        server.close()
+        logger.debug("Server closed")
+        umount_volumes(config.volumes)
+        logger.debug("User volumes unmounted")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
+    asyncio.run(main())
+
+    logger.info("Unmounting system filesystems")
+    system("umount /dev/shm")
+    system("umount /dev/pts")
+    system("umount -a")
+
+    logger.info("Sending reboot syscall")
+    # Send reboot syscall, see man page
+    # https://man7.org/linux/man-pages/man2/reboot.2.html
+    libc = ctypes.CDLL(None)
+    libc.syscall(169, 0xFEE1DEAD, 672274793, 0x4321FEDC, None)
+    # The exit should not happen due to system halt.
+    sys.exit(0)

From 40c5c5bdd406a8a5945799f5d287269b6fbda3a1 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Tue, 6 Jun 2023 18:04:43 +0200
Subject: [PATCH 390/990] Fix: Instances could not run

This adds modifications to enable instance messages and updates of instances based on rootfs updates.

Instances have their root filesystem writable by using device mappers between an immutable base and an overlay on the block level.

Co-authored-by: Andres D. Molins 
Co-authored-by: Hugo Herter 
---
 .../workflows/test-on-droplet-debian-11.yml   |   9 +
 .../test-on-droplet-ubuntu-22.04.yml          |   9 +
 examples/instance_message_from_aleph.json     |  68 +++
 firecracker/config.py                         |   8 +-
 firecracker/microvm.py                        |  67 +-
 runtimes/aleph-debian-11-python/init1.py      |   8 +-
 .../aleph-debian-11-python/update_inits.sh    |   0
 runtimes/instance-debian-rootfs/Dockerfile    |  59 ++
 .../create-debian-disk.sh                     |  33 +
 runtimes/instance-debian-rootfs/init0.sh      |  37 ++
 runtimes/instance-debian-rootfs/init1.py      | 570 ++++++++++++++++++
 runtimes/instance-debian-rootfs/inittab       |  22 +
 .../nginx/health-check.conf                   |   9 +
 .../instance-debian-rootfs/nginx/index.html   |  20 +
 .../instance-debian-rootfs/nginx/nginx.conf   |  31 +
 .../instance-debian-rootfs/update_inits.sh    |  14 +
 vm_supervisor/INSTANCES.md                    |  44 ++
 vm_supervisor/messages.py                     |  29 +-
 vm_supervisor/models.py                       |  50 +-
 vm_supervisor/pool.py                         |  15 +-
 vm_supervisor/reactor.py                      |   6 +-
 vm_supervisor/resources.py                    |   1 +
 vm_supervisor/run.py                          |   6 +-
 vm_supervisor/storage.py                      | 208 +++++--
 vm_supervisor/tasks.py                        |  10 +-
 vm_supervisor/views.py                        |  13 +
 vm_supervisor/vm/firecracker_microvm.py       |  85 +--
 27 files changed, 1282 insertions(+), 149 deletions(-)
 create mode 100644 examples/instance_message_from_aleph.json
 mode change 100644 => 100755 runtimes/aleph-debian-11-python/update_inits.sh
 create mode 100644 runtimes/instance-debian-rootfs/Dockerfile
 create mode 100755 runtimes/instance-debian-rootfs/create-debian-disk.sh
 create mode 100644 runtimes/instance-debian-rootfs/init0.sh
 create mode 100644 runtimes/instance-debian-rootfs/init1.py
 create mode 100644 runtimes/instance-debian-rootfs/inittab
 create mode 100644 runtimes/instance-debian-rootfs/nginx/health-check.conf
 create mode 100644 runtimes/instance-debian-rootfs/nginx/index.html
 create mode 100644 runtimes/instance-debian-rootfs/nginx/nginx.conf
 create mode 100755 runtimes/instance-debian-rootfs/update_inits.sh
 create mode 100644 vm_supervisor/INSTANCES.md

diff --git a/.github/workflows/test-on-droplet-debian-11.yml b/.github/workflows/test-on-droplet-debian-11.yml
index 9fc7f8240..e7fc4b970 100644
--- a/.github/workflows/test-on-droplet-debian-11.yml
+++ b/.github/workflows/test-on-droplet-debian-11.yml
@@ -63,6 +63,7 @@ jobs:
           scp packaging/target/aleph-vm.debian-11.deb root@${DROPLET_IPV4}:/opt
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.debian-11.deb"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
+          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
 
       - name: Test Aleph-VM on the Droplet
@@ -73,6 +74,14 @@ jobs:
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
 
+      - name: Schedule an instance on the Droplet by faking a call from the scheduler
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          curl --retry 5 --max-time 10 --fail -X POST -H "Content-Type: application/json" \
+            -H "X-Auth-Signature: test" \
+            -d '{"persistent_vms": [], "instances": ["INSTANCE-HASH-TODO-FIXME"]}' \
+            "http://${DROPLET_IPV4}:4020/control/allocations"
+
       - name: Cleanup
         if: always()
         run: |
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index e6a0ce835..cc9633d57 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -68,6 +68,7 @@ jobs:
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.ubuntu-22.04.deb"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_DNS_RESOLUTION=resolvectl >> /etc/aleph-vm/supervisor.env"
+          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
 
       - name: Test Aleph-VM on the Droplet
@@ -78,6 +79,14 @@ jobs:
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
 
+      - name: Schedule an instance on the Droplet by faking a call from the scheduler
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-ubuntu-22-04 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          curl --retry 5 --max-time 10 --fail -X POST -H "Content-Type: application/json" \
+            -H "X-Auth-Signature: test" \
+            -d '{"persistent_vms": [], "instances": ["INSTANCE-HASH-TODO-FIXME"]}' \
+            "http://${DROPLET_IPV4}:4020/control/allocations"
+
       - name: Cleanup
         if: always()
         run: |
diff --git a/examples/instance_message_from_aleph.json b/examples/instance_message_from_aleph.json
new file mode 100644
index 000000000..8302d68c7
--- /dev/null
+++ b/examples/instance_message_from_aleph.json
@@ -0,0 +1,68 @@
+{
+    "chain": "ETH",
+    "item_hash": "fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-hash",
+    "sender": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba",
+    "type": "INSTANCE",
+    "channel": "Fun-dApps",
+    "confirmed": true,
+    "content": {
+        "address": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba",
+        "allow_amend": false,
+        "variables": {
+            "VM_CUSTOM_NUMBER": "32"
+        },
+        "environment": {
+            "reproducible": true,
+            "internet": true,
+            "aleph_api": true,
+            "shared_cache": true
+        },
+        "resources": {
+            "vcpus": 1,
+            "memory": 128,
+            "seconds": 30
+        },
+        "rootfs": {
+            "parent": {
+                "ref": "549ec451d9b099cad112d4aaa2c00ac40fb6729a92ff252ff22eef0b5c3cb613",
+                "use_latest": true
+            },
+            "persistence": "host",
+            "size_mib": 20000
+        },
+        "cloud_config": {
+            "password": "password",
+            "chpasswd": {
+                "expire": "False"
+            }
+        },
+        "volumes": [
+            {
+                "mount": "/opt/venv",
+                "ref": "5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51",
+                "use_latest": false
+            },
+            {
+                "comment": "Working data persisted on the VM supervisor, not available on other nodes",
+                "mount": "/var/lib/example",
+                "name": "data",
+                "persistence": "host",
+                "size_mib": 5
+            }
+        ],
+        "replaces": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba",
+        "time": 1619017773.8950517
+    },
+    "item_content": "{\"address\":\"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\",\"allow_amend\":false,\"variables\":{\"VM_CUSTOM_NUMBER\":\"32\"},\"environment\":{\"reproducible\":true,\"internet\":true,\"aleph_api\":true,\"shared_cache\":true},\"resources\":{\"vcpus\":1,\"memory\":128,\"seconds\":30},\"rootfs\":{\"parent\":{\"ref\":\"549ec451d9b099cad112d4aaa2c00ac40fb6729a92ff252ff22eef0b5c3cb613\",\"use_latest\":true},\"persistence\":\"host\",\"size_mib\":20000},\"cloud_config\":{\"password\":\"password\",\"chpasswd\":{\"expire\":\"False\"}},\"volumes\":[{\"mount\":\"/opt/venv\",\"ref\":\"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\",\"use_latest\":false},{\"comment\":\"Working data persisted on the VM supervisor, not available on other nodes\",\"mount\":\"/var/lib/example\",\"name\":\"data\",\"persistence\":\"host\",\"size_mib\":5}],\"replaces\":\"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\",\"time\":1619017773.8950517}",
+    "item_type": "inline",
+    "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b",
+    "size": 749,
+    "time": 1619017773.8950577,
+    "confirmations": [
+        {
+            "chain": "ETH",
+            "height": 12284734,
+            "hash": "0x67f2f3cde5e94e70615c92629c70d22dc959a118f46e9411b29659c2fce87cdc"
+        }
+    ]
+}
diff --git a/firecracker/config.py b/firecracker/config.py
index 94154595f..6db70cb71 100644
--- a/firecracker/config.py
+++ b/firecracker/config.py
@@ -14,8 +14,12 @@ class BootSource(BaseModel):
     )
 
     @staticmethod
-    def args(enable_console: bool = True):
-        default = "reboot=k panic=1 pci=off ro noapic nomodules random.trust_cpu=on"
+    def args(enable_console: bool = True, writable: bool = False):
+        default = "reboot=k panic=1 pci=off noapic nomodules random.trust_cpu=on"
+        if writable:
+            default = default + " rw"
+        else:
+            default = default + " ro"
         if enable_console:
             return "console=ttyS0 " + default
         else:
diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 9508ec1a1..b6001dafc 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -2,7 +2,9 @@
 import json
 import logging
 import os.path
+import shutil
 import string
+import subprocess
 from asyncio import Task
 from asyncio.base_events import Server
 from os import getuid
@@ -18,6 +20,7 @@
 
 VSOCK_PATH = "/tmp/v.sock"
 JAILER_BASE_DIRECTORY = "/var/lib/aleph/vm/jailer"
+DEVICE_BASE_DIRECTORY = "/dev/mapper"
 
 
 class MicroVMFailedInit(Exception):
@@ -29,7 +32,6 @@ class JSONBytesEncoder(json.JSONEncoder):
 
     # overload method default
     def default(self, obj):
-
         # Match all the types you want to handle in your converter
         if isinstance(obj, bytes):
             return obj.decode()
@@ -61,14 +63,15 @@ async def setfacl():
 class MicroVM:
     vm_id: int
     use_jailer: bool
-    firecracker_bin_path: str
-    jailer_bin_path: Optional[str]
+    firecracker_bin_path: Path
+    jailer_bin_path: Optional[Path]
     proc: Optional[asyncio.subprocess.Process] = None
     stdout_task: Optional[Task] = None
     stderr_task: Optional[Task] = None
     config_file_path: Optional[Path] = None
     drives: List[Drive]
     init_timeout: float
+    mounted_rootfs: Optional[Path] = None
 
     _unix_socket: Server
 
@@ -98,9 +101,9 @@ def vsock_path(self):
     def __init__(
         self,
         vm_id: int,
-        firecracker_bin_path: str,
+        firecracker_bin_path: Path,
         use_jailer: bool = True,
-        jailer_bin_path: Optional[str] = None,
+        jailer_bin_path: Optional[Path] = None,
         init_timeout: float = 5.0,
     ):
         self.vm_id = vm_id
@@ -132,6 +135,7 @@ def prepare_jailer(self):
         system(f"chown jailman:jailman {self.jailer_path}/tmp/")
         #
         system(f"mkdir -p {self.jailer_path}/opt")
+        system(f"mkdir -p {self.jailer_path}/dev/mapper")
 
         # system(f"cp disks/rootfs.ext4 {self.jailer_path}/opt")
         # system(f"cp hello-vmlinux.bin {self.jailer_path}/opt")
@@ -241,19 +245,25 @@ async def start_jailed_firecracker(
         )
         return self.proc
 
-    def enable_kernel(self, kernel_image_path: str) -> str:
+    def enable_kernel(self, kernel_image_path: Path) -> Path:
         """Make a kernel available to the VM.
 
         Creates a symlink to the kernel file if jailer is in use.
         """
         if self.use_jailer:
-            kernel_filename = Path(kernel_image_path).name
+            kernel_filename = kernel_image_path.name
             jailer_kernel_image_path = f"/opt/{kernel_filename}"
             os.link(kernel_image_path, f"{self.jailer_path}{jailer_kernel_image_path}")
             kernel_image_path = jailer_kernel_image_path
         return kernel_image_path
 
-    def enable_rootfs(self, path_on_host: str) -> str:
+    def enable_rootfs(self, path_on_host: Path) -> Path:
+        if path_on_host.is_file():
+            return self.enable_file_rootfs(path_on_host)
+        elif path_on_host.is_block_device():
+            return self.enable_device_mapper_rootfs(path_on_host)
+
+    def enable_file_rootfs(self, path_on_host: Path) -> Path:
         """Make a rootfs available to the VM.
 
         Creates a symlink to the rootfs file if jailer is in use.
@@ -262,15 +272,38 @@ def enable_rootfs(self, path_on_host: str) -> str:
             rootfs_filename = Path(path_on_host).name
             jailer_path_on_host = f"/opt/{rootfs_filename}"
             os.link(path_on_host, f"{self.jailer_path}/{jailer_path_on_host}")
-            return jailer_path_on_host
+            return Path(jailer_path_on_host)
         else:
             return path_on_host
 
+    def enable_device_mapper_rootfs(self, path_on_host: Path) -> Path:
+        """Mount a rootfs to the VM.
+        """
+        self.mounted_rootfs = path_on_host
+        if not self.use_jailer:
+            return path_on_host
+
+        rootfs_filename = path_on_host.name
+        device_jailer_path = Path(DEVICE_BASE_DIRECTORY) / rootfs_filename
+        final_path = Path(self.jailer_path) / str(device_jailer_path).strip("/")
+        if not final_path.is_block_device():
+            jailer_device_vm_path = Path(f"{self.jailer_path}/{DEVICE_BASE_DIRECTORY}")
+            jailer_device_vm_path.mkdir(exist_ok=True, parents=True)
+            rootfs_device = path_on_host.resolve()
+            # Copy the /dev/dm-{device_id} special block file that is the real mapping destination on Jailer
+            os.system(f"cp -vap {rootfs_device} {self.jailer_path}/dev/")
+            path_to_mount = jailer_device_vm_path / rootfs_filename
+            if not path_to_mount.is_symlink():
+                path_to_mount.symlink_to(rootfs_device)
+            os.system(f"chown -Rh jailman:jailman {self.jailer_path}/dev")
+
+        return device_jailer_path
+
     @staticmethod
     def compute_device_name(index: int) -> str:
         return f"vd{string.ascii_lowercase[index + 1]}"
 
-    def enable_drive(self, drive_path: str, read_only: bool = True) -> Drive:
+    def enable_drive(self, drive_path: Path, read_only: bool = True) -> Drive:
         """Make a volume available to the VM.
 
         Creates a symlink to the volume file if jailer is in use.
@@ -278,14 +311,14 @@ def enable_drive(self, drive_path: str, read_only: bool = True) -> Drive:
         index = len(self.drives)
         device_name = self.compute_device_name(index)
         if self.use_jailer:
-            drive_filename = Path(drive_path).name
+            drive_filename = drive_path.name
             jailer_path_on_host = f"/opt/{drive_filename}"
             os.link(drive_path, f"{self.jailer_path}/{jailer_path_on_host}")
             drive_path = jailer_path_on_host
 
         drive = Drive(
             drive_id=device_name,
-            path_on_host=Path(drive_path),
+            path_on_host=drive_path,
             is_root_device=False,
             is_read_only=read_only,
         )
@@ -391,7 +424,7 @@ async def teardown(self):
             await asyncio.wait_for(self.shutdown(), timeout=5)
         except asyncio.TimeoutError:
             logger.exception(f"Timeout during VM shutdown vm={self.vm_id}")
-        logger.debug("Waiting for one second for the process to shudown")
+        logger.debug("Waiting for one second for the process to shutdown")
         await asyncio.sleep(1)
         await self.stop()
 
@@ -400,6 +433,14 @@ async def teardown(self):
         if self.stderr_task:
             self.stderr_task.cancel()
 
+        if self.mounted_rootfs:
+            logger.debug("Waiting for one second for the VM to shutdown")
+            await asyncio.sleep(1)
+            root_fs = self.mounted_rootfs.name
+            os.system(f"dmsetup remove {root_fs}")
+            if self.use_jailer:
+                shutil.rmtree(self.jailer_path)
+
         if self._unix_socket:
             logger.debug("Closing unix socket")
             self._unix_socket.close()
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index 47f435f62..ed986b449 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -57,12 +57,12 @@ class Volume:
 
 @dataclass
 class ConfigurationPayload:
-    code: bytes
-    encoding: Encoding
-    entrypoint: str
     input_data: bytes
     interface: Interface
     vm_hash: str
+    code: bytes = None
+    encoding: Encoding = None
+    entrypoint: str = None
     ip: Optional[str] = None
     route: Optional[str] = None
     dns_servers: List[str] = field(default_factory=list)
@@ -236,7 +236,7 @@ def setup_code_executable(
 
 
 def setup_code(
-    code: bytes, encoding: Encoding, entrypoint: str, interface: Interface
+    code: Optional[bytes], encoding: Optional[Encoding], entrypoint: Optional[str], interface: Interface
 ) -> Union[ASGIApplication, subprocess.Popen]:
 
     if interface == Interface.asgi:
diff --git a/runtimes/aleph-debian-11-python/update_inits.sh b/runtimes/aleph-debian-11-python/update_inits.sh
old mode 100644
new mode 100755
diff --git a/runtimes/instance-debian-rootfs/Dockerfile b/runtimes/instance-debian-rootfs/Dockerfile
new file mode 100644
index 000000000..29f9b5a85
--- /dev/null
+++ b/runtimes/instance-debian-rootfs/Dockerfile
@@ -0,0 +1,59 @@
+# Pull the minimal Debian image
+FROM debian
+
+# Install Nginx
+RUN apt-get -y update && apt-get -y install nginx
+
+COPY nginx/index.html /usr/share/nginx/html/index.html
+COPY nginx/health-check.conf /etc/nginx/conf.d/health-check.conf
+
+# Install all basic dependencies
+RUN apt-get install -y --no-install-recommends --no-install-suggests \
+  python3-minimal \
+  openssh-server \
+  socat libsecp256k1-0
+
+# Install all needed python modules
+RUN apt-get install -y --no-install-recommends --no-install-suggests \
+  python3-aiohttp python3-msgpack \
+  python3-setuptools python3-dev \
+  python3-pip python3-cytoolz python3-pydantic
+
+# Install NodeJS and some tools
+RUN apt-get install -y --no-install-recommends --no-install-suggests \
+  iproute2 unzip nodejs npm build-essential iputils-ping curl
+
+# Install Docker
+RUN apt-get install -y --no-install-recommends --no-install-suggests \
+  docker.io cgroupfs-mount nftables
+
+# Install Aleph dependencies
+RUN pip3 install 'fastapi~=0.71.0'
+RUN pip3 install 'aleph-client>=0.4.6' 'coincurve==15.0.0'
+
+# Compile all Python bytecode
+RUN python3 -m compileall -f /usr/local/lib/python3.9
+
+# Enable root login by ssh
+RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
+
+# Generate SSH host keys
+#RUN systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
+#RUN systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key
+#RUN systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key
+#RUN systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ed25519 -f /etc/ssh/ssh_host_ed25519_key
+
+# Set up a login terminal on the serial console (ttyS0):
+RUN ln -s agetty /etc/init.d/agetty.ttyS0
+RUN echo ttyS0 > /etc/securetty
+
+# Set root password
+RUN echo "root:toor" | /usr/sbin/chpasswd
+
+# Reduce size
+RUN rm -fr /root/.cache
+RUN rm -fr /var/cache
+RUN mkdir -p /var/cache/apt/archives/partial
+RUN rm -fr /usr/share/doc
+RUN rm -fr /usr/share/man
+RUN rm -fr /var/lib/apt/lists/
diff --git a/runtimes/instance-debian-rootfs/create-debian-disk.sh b/runtimes/instance-debian-rootfs/create-debian-disk.sh
new file mode 100755
index 000000000..42202047c
--- /dev/null
+++ b/runtimes/instance-debian-rootfs/create-debian-disk.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+umount /mnt/vm
+rm ./rootfs.ext4
+mkdir -p /mnt/vm
+
+set -euf
+
+echo "Creating rootfs.ext4 file"
+# Create a 1,5 GB partition
+dd if=/dev/zero of=rootfs.ext4 bs=1MB count=1500
+mkfs.ext4 rootfs.ext4
+mount rootfs.ext4 /mnt/vm
+
+echo "Building Docker image"
+rm -rf ./docker-image
+docker buildx build -t docker-image --output type=local,dest=./docker-image .
+
+echo "Adding customizations"
+# Add custom inittab
+cp -vap ./inittab ./docker-image/etc/inittab
+# Copying init scripts
+cp ./init0.sh ./docker-image/sbin/init
+cp ./init1.py ./docker-image/root/init1.py
+chmod +x ./docker-image/sbin/init
+chmod +x ./docker-image/root/init1.py
+
+echo "Copying Docker image content to final rootfs file"
+cp -vap ./docker-image/. /mnt/vm
+umount /mnt/vm
+
+echo "Cleaning Docker generated files"
+rm -rf ./docker-image
diff --git a/runtimes/instance-debian-rootfs/init0.sh b/runtimes/instance-debian-rootfs/init0.sh
new file mode 100644
index 000000000..8890d0500
--- /dev/null
+++ b/runtimes/instance-debian-rootfs/init0.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+set -euf
+
+mount -t proc proc /proc -o nosuid,noexec,nodev
+
+log() {
+    echo "$(cat /proc/uptime | awk '{printf $1}')" '|S' "$@"
+}
+log "init0.sh is launching"
+
+mkdir -p /dev/pts
+mkdir -p /dev/shm
+
+mount -t sysfs sys /sys -o nosuid,noexec,nodev
+mount -t tmpfs run /run -o mode=0755,nosuid,nodev
+#mount -t devtmpfs dev /dev -o mode=0755,nosuid
+mount -t devpts devpts /dev/pts -o mode=0620,gid=5,nosuid,noexec
+mount -t tmpfs shm /dev/shm -omode=1777,nosuid,nodev
+
+# List block devices
+lsblk
+
+#cat /proc/sys/kernel/random/entropy_avail
+
+# TODO: Move in init1
+mkdir -p /run/sshd
+/usr/sbin/sshd &
+service nginx start
+log "SSH UP"
+
+log "Setup socat"
+socat UNIX-LISTEN:/tmp/socat-socket,fork,reuseaddr VSOCK-CONNECT:2:53 &
+log "Socat ready"
+
+# Replace this script with the manager
+exec /root/init1.py
diff --git a/runtimes/instance-debian-rootfs/init1.py b/runtimes/instance-debian-rootfs/init1.py
new file mode 100644
index 000000000..99b4a69fa
--- /dev/null
+++ b/runtimes/instance-debian-rootfs/init1.py
@@ -0,0 +1,570 @@
+#!/usr/bin/python3 -OO
+import base64
+import logging
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    format="%(relativeCreated)4f |V %(levelname)s | %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+logger.debug("Imports starting")
+
+import ctypes
+import asyncio
+import os
+import socket
+from enum import Enum
+import subprocess
+import sys
+import traceback
+from contextlib import redirect_stdout
+from dataclasses import dataclass, field
+from io import StringIO
+from os import system
+from shutil import make_archive
+from typing import Optional, Dict, Any, Tuple, List, NewType, Union, AsyncIterable
+
+import aiohttp
+import msgpack
+
+logger.debug("Imports finished")
+
+ASGIApplication = NewType("ASGIApplication", Any)
+
+
+class Encoding(str, Enum):
+    plain = "plain"
+    zip = "zip"
+    squashfs = "squashfs"
+
+
+class Interface(str, Enum):
+    asgi = "asgi"
+    executable = "executable"
+
+
+class ShutdownException(Exception):
+    pass
+
+
+@dataclass
+class Volume:
+    mount: str
+    device: str
+    read_only: bool
+
+
+@dataclass
+class ConfigurationPayload:
+    input_data: bytes
+    interface: Interface
+    vm_hash: str
+    code: Optional[bytes] = None
+    encoding: Optional[Encoding] = None
+    entrypoint: Optional[str] = None
+    ip: Optional[str] = None
+    route: Optional[str] = None
+    dns_servers: List[str] = field(default_factory=list)
+    volumes: List[Volume] = field(default_factory=list)
+    variables: Optional[Dict[str, str]] = None
+
+
+@dataclass
+class RunCodePayload:
+    scope: Dict
+
+
+# Open a socket to receive instructions from the host
+s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
+s.bind((socket.VMADDR_CID_ANY, 52))
+s.listen()
+
+# Send the host that we are ready
+s0 = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
+s0.connect((2, 52))
+s0.close()
+
+# Configure aleph-client to use the guest API
+os.environ["ALEPH_API_HOST"] = "http://localhost"
+os.environ["ALEPH_API_UNIX_SOCKET"] = "/tmp/socat-socket"
+os.environ["ALEPH_REMOTE_CRYPTO_HOST"] = "http://localhost"
+os.environ["ALEPH_REMOTE_CRYPTO_UNIX_SOCKET"] = "/tmp/socat-socket"
+
+logger.debug("init1.py is launching")
+
+
+def setup_hostname(hostname: str):
+    os.environ["ALEPH_ADDRESS_TO_USE"] = hostname
+    system(f"hostname {hostname}")
+
+
+def setup_variables(variables: Optional[Dict[str, str]]):
+    if variables is None:
+        return
+    for key, value in variables.items():
+        os.environ[key] = value
+
+
+def setup_network(
+    ip: Optional[str], route: Optional[str], dns_servers: Optional[List[str]] = None
+):
+    """Setup the system with info from the host."""
+    dns_servers = dns_servers or []
+    if not os.path.exists("/sys/class/net/eth0"):
+        logger.info("No network interface eth0")
+        return
+
+    if not ip:
+        logger.info("No network IP")
+        return
+
+    logger.debug("Setting up networking")
+    system("ip addr add 127.0.0.1/8 dev lo brd + scope host")
+    system("ip addr add ::1/128 dev lo")
+    system("ip link set lo up")
+    if "/" in ip:
+        # Forward compatibility with future supervisors that pass the mask with the IP.
+        system(f"ip addr add {ip} dev eth0")
+    else:
+        logger.warning("Not passing the mask with the IP is deprecated and will be unsupported")
+        system(f"ip addr add {ip}/24 dev eth0")
+    system("ip link set eth0 up")
+
+    if route:
+        system(f"ip route add default via {route} dev eth0")
+        logger.debug(f"IP and route set: {ip} via {route}")
+    else:
+        logger.warning("IP set with no network route")
+
+    with open("/etc/resolv.conf", "wb") as resolvconf_fd:
+        for server in dns_servers:
+            resolvconf_fd.write(f"nameserver {server}\n".encode())
+
+
+def setup_input_data(input_data: bytes):
+    logger.debug("Extracting data")
+    if input_data:
+        # Unzip in /data
+        if not os.path.exists("/opt/input.zip"):
+            open("/opt/input.zip", "wb").write(input_data)
+            os.makedirs("/data", exist_ok=True)
+            os.system("unzip -q /opt/input.zip -d /data")
+
+
+def setup_volumes(volumes: List[Volume]):
+    for volume in volumes:
+        logger.debug(f"Mounting /dev/{volume.device} on {volume.mount}")
+        os.makedirs(volume.mount, exist_ok=True)
+        if volume.read_only:
+            system(f"mount -t squashfs -o ro /dev/{volume.device} {volume.mount}")
+        else:
+            system(f"mount -o rw /dev/{volume.device} {volume.mount}")
+
+    system("mount")
+
+
+def setup_code_asgi(
+    code: bytes, encoding: Encoding, entrypoint: str
+) -> ASGIApplication:
+    # Allow importing packages from /opt/packages
+    sys.path.append("/opt/packages")
+
+    logger.debug("Extracting code")
+    app: ASGIApplication
+    if encoding == Encoding.squashfs:
+        sys.path.append("/opt/code")
+        module_name, app_name = entrypoint.split(":", 1)
+        logger.debug("import module")
+        module = __import__(module_name)
+        for level in module_name.split(".")[1:]:
+            module = getattr(module, level)
+        app = getattr(module, app_name)
+    elif encoding == Encoding.zip:
+        # Unzip in /opt and import the entrypoint from there
+        if not os.path.exists("/opt/archive.zip"):
+            open("/opt/archive.zip", "wb").write(code)
+            logger.debug("Run unzip")
+            os.system("unzip -q /opt/archive.zip -d /opt")
+        sys.path.append("/opt")
+        module_name, app_name = entrypoint.split(":", 1)
+        logger.debug("import module")
+        module = __import__(module_name)
+        for level in module_name.split(".")[1:]:
+            module = getattr(module, level)
+        app = getattr(module, app_name)
+    elif encoding == Encoding.plain:
+        # Execute the code and extract the entrypoint
+        locals: Dict[str, Any] = {}
+        exec(code, globals(), locals)
+        app = locals[entrypoint]
+    else:
+        raise ValueError(f"Unknown encoding '{encoding}'")
+    return app
+
+
+def setup_code_executable(
+    code: Optional[bytes], encoding: Optional[Encoding], entrypoint: Optional[str]
+) -> subprocess.Popen:
+    if not code:
+        logger.debug("No code, it's an instance")
+        process = subprocess.Popen(["/bin/sleep", "infinity"])
+        return process
+    logger.debug("Extracting code")
+    if encoding == Encoding.squashfs:
+        path = f"/opt/code/{entrypoint}"
+        if not os.path.isfile(path):
+            os.system("find /opt/code/")
+            raise FileNotFoundError(f"No such file: {path}")
+        os.system(f"chmod +x {path}")
+    elif encoding == Encoding.zip:
+        open("/opt/archive.zip", "wb").write(code)
+        logger.debug("Run unzip")
+        os.makedirs("/opt/code", exist_ok=True)
+        os.system("unzip /opt/archive.zip -d /opt/code")
+        path = f"/opt/code/{entrypoint}"
+        if not os.path.isfile(path):
+            os.system("find /opt/code")
+            raise FileNotFoundError(f"No such file: {path}")
+        os.system(f"chmod +x {path}")
+    elif encoding == Encoding.plain:
+        os.makedirs("/opt/code", exist_ok=True)
+        path = f"/opt/code/executable {entrypoint}"
+        open(path, "wb").write(code)
+        os.system(f"chmod +x {path}")
+    else:
+        raise ValueError(f"Unknown encoding '{encoding}'. This should never happen.")
+
+    process = subprocess.Popen(path)
+    return process
+
+
+def setup_code(
+    code: Optional[bytes], encoding: Optional[Encoding], entrypoint: Optional[str], interface: Interface
+) -> Union[ASGIApplication, subprocess.Popen]:
+
+    if interface == Interface.asgi:
+        return setup_code_asgi(code=code, encoding=encoding, entrypoint=entrypoint)
+    elif interface == Interface.executable:
+        return setup_code_executable(
+            code=code, encoding=encoding, entrypoint=entrypoint
+        )
+    else:
+        raise ValueError("Invalid interface. This should never happen.")
+
+
+async def run_python_code_http(
+    application: ASGIApplication, scope: dict
+) -> Tuple[Dict, Dict, str, Optional[bytes]]:
+
+    logger.debug("Running code")
+    with StringIO() as buf, redirect_stdout(buf):
+        # Execute in the same process, saves ~20ms than a subprocess
+
+        # The body should not be part of the ASGI scope itself
+        body: bytes = scope.pop("body")
+
+        async def receive():
+            type_ = (
+                "http.request"
+                if scope["type"] in ("http", "websocket")
+                else "aleph.message"
+            )
+            return {"type": type_, "body": body, "more_body": False}
+
+        send_queue: asyncio.Queue = asyncio.Queue()
+
+        async def send(dico):
+            await send_queue.put(dico)
+
+        # TODO: Better error handling
+        logger.debug("Awaiting application...")
+        await application(scope, receive, send)
+
+        logger.debug("Waiting for headers")
+        headers: Dict
+        if scope["type"] == "http":
+            headers = await send_queue.get()
+        else:
+            headers = {}
+
+        logger.debug("Waiting for body")
+        body: Dict = await send_queue.get()
+
+        logger.debug("Waiting for buffer")
+        output = buf.getvalue()
+
+        logger.debug(f"Headers {headers}")
+        logger.debug(f"Body {body}")
+        logger.debug(f"Output {output}")
+
+    logger.debug("Getting output data")
+    output_data: bytes
+    if os.path.isdir("/data") and os.listdir("/data"):
+        make_archive("/opt/output", "zip", "/data")
+        with open("/opt/output.zip", "rb") as output_zipfile:
+            output_data = output_zipfile.read()
+    else:
+        output_data = b""
+
+    logger.debug("Returning result")
+    return headers, body, output, output_data
+
+
+async def make_request(session, scope):
+    async with session.request(
+        scope["method"],
+        url="http://localhost:8080{}".format(scope["path"]),
+        params=scope["query_string"],
+        headers=[(a.decode("utf-8"), b.decode("utf-8")) for a, b in scope["headers"]],
+        data=scope.get("body", None),
+    ) as resp:
+        headers = {
+            "headers": [
+                (a.encode("utf-8"), b.encode("utf-8")) for a, b in resp.headers.items()
+            ],
+            "status": resp.status,
+        }
+        body = {"body": await resp.content.read()}
+    return headers, body
+
+
+async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[bytes]]:
+    logger.debug("Calling localhost")
+
+    tries = 0
+    headers = None
+    body = None
+
+    timeout = aiohttp.ClientTimeout(total=5)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        while not body:
+            try:
+                tries += 1
+                headers, body = await make_request(session, scope)
+            except aiohttp.ClientConnectorError:
+                if tries > 20:
+                    raise
+                await asyncio.sleep(0.05)
+
+    output = ""  # Process stdout is not captured per request
+    output_data = None
+    logger.debug("Returning result")
+    return headers, body, output, output_data
+
+
+async def process_instruction(
+    instruction: bytes,
+    interface: Interface,
+    application: Union[ASGIApplication, subprocess.Popen],
+) -> AsyncIterable[bytes]:
+
+    if instruction == b"halt":
+        logger.info("Received halt command")
+        system("sync")
+        logger.debug("Filesystems synced")
+        if isinstance(application, subprocess.Popen):
+            application.terminate()
+            logger.debug("Application terminated")
+            # application.communicate()
+        else:
+            # Close the cached session in aleph_client:
+            from aleph_client.asynchronous import get_fallback_session
+
+            session: aiohttp.ClientSession = get_fallback_session()
+            await session.close()
+            logger.debug("Aiohttp cached session closed")
+        yield b"STOP\n"
+        logger.debug("Supervisor informed of halt")
+        raise ShutdownException
+    elif instruction.startswith(b"!"):
+        # Execute shell commands in the form `!ls /`
+        msg = instruction[1:].decode()
+        try:
+            process_output = subprocess.check_output(
+                msg, stderr=subprocess.STDOUT, shell=True
+            )
+            yield process_output
+        except subprocess.CalledProcessError as error:
+            yield str(error).encode() + b"\n" + error.output
+    else:
+        # Python
+        logger.debug("msgpack.loads (")
+        msg_ = msgpack.loads(instruction, raw=False)
+        logger.debug("msgpack.loads )")
+        payload = RunCodePayload(**msg_)
+
+        output: Optional[str] = None
+        try:
+            headers: Dict
+            body: Dict
+            output_data: Optional[bytes]
+
+            if interface == Interface.asgi:
+                headers, body, output, output_data = await run_python_code_http(
+                    application=application, scope=payload.scope
+                )
+            elif interface == Interface.executable:
+                headers, body, output, output_data = await run_executable_http(
+                    scope=payload.scope
+                )
+            else:
+                raise ValueError("Unknown interface. This should never happen")
+
+            result = {
+                "headers": headers,
+                "body": body,
+                "output": output,
+                "output_data": output_data,
+            }
+            yield msgpack.dumps(result, use_bin_type=True)
+        except Exception as error:
+            yield msgpack.dumps(
+                {
+                    "error": str(error),
+                    "traceback": str(traceback.format_exc()),
+                    "output": output,
+                }
+            )
+
+
+def receive_data_length(client) -> int:
+    """Receive the length of the data to follow."""
+    buffer = b""
+    for _ in range(9):
+        byte = client.recv(1)
+        if byte == b"\n":
+            break
+        else:
+            buffer += byte
+    return int(buffer)
+
+
+def load_configuration(data: bytes) -> ConfigurationPayload:
+    msg_ = msgpack.loads(data, raw=False)
+    msg_["volumes"] = [Volume(**volume_dict) for volume_dict in msg_.get("volumes")]
+    return ConfigurationPayload(**msg_)
+
+
+def receive_config(client) -> ConfigurationPayload:
+    length = receive_data_length(client)
+    data = b""
+    while len(data) < length:
+        data += client.recv(1024 * 1024)
+    return load_configuration(data)
+
+
+def setup_system(config: ConfigurationPayload):
+    # Linux host names are limited to 63 characters. We therefore use the base32 representation
+    # of the item_hash instead of its common base16 representation.
+    item_hash_binary: bytes = base64.b16decode(config.vm_hash.encode().upper())
+    hostname = base64.b32encode(item_hash_binary).decode().strip('=').lower()
+    setup_hostname(hostname)
+
+    setup_variables(config.variables)
+    setup_volumes(config.volumes)
+    setup_network(config.ip, config.route, config.dns_servers)
+    setup_input_data(config.input_data)
+    logger.debug("Setup finished")
+
+
+def umount_volumes(volumes: List[Volume]):
+    "Umount user related filesystems"
+    system("sync")
+    for volume in volumes:
+        logger.debug(f"Umounting /dev/{volume.device} on {volume.mount}")
+        system(f"umount {volume.mount}")
+
+
+async def main():
+    client, addr = s.accept()
+
+    logger.debug("Receiving setup...")
+    config = receive_config(client)
+    setup_system(config)
+
+    try:
+        app: Union[ASGIApplication, subprocess.Popen] = setup_code(
+            config.code, config.encoding, config.entrypoint, config.interface
+        )
+        client.send(msgpack.dumps({"success": True}))
+    except Exception as error:
+        client.send(
+            msgpack.dumps(
+                {
+                    "success": False,
+                    "error": str(error),
+                    "traceback": str(traceback.format_exc()),
+                }
+            )
+        )
+        logger.exception("Program could not be started")
+        raise
+
+    class ServerReference:
+        "Reference used to close the server from within `handle_instruction"
+        server: asyncio.AbstractServer
+
+    server_reference = ServerReference()
+
+    async def handle_instruction(reader, writer):
+        data = await reader.read(1000_1000)  # Max 1 Mo
+
+        logger.debug("Init received msg")
+        if logger.level <= logging.DEBUG:
+            data_to_print = f"{data[:500]}..." if len(data) > 500 else data
+            logger.debug(f"<<<\n\n{data_to_print}\n\n>>>")
+
+        try:
+            async for result in process_instruction(
+                instruction=data, interface=config.interface, application=app
+            ):
+                writer.write(result)
+                await writer.drain()
+
+                logger.debug("Instruction processed")
+        except ShutdownException:
+            logger.info("Initiating shutdown")
+            writer.write(b"STOPZ\n")
+            await writer.drain()
+            logger.debug("Shutdown confirmed to supervisor")
+            server_reference.server.close()
+            logger.debug("Supervisor socket server closed")
+        finally:
+            writer.close()
+
+    server = await asyncio.start_server(handle_instruction, sock=s)
+    server_reference.server = server
+
+    addr = server.sockets[0].getsockname()
+    print(f"Serving on {addr}")
+
+    try:
+        async with server:
+            await server.serve_forever()
+    except asyncio.CancelledError:
+        logger.debug("Server was properly cancelled")
+    finally:
+        logger.warning("System shutdown")
+        server.close()
+        logger.debug("Server closed")
+        umount_volumes(config.volumes)
+        logger.debug("User volumes unmounted")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
+    asyncio.run(main())
+
+    logger.info("Unmounting system filesystems")
+    system("umount /dev/shm")
+    system("umount /dev/pts")
+    system("umount -a")
+
+    logger.info("Sending reboot syscall")
+    # Send reboot syscall, see man page
+    # https://man7.org/linux/man-pages/man2/reboot.2.html
+    libc = ctypes.CDLL(None)
+    libc.syscall(169, 0xFEE1DEAD, 672274793, 0x4321FEDC, None)
+    # The exit should not happen due to system halt.
+    sys.exit(0)
diff --git a/runtimes/instance-debian-rootfs/inittab b/runtimes/instance-debian-rootfs/inittab
new file mode 100644
index 000000000..7f79023b9
--- /dev/null
+++ b/runtimes/instance-debian-rootfs/inittab
@@ -0,0 +1,22 @@
+# /etc/inittab
+
+::sysinit:/sbin/init sysinit
+::sysinit:/sbin/init boot
+::wait:/sbin/init default
+
+# Set up a couple of getty's
+tty1::respawn:/sbin/getty 38400 tty1
+tty2::respawn:/sbin/getty 38400 tty2
+tty3::respawn:/sbin/getty 38400 tty3
+tty4::respawn:/sbin/getty 38400 tty4
+tty5::respawn:/sbin/getty 38400 tty5
+tty6::respawn:/sbin/getty 38400 tty6
+
+# Put a getty on the serial port
+ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100
+
+# Stuff to do for the 3-finger salute
+::ctrlaltdel:/sbin/reboot
+
+# Stuff to do before rebooting
+::shutdown:/sbin/init shutdown
\ No newline at end of file
diff --git a/runtimes/instance-debian-rootfs/nginx/health-check.conf b/runtimes/instance-debian-rootfs/nginx/health-check.conf
new file mode 100644
index 000000000..d777bfe03
--- /dev/null
+++ b/runtimes/instance-debian-rootfs/nginx/health-check.conf
@@ -0,0 +1,9 @@
+server {
+    listen       8080;
+    server_name  localhost;
+
+    location / {
+        return 200 "healthy\n";
+        add_header Content-Type text/plain;
+    }
+}
\ No newline at end of file
diff --git a/runtimes/instance-debian-rootfs/nginx/index.html b/runtimes/instance-debian-rootfs/nginx/index.html
new file mode 100644
index 000000000..82e6ef7aa
--- /dev/null
+++ b/runtimes/instance-debian-rootfs/nginx/index.html
@@ -0,0 +1,20 @@
+
+
+  
+    
+    
+    
+    Hello World - Nginx Docker
+    
+  
+  
+    
    
+      Hello World
+    
    
+  
+
diff --git a/runtimes/instance-debian-rootfs/nginx/nginx.conf b/runtimes/instance-debian-rootfs/nginx/nginx.conf
new file mode 100644
index 000000000..ee8c7fd6b
--- /dev/null
+++ b/runtimes/instance-debian-rootfs/nginx/nginx.conf
@@ -0,0 +1,31 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log notice;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}
\ No newline at end of file
diff --git a/runtimes/instance-debian-rootfs/update_inits.sh b/runtimes/instance-debian-rootfs/update_inits.sh
new file mode 100755
index 000000000..55a1c99b1
--- /dev/null
+++ b/runtimes/instance-debian-rootfs/update_inits.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+rm ./rootfs.squashfs
+
+set -euf
+
+cp ./init0.sh ./rootfs/sbin/init
+cp ./init1.py ./rootfs/root/init1.py
+chmod +x ./rootfs/sbin/init
+chmod +x ./rootfs/root/init1.py
+
+mksquashfs ./rootfs/ ./rootfs.squashfs
+
+echo "OK"
diff --git a/vm_supervisor/INSTANCES.md b/vm_supervisor/INSTANCES.md
new file mode 100644
index 000000000..dbe07bfe2
--- /dev/null
+++ b/vm_supervisor/INSTANCES.md
@@ -0,0 +1,44 @@
+
+# Instance Messages
+
+Support of Instance message in the aleph-message repository was added in this PR:
+https://github.com/aleph-im/aleph-message/pull/48
+
+## Changes added
+
+### Aleph message repository
+
+I added a new type of message called `InstanceMessage`, with the changes that we designed for VM instances.
+The content of this message is a new type called `InstanceContent`, that replaces the field `runtime` by `rootfs` that
+instead be an Immutable volume becomes a Persistent volume and adds a new field inside called `parent`, that will be the
+item hash of the base filesystem of the VM. We will create a .ext4 file with the size of the volume and **"attach"** to it
+the base filesystem.
+
+Note that this filesystem should be in **.ext4** format, cannot be an **squashfs**
+file, because we will map it as a block device inside the machine.
+
+Also, I added a union type for Instance messages and Program message called `ExecutableMessage` and also a new one called
+`ExecutableContent` as union of Instance and program content types.
+
+### Aleph VM repository
+
+I have created a function called `create_devmapper` in _**vm_supervisor/storage.py**_. This method can create a
+dev-mapper device base on the parent reference. I followed 
+[this](https://community.aleph.im/t/deploying-mutable-vm-instances-on-aleph/56/2) implementation.
+
+In the _**firecracker/microvm.py**_ file I added the `mount_rootfs` method to mount the block device in the case that we
+use jailer and also assign correct permissions. And when the VM goes down, I clear all these configurations in the
+`teardown` process. As link a block device in a chroot doesn't work I had to do a workaround that consists of copy all
+the "dm-*" block devices in the chroot and mount the entire `/dev/mapper` folder in the chroot to make it work. I didn't
+found a better solution to it.
+
+Also, I added support to run a writable root filesystem in Firecracker. I have bypassed all the parts that we pass and
+use the **_"code"_** properties, like the encoding or the entrypoint.
+
+A new instance message example has been added in **_examples/instance_message_from_aleph.json_**.
+
+### Current status
+
+Now the Dev-mapper device works well, Firecracker loads it in write state, but we need to fix 2 things:
+- Route the requests from the CRN to the Firecracker VM on any port, not only using the 8080.
+- ~~- Use the entire hard disk inside VM, because now only detects the size of the rootfs.~~(Done)
diff --git a/vm_supervisor/messages.py b/vm_supervisor/messages.py
index 6c2bc8fa6..505427a16 100644
--- a/vm_supervisor/messages.py
+++ b/vm_supervisor/messages.py
@@ -4,13 +4,13 @@
 
 from aiohttp import ClientConnectorError, ClientResponseError
 from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable
-from aleph_message.models import ProgramMessage
+from aleph_message.models import ExecutableMessage, MessageType
 
 from .models import VmHash
 from .storage import get_latest_amend, get_message
 
 
-async def try_get_message(ref: str) -> ProgramMessage:
+async def try_get_message(ref: str) -> ExecutableMessage:
     """Get the message or raise an aiohttp HTTP error"""
     try:
         return await get_message(ref)
@@ -49,17 +49,24 @@ async def update_with_latest_ref(obj):
         return obj
 
 
-async def update_message(message: ProgramMessage):
-    # Load amends
-    await asyncio.gather(
-        update_with_latest_ref(message.content.runtime),
-        update_with_latest_ref(message.content.code),
-        update_with_latest_ref(message.content.data),
-        *(update_with_latest_ref(volume) for volume in (message.content.volumes or [])),
-    )
+async def update_message(message: ExecutableMessage):
+    if message.type == MessageType.program:
+        # Load amends
+        await asyncio.gather(
+            update_with_latest_ref(message.content.runtime),
+            update_with_latest_ref(message.content.code),
+            update_with_latest_ref(message.content.data),
+            *(update_with_latest_ref(volume) for volume in (message.content.volumes or [])),
+        )
+    else:
+        assert message.type == MessageType.instance
+        await asyncio.gather(
+            update_with_latest_ref(message.content.rootfs.parent),
+            *(update_with_latest_ref(volume) for volume in (message.content.volumes or [])),
+        )
 
 
-async def load_updated_message(ref: VmHash) -> Tuple[ProgramMessage, ProgramMessage]:
+async def load_updated_message(ref: VmHash) -> Tuple[ExecutableMessage, ExecutableMessage]:
     original_message = await try_get_message(ref)
     message = copy.deepcopy(original_message)
     await update_message(message)
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 9e734471c..0ccb2435a 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -7,7 +7,8 @@
 from datetime import datetime
 from typing import Dict, NewType, Optional
 
-from aleph_message.models import ProgramContent
+from aleph_message.models import ExecutableContent, InstanceContent
+from aleph_message.models.execution.base import MachineType
 
 from .conf import settings
 from .metrics import ExecutionRecord, save_execution_data, save_record
@@ -21,7 +22,6 @@
 
 VmHash = NewType("VmHash", str)
 
-
 @dataclass
 class VmExecutionTimes:
     defined_at: datetime
@@ -45,8 +45,8 @@ class VmExecution:
 
     uuid: uuid.UUID  # Unique identifier of this execution
     vm_hash: VmHash
-    original: ProgramContent
-    program: ProgramContent
+    original: ExecutableContent
+    message: ExecutableContent
     resources: Optional[AlephFirecrackerResources] = None
     vm: Optional[AlephFirecrackerVM] = None
 
@@ -59,6 +59,7 @@ class VmExecution:
     update_task: Optional[asyncio.Task] = None
 
     persistent: bool = False
+    is_instance: bool = False
 
     @property
     def is_running(self):
@@ -73,16 +74,17 @@ def vm_id(self) -> Optional[int]:
         return self.vm.vm_id if self.vm else None
 
     def __init__(
-        self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent
+        self, vm_hash: VmHash, message: ExecutableContent, original: ExecutableContent
     ):
         self.uuid = uuid.uuid1()  # uuid1() includes the hardware address and timestamp
         self.vm_hash = vm_hash
-        self.program = program
+        self.message = message
         self.original = original
         self.times = VmExecutionTimes(defined_at=datetime.now())
         self.ready_event = asyncio.Event()
         self.concurrent_runs = 0
         self.runs_done_event = asyncio.Event()
+        self.is_instance = isinstance(self.message, InstanceContent)
 
     def to_dict(self) -> Dict:
         return {
@@ -96,7 +98,7 @@ def to_json(self, indent: Optional[int] = None) -> str:
     async def prepare(self):
         """Download VM required files"""
         self.times.preparing_at = datetime.now()
-        resources = AlephFirecrackerResources(self.program, namespace=self.vm_hash)
+        resources = AlephFirecrackerResources(self.message, namespace=self.vm_hash)
         await resources.download_all()
         self.times.prepared_at = datetime.now()
         self.resources = resources
@@ -111,9 +113,10 @@ async def create(
             vm_id=vm_id,
             vm_hash=self.vm_hash,
             resources=self.resources,
-            enable_networking=self.program.environment.internet,
-            hardware_resources=self.program.resources,
+            enable_networking=self.message.environment.internet,
+            hardware_resources=self.message.resources,
             tap_interface=tap_interface,
+            is_instance=self.is_instance,
         )
         try:
             await vm.setup()
@@ -187,16 +190,25 @@ def start_watching_for_updates(self, pubsub: PubSub):
             )
 
     async def watch_for_updates(self, pubsub: PubSub):
-        await pubsub.msubscribe(
-            self.original.code.ref,
-            self.original.runtime.ref,
-            self.original.data.ref if self.original.data else None,
-            *(
-                volume.ref
-                for volume in (self.original.volumes or [])
-                if hasattr(volume, "ref")
-            ),
-        )
+        if self.is_instance:
+            await pubsub.msubscribe(
+                *(
+                    volume.ref
+                    for volume in (self.original.volumes or [])
+                    if hasattr(volume, "ref")
+                ),
+            )
+        else:
+            await pubsub.msubscribe(
+                self.original.code.ref,
+                self.original.runtime.ref,
+                self.original.data.ref if self.original.data else None,
+                *(
+                    volume.ref
+                    for volume in (self.original.volumes or [])
+                    if hasattr(volume, "ref")
+                ),
+            )
         logger.debug("Update received, stopping VM...")
         await self.stop()
 
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 2f889955b..e7ad57a47 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -2,12 +2,12 @@
 import logging
 from typing import Dict, Iterable, Optional
 
-from aleph_message.models import ProgramContent, ProgramMessage
+from aleph_message.models import ExecutableMessage
 
 from vm_supervisor.network.hostnetwork import Network
 
 from .conf import settings
-from .models import VmExecution, VmHash
+from .models import VmExecution, VmHash, ExecutableContent
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +23,7 @@ class VmPool:
 
     counter: int  # Used to provide distinct ids to network interfaces
     executions: Dict[VmHash, VmExecution]
-    message_cache: Dict[str, ProgramMessage] = {}
+    message_cache: Dict[str, ExecutableMessage] = {}
     network: Optional[Network]
 
     def __init__(self):
@@ -40,10 +40,10 @@ def __init__(self):
         )
 
     async def create_a_vm(
-        self, vm_hash: VmHash, program: ProgramContent, original: ProgramContent
+        self, vm_hash: VmHash, message: ExecutableContent, original: ExecutableContent
     ) -> VmExecution:
         """Create a new Aleph Firecracker VM from an Aleph function message."""
-        execution = VmExecution(vm_hash=vm_hash, program=program, original=original)
+        execution = VmExecution(vm_hash=vm_hash, message=message, original=original)
         self.executions[vm_hash] = execution
         await execution.prepare()
         vm_id = self.get_unique_vm_id()
@@ -115,3 +115,8 @@ def get_persistent_executions(self) -> Iterable[VmExecution]:
         for vm_hash, execution in self.executions.items():
             if execution.persistent and execution.is_running:
                 yield execution
+
+    def get_instance_executions(self) -> Iterable[VmExecution]:
+        for vm_hash, execution in self.executions.items():
+            if execution.is_instance and execution.is_running:
+                yield execution
diff --git a/vm_supervisor/reactor.py b/vm_supervisor/reactor.py
index a8eb4d9f7..217bc2f2d 100644
--- a/vm_supervisor/reactor.py
+++ b/vm_supervisor/reactor.py
@@ -25,7 +25,7 @@ def is_equal_or_includes(value, compare_to) -> bool:
         raise ValueError("Unsupported value")
 
 
-def subscription_matches(subscription: Subscription, message: ProgramMessage) -> bool:
+def subscription_matches(subscription: Subscription, message: AlephMessage) -> bool:
     if not subscription:
         # Require at least one value to match
         return False
@@ -38,7 +38,7 @@ def subscription_matches(subscription: Subscription, message: ProgramMessage) ->
 class Reactor:
 
     pubsub: PubSub
-    listeners: List[ProgramMessage]
+    listeners: List[AlephMessage]
 
     def __init__(self, pubsub: PubSub):
         self.pubsub = pubsub
@@ -67,7 +67,7 @@ async def trigger(self, message: AlephMessage):
         for coroutine in coroutines:
             create_task_log_exceptions(coroutine)
 
-    def register(self, message: ProgramMessage):
+    def register(self, message: AlephMessage):
         if message.content.on.message:
             self.listeners.append(message)
         else:
diff --git a/vm_supervisor/resources.py b/vm_supervisor/resources.py
index 80341bb3c..d914240f8 100644
--- a/vm_supervisor/resources.py
+++ b/vm_supervisor/resources.py
@@ -122,5 +122,6 @@ async def about_system_usage(request: web.Request):
 
 class Allocation(BaseModel):
     persistent_vms: Set[str]
+    instances: Optional[Set[str]] = None
     on_demand_vms: Optional[Set[str]] = None
     jobs: Optional[Set] = None
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 2e5ca0ff2..60b01af67 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -52,7 +52,7 @@ async def create_vm_execution(vm_hash: VmHash) -> VmExecution:
     try:
         execution = await pool.create_a_vm(
             vm_hash=vm_hash,
-            program=message.content,
+            message=message.content,
             original=original_message.content,
         )
     except ResourceDownloadError as error:
@@ -64,7 +64,7 @@ async def create_vm_execution(vm_hash: VmHash) -> VmExecution:
     except VmSetupError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(reason="Error during program initialisation")
+        raise HTTPInternalServerError(reason="Error during vm initialisation")
     except MicroVMFailedInit as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
@@ -145,7 +145,7 @@ async def run_code_on_request(
         headers.update(
             {
                 "Aleph-Program-ItemHash": execution.vm_hash,
-                "Aleph-Program-Code-Ref": execution.program.code.ref,
+                "Aleph-Program-Code-Ref": execution.message.code.ref if not execution.is_instance else None,
                 # "Aleph-Compute-Vm-Id": str(execution.vm.vm_id),
             }
         )
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 5869571ca..b1defe02c 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -11,59 +11,64 @@
 import os
 import re
 import sys
+import subprocess
 from os.path import isfile, join
 from pathlib import Path
 from shutil import make_archive
+from typing import Union
 
 import aiohttp
-from aleph_message.models import ProgramMessage
-from aleph_message.models.execution.base import (
-    Encoding,
-)
+from aleph_message.models import ExecutableMessage, InstanceMessage, ProgramMessage, MessageType
 from aleph_message.models.execution.volume import (
     ImmutableVolume,
     MachineVolume,
+    ImmutableVolume,
     PersistentVolume,
     VolumePersistence,
 )
+from aleph_message.models.execution.program import Encoding
+from aleph_message.models.execution.instance import RootfsVolume
 
 from .conf import settings
 
 logger = logging.getLogger(__name__)
 
+DEVICE_MAPPER_DIRECTORY = "/dev/mapper"
+
 
 async def download_file(url: str, local_path: Path) -> None:
     # TODO: Limit max size of download to the message specification
     if isfile(local_path):
         logger.debug(f"File already exists: {local_path}")
-    else:
-        tmp_path = f"{local_path}.part"
-        logger.debug(f"Downloading {url} -> {tmp_path}")
-        async with aiohttp.ClientSession() as session:
-            resp = await session.get(url)
-            resp.raise_for_status()
+        return
+
+    tmp_path = f"{local_path}.part"
+    logger.debug(f"Downloading {url} -> {tmp_path}")
+    async with aiohttp.ClientSession() as session:
+        resp = await session.get(url)
+        resp.raise_for_status()
+        try:
+            with open(tmp_path, "wb") as cache_file:
+                counter = 0
+                while True:
+                    chunk = await resp.content.read(65536)
+                    if not chunk:
+                        break
+                    cache_file.write(chunk)
+                    counter += 1
+                    if not (counter % 20):
+                        sys.stdout.write(".")
+                        sys.stdout.flush()
+
+            os.rename(tmp_path, local_path)
+            logger.debug(f"Download complete, moved {tmp_path} -> {local_path}")
+        except Exception:
+            # Ensure no partial file is left
             try:
-                with open(tmp_path, "wb") as cache_file:
-                    counter = 0
-                    while True:
-                        chunk = await resp.content.read(65536)
-                        if not chunk:
-                            break
-                        cache_file.write(chunk)
-                        counter += 1
-                        if not (counter % 20):
-                            sys.stdout.write(".")
-                            sys.stdout.flush()
-
-                os.rename(tmp_path, local_path)
-                logger.debug(f"Download complete, moved {tmp_path} -> {local_path}")
-            except Exception:
-                # Ensure no partial file is left
-                try:
-                    os.remove(tmp_path)
-                except FileNotFoundError:
-                    pass
-                raise
+                os.remove(tmp_path)
+            except FileNotFoundError:
+                pass
+            raise
 
 
 async def get_latest_amend(item_hash: str) -> str:
@@ -79,7 +84,7 @@ async def get_latest_amend(item_hash: str) -> str:
             return result or item_hash
 
 
-async def get_message(ref: str) -> ProgramMessage:
+async def get_message(ref: str) -> ExecutableMessage:
     if settings.FAKE_DATA_PROGRAM:
         cache_path = settings.FAKE_DATA_MESSAGE
     else:
@@ -94,7 +99,9 @@ async def get_message(ref: str) -> ProgramMessage:
             msg["item_hash"] = hashlib.sha256(
                 msg["item_content"].encode("utf-8")
             ).hexdigest()
-        return ProgramMessage.parse_obj(msg)
+        if msg["type"] == MessageType.program:
+            return ProgramMessage.parse_obj(msg)
+        return InstanceMessage.parse_obj(msg)
 
 
 async def get_code_path(ref: str) -> Path:
@@ -142,6 +149,8 @@ async def get_runtime_path(ref: str) -> Path:
     cache_path = Path(join(settings.RUNTIME_CACHE, ref))
     url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}"
     await download_file(url, cache_path)
+    if settings.USE_JAILER:
+        os.system(f"chown jailman:jailman {cache_path}")
     return cache_path
 
 
@@ -157,28 +166,127 @@ def create_ext4(path: Path, size_mib: int) -> bool:
     return True
 
 
+async def create_volume_file(volume: Union[PersistentVolume, RootfsVolume], namespace: str) -> Path:
+    volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
+    path = Path(settings.PERSISTENT_VOLUMES_DIR) / namespace / f"{volume_name}.ext4"
+    if not path.is_file():
+        logger.debug(f"Creating {volume.size_mib}MB volume")
+        os.system(f"dd if=/dev/zero of={path} bs=1M count={volume.size_mib}")
+        if settings.USE_JAILER:
+            os.system(f"chown jailman:jailman {path}")
+    return path
+
+
+async def create_loopback_device(path: Path, read_only: bool = False) -> str:
+    command_args = [
+        "losetup",
+        "--find",
+        "--show"
+    ]
+    if read_only:
+        command_args.append("--read-only")
+    command_args.append(str(path))
+    loop_device = subprocess.run(
+        command_args,
+        check=True,
+        capture_output=True,
+        encoding="UTF-8").stdout.strip()
+    return loop_device
+
+
+def get_block_size(device_path: Path) -> str:
+    block_size = subprocess.run(
+        ["blockdev", "--getsz", device_path],
+        check=True,
+        capture_output=True,
+        encoding="UTF-8").stdout.strip()
+    return block_size
+
+
+def create_mapped_device(device_name: str, table_command: str) -> None:
+    subprocess.run(f"dmsetup create {device_name}",
+                   input=table_command,
+                   text=True,
+                   shell=True,
+                   check=True,
+                   stdout=subprocess.PIPE,
+                   stderr=subprocess.PIPE)
+
+
+def e2fs_check_and_resize(device_path: Path) -> None:
+    os.system(f"e2fsck -fy {device_path}")
+    os.system(f"resize2fs {device_path}")
+
+
+async def create_devmapper(volume: Union[PersistentVolume, RootfsVolume], namespace: str) -> Path:
+    """It creates a /dev/mapper/DEVICE inside the VM, that is an extended mapped device of the volume specified.
+    We follow the steps described here: https://community.aleph.im/t/deploying-mutable-vm-instances-on-aleph/56/2"""
+    volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
+    mapped_volume_name = f"{namespace}_{volume_name}"
+    path_mapped_volume_name = Path(DEVICE_MAPPER_DIRECTORY) / mapped_volume_name
+
+    if path_mapped_volume_name.is_block_device():
+        return path_mapped_volume_name
+
+    volume_path = await create_volume_file(volume, namespace)
+    parent_path = await get_runtime_path(volume.parent.ref)
+
+    base_loop_device = await create_loopback_device(parent_path, read_only=True)
+    base_block_size = get_block_size(parent_path)
+    extended_loop_device = await create_loopback_device(volume_path)
+    extended_block_size = get_block_size(volume_path)
+
+    base_table_command = f"0 {base_block_size} linear {base_loop_device} 0\n{base_block_size} {extended_block_size} zero"
+    base_volume_name = volume.parent.ref
+    path_base_device_name = Path(DEVICE_MAPPER_DIRECTORY) / base_volume_name
+    if not path_base_device_name.is_block_device():
+        create_mapped_device(base_volume_name, base_table_command)
+
+    snapshot_table_command = f"0 {extended_block_size} snapshot {path_base_device_name} {extended_loop_device} P 8"
+    create_mapped_device(mapped_volume_name, snapshot_table_command)
+
+    e2fs_check_and_resize(path_mapped_volume_name)
+    if settings.USE_JAILER:
+        os.system(f"chown jailman:jailman {path_base_device_name}")
+        os.system(f"chown jailman:jailman {path_mapped_volume_name}")
+    return path_mapped_volume_name
+
+
+async def get_existing_file(ref: str) -> Path:
+    if settings.FAKE_DATA_PROGRAM and settings.FAKE_DATA_VOLUME:
+        return Path(settings.FAKE_DATA_VOLUME)
+
+    cache_path = Path(join(settings.DATA_CACHE, ref))
+    url = f"{settings.CONNECTOR_URL}/download/data/{ref}"
+    await download_file(url, cache_path)
+    if settings.USE_JAILER:
+        os.system(f"chown jailman:jailman {cache_path}")
+    return cache_path
+
+
 async def get_volume_path(volume: MachineVolume, namespace: str) -> Path:
     if isinstance(volume, ImmutableVolume):
         ref = volume.ref
-        if settings.FAKE_DATA_PROGRAM and settings.FAKE_DATA_VOLUME:
-            return Path(settings.FAKE_DATA_VOLUME)
-
-        cache_path = Path(join(settings.DATA_CACHE, ref))
-        url = f"{settings.CONNECTOR_URL}/download/data/{ref}"
-        await download_file(url, cache_path)
-        return cache_path
-    elif isinstance(volume, PersistentVolume):
+        return await get_existing_file(ref)
+    elif isinstance(volume, PersistentVolume) or isinstance(volume, RootfsVolume):
+        volume_name = volume.name if isinstance(volume, RootfsVolume) else "rootfs"
         if volume.persistence != VolumePersistence.host:
             raise NotImplementedError("Only 'host' persistence is supported")
-        if not re.match(r"^[\w\-_/]+$", volume.name):
-            raise ValueError(f"Invalid value for volume name: {volume.name}")
+        if not re.match(r"^[\w\-_/]+$", volume_name):
+            raise ValueError(f"Invalid value for volume name: {volume_name}")
         os.makedirs(join(settings.PERSISTENT_VOLUMES_DIR, namespace), exist_ok=True)
-        volume_path = Path(
-            join(settings.PERSISTENT_VOLUMES_DIR, namespace, f"{volume.name}.ext4")
-        )
-        await asyncio.get_event_loop().run_in_executor(
-            None, create_ext4, volume_path, volume.size_mib
-        )
-        return volume_path
+        if volume.parent:
+            device_path = await asyncio.get_event_loop().run_in_executor(
+                None, create_devmapper, volume, namespace
+            )
+            return device_path
+        else:
+            volume_path = Path(
+                join(settings.PERSISTENT_VOLUMES_DIR, namespace, f"{volume_name}.ext4")
+            )
+            await asyncio.get_event_loop().run_in_executor(
+                None, create_ext4, volume_path, volume.size_mib
+            )
+            return volume_path
     else:
         raise NotImplementedError("Only immutable volumes are supported")
diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py
index f2b78c99c..42b29f8ec 100644
--- a/vm_supervisor/tasks.py
+++ b/vm_supervisor/tasks.py
@@ -8,10 +8,9 @@
 import aiohttp
 import pydantic
 from aiohttp import web
-from aleph_message import parse_message
-from aleph_message.models import BaseMessage, ProgramMessage
 from yarl import URL
 
+from aleph_message.models import AlephMessage, ProgramMessage, InstanceMessage, parse_message
 from .conf import settings
 from .messages import load_updated_message
 from .models import VmHash
@@ -36,7 +35,7 @@ async def retry_generator(
         retry_delay = max(retry_delay * 2, max_seconds)
 
 
-async def subscribe_via_ws(url) -> AsyncIterable[BaseMessage]:
+async def subscribe_via_ws(url) -> AsyncIterable[AlephMessage]:
     logger.debug("subscribe_via_ws()")
     async with aiohttp.ClientSession() as session:
         async with session.ws_connect(url) as ws:
@@ -117,8 +116,9 @@ async def start_watch_for_messages_task(app: web.Application):
     sample_message, _ = await load_updated_message(
         ref=VmHash("cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e")
     )
-    assert sample_message.content.on.message, sample_message
-    reactor.register(sample_message)
+    if isinstance(sample_message, ProgramMessage):
+        assert sample_message.content.on.message, sample_message
+        reactor.register(sample_message)
 
     app["pubsub"] = pubsub
     app["reactor"] = reactor
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index fda5c7889..fd1f8606b 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -215,6 +215,19 @@ async def update_allocations(request: web.Request):
             await execution.stop()
             execution.persistent = False
 
+    # Start Instances
+    for instance_hash in allocation.instances:
+        instance_hash = VmHash(instance_hash)
+        logger.info(f"Starting instance {instance_hash}")
+        await start_persistent_vm(instance_hash, pubsub)
+
+    # Stop Instances
+    for execution in pool.get_instance_executions():
+        if execution.vm_hash not in allocation.instances:
+            logger.info(f"Stopping instance {execution.vm_hash}")
+            await execution.stop()
+            execution.persistent = False
+
     # Log unsupported features
     if allocation.on_demand_vms:
         logger.warning("Not supported yet: 'allocation.on_demand_vms'")
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_microvm.py
index 549aa570b..2b2c450d9 100644
--- a/vm_supervisor/vm/firecracker_microvm.py
+++ b/vm_supervisor/vm/firecracker_microvm.py
@@ -33,9 +33,10 @@
 from guest_api.__main__ import run_guest_api
 
 from ..conf import settings
+from ..models import ExecutableContent
 from ..network.firewall import teardown_nftables_for_vm
 from ..network.interfaces import TapInterface
-from ..storage import get_code_path, get_data_path, get_runtime_path, get_volume_path
+from ..storage import get_code_path, get_data_path, get_runtime_path, get_volume_path, create_devmapper
 
 logger = logging.getLogger(__name__)
 set_start_method("spawn")
@@ -87,12 +88,12 @@ class HostVolume:
 
 @dataclass
 class ConfigurationPayload:
-    code: bytes
-    encoding: Encoding
-    entrypoint: str
     input_data: bytes
     interface: Interface
     vm_hash: str
+    code: Optional[bytes] = None
+    encoding: Optional[Encoding] = None
+    entrypoint: Optional[str] = None
     ip: Optional[str] = None
     route: Optional[str] = None
     dns_servers: List[str] = field(default_factory=list)
@@ -120,33 +121,40 @@ def as_msgpack(self) -> bytes:
 
 class AlephFirecrackerResources:
 
-    message_content: ProgramContent
+    message_content: ExecutableContent
 
     kernel_image_path: Path
-    code_path: Path
-    code_encoding: Encoding
-    code_entrypoint: str
+    code_path: Optional[Path]
+    code_encoding: Optional[Encoding]
+    code_entrypoint: Optional[str]
     rootfs_path: Path
     volumes: List[HostVolume]
     volume_paths: Dict[str, Path]
     data_path: Optional[Path]
     namespace: str
 
-    def __init__(self, message_content: ProgramContent, namespace: str):
+    def __init__(self, message_content: ExecutableContent, namespace: str):
         self.message_content = message_content
-        self.code_encoding = message_content.code.encoding
-        self.code_entrypoint = message_content.code.entrypoint
         self.namespace = namespace
+        if hasattr(message_content, "code"):
+            self.code_encoding = message_content.code.encoding
+            self.code_entrypoint = message_content.code.entrypoint
+        else:
+            self.code_path = None
+            self.code_encoding = None
+            self.code_entrypoint = None
 
     def to_dict(self):
         return self.__dict__
 
     async def download_kernel(self):
         # Assumes kernel is already present on the host
-        self.kernel_image_path = settings.LINUX_PATH
+        self.kernel_image_path = Path(settings.LINUX_PATH)
         assert isfile(self.kernel_image_path)
 
     async def download_code(self):
+        if not hasattr(self.message_content, "code"):
+            return
         code_ref: str = self.message_content.code.ref
         try:
             self.code_path = await get_code_path(code_ref)
@@ -155,21 +163,26 @@ async def download_code(self):
         assert isfile(self.code_path), f"Code not found on '{self.code_path}'"
 
     async def download_runtime(self):
-        runtime_ref: str = self.message_content.runtime.ref
-        try:
-            self.rootfs_path = await get_runtime_path(runtime_ref)
-        except ClientResponseError as error:
-            raise ResourceDownloadError(error)
-        assert isfile(self.rootfs_path), f"Runtime not found on {self.rootfs_path}"
-
-    async def download_data(self):
-        if self.message_content.data:
-            data_ref: str = self.message_content.data.ref
+        if hasattr(self.message_content, "rootfs"):
+            self.rootfs_path = await create_devmapper(self.message_content.rootfs, self.namespace)
+            assert self.rootfs_path.is_block_device(), f"Runtime not found on {self.rootfs_path}"
+        else:
+            runtime_ref: str = self.message_content.runtime.ref
             try:
-                self.data_path = await get_data_path(data_ref)
+                self.rootfs_path = await get_runtime_path(runtime_ref)
             except ClientResponseError as error:
                 raise ResourceDownloadError(error)
-            assert isfile(self.data_path)
+            assert isfile(self.rootfs_path), f"Runtime not found on {self.rootfs_path}"
+
+    async def download_data(self):
+        if hasattr(self.message_content, "data"):
+            if self.message_content.data:
+                data_ref: str = self.message_content.data.ref
+                try:
+                    self.data_path = await get_data_path(data_ref)
+                except ClientResponseError as error:
+                    raise ResourceDownloadError(error)
+                assert isfile(self.data_path)
         else:
             self.data_path = None
 
@@ -191,8 +204,8 @@ async def download_volumes(self):
     async def download_all(self):
         await asyncio.gather(
             self.download_kernel(),
-            self.download_code(),
             self.download_runtime(),
+            self.download_code(),
             self.download_volumes(),
             self.download_data(),
         )
@@ -212,6 +225,7 @@ class AlephFirecrackerVM:
     resources: AlephFirecrackerResources
     enable_console: bool
     enable_networking: bool
+    is_instance: bool
     hardware_resources: MachineResources
     fvm: Optional[MicroVM] = None
     guest_api_process: Optional[Process] = None
@@ -226,6 +240,7 @@ def __init__(
         enable_console: Optional[bool] = None,
         hardware_resources: MachineResources = MachineResources(),
         tap_interface: Optional[TapInterface] = None,
+        is_instance: bool = False,
     ):
         self.vm_id = vm_id
         self.vm_hash = vm_hash
@@ -236,6 +251,7 @@ def __init__(
         self.enable_console = enable_console
         self.hardware_resources = hardware_resources
         self.tap_interface = tap_interface
+        self.is_instance = is_instance
 
     def to_dict(self):
         if self.fvm.proc and psutil:
@@ -282,19 +298,19 @@ async def setup(self):
                 kernel_image_path=Path(
                     fvm.enable_kernel(self.resources.kernel_image_path)
                 ),
-                boot_args=BootSource.args(enable_console=self.enable_console),
+                boot_args=BootSource.args(enable_console=self.enable_console, writable=self.is_instance),
             ),
             drives=[
                 Drive(
                     drive_id="rootfs",
-                    path_on_host=Path(fvm.enable_rootfs(self.resources.rootfs_path)),
+                    path_on_host=fvm.enable_rootfs(self.resources.rootfs_path),
                     is_root_device=True,
-                    is_read_only=True,
+                    is_read_only=not self.is_instance,
                 ),
             ]
             + (
                 [fvm.enable_drive(self.resources.code_path)]
-                if self.resources.code_encoding == Encoding.squashfs
+                if hasattr(self.resources, "code_encoding") and self.resources.code_encoding == Encoding.squashfs
                 else []
             )
             + [
@@ -344,17 +360,18 @@ async def configure(self):
         """Configure the VM by sending configuration info to it's init"""
 
         if (
-            self.resources.data_path
+            hasattr(self.resources, "data_path") and self.resources.data_path
             and os.path.getsize(self.resources.data_path)
             > settings.MAX_DATA_ARCHIVE_SIZE
         ):
             raise FileTooLargeError(f"Data file too large to pass as an inline zip")
 
-        input_data: bytes = load_file_content(self.resources.data_path)
+        input_data: bytes = load_file_content(self.resources.data_path) if \
+            hasattr(self.resources, "data_path") else None
 
         interface = (
             Interface.asgi
-            if ":" in self.resources.code_entrypoint
+            if self.resources.code_entrypoint and ":" in self.resources.code_entrypoint
             else Interface.executable
         )
 
@@ -371,7 +388,7 @@ async def configure(self):
             ]
         else:
             if (
-                self.resources.data_path
+                hasattr(self.resources, "data_path") and self.resources.data_path
                 and os.path.getsize(self.resources.code_path)
                 > settings.MAX_PROGRAM_ARCHIVE_SIZE
             ):
@@ -379,7 +396,7 @@ async def configure(self):
                     f"Program file too large to pass as an inline zip"
                 )
 
-            code: bytes = load_file_content(self.resources.code_path)
+            code: Optional[bytes] = load_file_content(self.resources.code_path) if self.resources.code_path else None
             volumes = [
                 Volume(
                     mount=volume.mount,

From d400f8ae01664f2bf2d44b7f0d610fe1e35d9852 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 12 Jun 2023 19:37:36 +0200
Subject: [PATCH 391/990] Refactor: Module contains code specific to program
 execution

Solution: Rename module `firecracker_microvm` into `firecracker_program`, in preparation to add new modules next to it.
---
 vm_supervisor/models.py                                     | 2 +-
 vm_supervisor/run.py                                        | 2 +-
 vm_supervisor/vm/__init__.py                                | 6 +++---
 .../vm/{firecracker_microvm.py => firecracker_program.py}   | 0
 4 files changed, 5 insertions(+), 5 deletions(-)
 rename vm_supervisor/vm/{firecracker_microvm.py => firecracker_program.py} (100%)

diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 0ccb2435a..5fe3f763d 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -16,7 +16,7 @@
 from .pubsub import PubSub
 from .utils import create_task_log_exceptions, dumps_for_json
 from .vm import AlephFirecrackerVM
-from .vm.firecracker_microvm import AlephFirecrackerResources
+from .vm.firecracker_program import AlephFirecrackerResources
 
 logger = logging.getLogger(__name__)
 
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 60b01af67..8aac6549c 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -14,7 +14,7 @@
 from .models import VmExecution, VmHash
 from .pool import VmPool
 from .pubsub import PubSub
-from .vm.firecracker_microvm import (
+from .vm.firecracker_program import (
     FileTooLargeError,
     ResourceDownloadError,
     VmSetupError,
diff --git a/vm_supervisor/vm/__init__.py b/vm_supervisor/vm/__init__.py
index 414fd5790..e263ec434 100644
--- a/vm_supervisor/vm/__init__.py
+++ b/vm_supervisor/vm/__init__.py
@@ -1,4 +1,4 @@
-from . import firecracker_microvm
-from .firecracker_microvm import AlephFirecrackerVM
+from . import firecracker_program
+from .firecracker_program import AlephFirecrackerVM
 
-__all__ = ("firecracker_microvm", "AlephFirecrackerVM")
+__all__ = ("firecracker_program", "AlephFirecrackerVM")
diff --git a/vm_supervisor/vm/firecracker_microvm.py b/vm_supervisor/vm/firecracker_program.py
similarity index 100%
rename from vm_supervisor/vm/firecracker_microvm.py
rename to vm_supervisor/vm/firecracker_program.py

From 7e4b93b56acd3c2cf3f12cea4efc06fd0f442d26 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 12 Jun 2023 19:50:56 +0200
Subject: [PATCH 392/990] Intermediate: Copy code from branch
 `andres_split_microvm_class`

---
 vm_supervisor/vm/firecracker_executable.py | 322 +++++++++++++++++++++
 vm_supervisor/vm/firecracker_program.py    | 241 +++------------
 2 files changed, 360 insertions(+), 203 deletions(-)
 create mode 100644 vm_supervisor/vm/firecracker_executable.py

diff --git a/vm_supervisor/vm/firecracker_executable.py b/vm_supervisor/vm/firecracker_executable.py
new file mode 100644
index 000000000..13cc2cebe
--- /dev/null
+++ b/vm_supervisor/vm/firecracker_executable.py
@@ -0,0 +1,322 @@
+import asyncio
+import dataclasses
+import logging
+import subprocess
+from dataclasses import dataclass, field
+from enum import Enum
+from multiprocessing import Process, set_start_method
+from os.path import exists, isfile
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import msgpack
+
+try:
+    import psutil as psutil
+except ImportError:
+    psutil = None
+from aiohttp import ClientResponseError
+from aleph_message.models.execution.environment import MachineResources
+
+from firecracker.config import (
+    BootSource,
+    Drive,
+    FirecrackerConfig,
+    MachineConfig,
+    NetworkInterface,
+    Vsock,
+)
+from firecracker.microvm import MicroVM, setfacl
+from guest_api.__main__ import run_guest_api
+
+from ..conf import settings
+from ..models import ExecutableContent
+from ..network.firewall import teardown_nftables_for_vm
+from ..network.interfaces import TapInterface
+from ..storage import get_volume_path
+
+logger = logging.getLogger(__name__)
+set_start_method("spawn")
+
+class ResourceDownloadError(ClientResponseError):
+    """An error occurred while downloading a VM resource file"""
+
+    def __init__(self, error: ClientResponseError):
+        super().__init__(
+            request_info=error.request_info,
+            history=error.history,
+            status=error.status,
+            message=error.message,
+            headers=error.headers,
+        )
+
+
+class Interface(str, Enum):
+    asgi = "asgi"
+    executable = "executable"
+
+
+@dataclass
+class Volume:
+    mount: str
+    device: str
+    read_only: bool
+
+
+@dataclass
+class HostVolume:
+    mount: str
+    path_on_host: Path
+    read_only: bool
+
+
+@dataclass
+class VMConfiguration:
+    interface: Interface
+    vm_hash: str
+    ip: Optional[str] = None
+    route: Optional[str] = None
+    dns_servers: List[str] = field(default_factory=list)
+    volumes: List[Volume] = field(default_factory=list)
+    variables: Optional[Dict[str, str]] = None
+
+    def as_msgpack(self) -> bytes:
+        return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True)
+
+
+@dataclass
+class ConfigurationResponse:
+    success: bool
+    error: Optional[str] = None
+    traceback: Optional[str] = None
+
+
+class AlephFirecrackerResources:
+
+    message_content: ExecutableContent
+
+    kernel_image_path: Path
+    rootfs_path: Path
+    volumes: List[HostVolume]
+    volume_paths: Dict[str, Path]
+    namespace: str
+
+    def __init__(self, message_content: ExecutableContent, namespace: str):
+        self.message_content = message_content
+        self.namespace = namespace
+
+    def to_dict(self):
+        return self.__dict__
+
+    async def download_kernel(self):
+        # Assumes kernel is already present on the host
+        self.kernel_image_path = Path(settings.LINUX_PATH)
+        assert isfile(self.kernel_image_path)
+
+    async def download_volumes(self):
+        volumes = []
+        # TODO: Download in parallel
+        for volume in self.message_content.volumes:
+            volumes.append(
+                HostVolume(
+                    mount=volume.mount,
+                    path_on_host=(
+                        await get_volume_path(volume=volume, namespace=self.namespace)
+                    ),
+                    read_only=volume.is_read_only(),
+                )
+            )
+        self.volumes = volumes
+
+    async def download_all(self):
+        await asyncio.gather(
+            self.download_kernel(),
+            self.download_volumes(),
+        )
+
+
+class VmSetupError(Exception):
+    pass
+
+
+class VmInitNotConnected(Exception):
+    pass
+
+
+class AlephFirecrackerVM:
+    vm_id: int
+    vm_hash: str
+    resources: AlephFirecrackerResources
+    enable_console: bool
+    enable_networking: bool
+    is_instance: bool
+    hardware_resources: MachineResources
+    vm_configuration: VMConfiguration
+    fvm: Optional[MicroVM] = None
+    guest_api_process: Optional[Process] = None
+    tap_interface: Optional[TapInterface] = None
+    fvm: MicroVM
+
+    def __init__(
+        self,
+        vm_id: int,
+        vm_hash: str,
+        resources: AlephFirecrackerResources,
+        enable_networking: bool = False,
+        enable_console: Optional[bool] = None,
+        hardware_resources: MachineResources = MachineResources(),
+        tap_interface: Optional[TapInterface] = None,
+    ):
+        self.vm_id = vm_id
+        self.vm_hash = vm_hash
+        self.resources = resources
+        self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING
+        if enable_console is None:
+            enable_console = settings.PRINT_SYSTEM_LOGS
+        self.enable_console = enable_console
+        self.hardware_resources = hardware_resources
+        self.tap_interface = tap_interface
+
+        self.fvm = MicroVM(
+            vm_id=self.vm_id,
+            firecracker_bin_path=settings.FIRECRACKER_PATH,
+            use_jailer=settings.USE_JAILER,
+            jailer_bin_path=settings.JAILER_PATH,
+            init_timeout=settings.INIT_TIMEOUT,
+        )
+        self.fvm.prepare_jailer()
+
+    def to_dict(self):
+        if self.fvm.proc and psutil:
+            try:
+                p = psutil.Process(self.fvm.proc.pid)
+                pid_info = {
+                    "status": p.status(),
+                    "create_time": p.create_time(),
+                    "cpu_times": p.cpu_times(),
+                    "cpu_percent": p.cpu_percent(),
+                    "memory_info": p.memory_info(),
+                    "io_counters": p.io_counters(),
+                    "open_files": p.open_files(),
+                    "connections": p.connections(),
+                    "num_threads": p.num_threads(),
+                    "num_ctx_switches": p.num_ctx_switches(),
+                }
+            except psutil.NoSuchProcess:
+                logger.warning("Cannot read process metrics (process not found)")
+                pid_info = None
+        else:
+            pid_info = None
+
+        return {
+            "process": pid_info,
+            **self.__dict__,
+        }
+
+    async def setup(self, config: FirecrackerConfig):
+        logger.debug("setup started")
+        await setfacl()
+
+        config = config or FirecrackerConfig(
+            boot_source=BootSource(
+                kernel_image_path=Path(
+                    self.fvm.enable_kernel(self.resources.kernel_image_path)
+                ),
+                boot_args=BootSource.args(enable_console=self.enable_console, writable=self.is_instance),
+            ),
+            drives=[
+                Drive(
+                    drive_id="rootfs",
+                    path_on_host=self.fvm.enable_rootfs(self.resources.rootfs_path),
+                    is_root_device=True,
+                    is_read_only=True,
+                ),
+            ]
+            + [
+                self.fvm.enable_drive(volume.path_on_host, read_only=volume.read_only)
+                for volume in self.resources.volumes
+            ],
+            machine_config=MachineConfig(
+                vcpu_count=self.hardware_resources.vcpus,
+                mem_size_mib=self.hardware_resources.memory,
+            ),
+            vsock=Vsock(),
+            network_interfaces=[
+                NetworkInterface(
+                    iface_id="eth0", host_dev_name=self.tap_interface.device_name
+                )
+            ]
+            if self.enable_networking
+            else [],
+        )
+
+        logger.debug(config.json(by_alias=True, exclude_none=True, indent=4))
+
+        try:
+            await self.fvm.start(config)
+            logger.debug("setup done")
+        except Exception:
+            await self.fvm.teardown()
+            teardown_nftables_for_vm(self.vm_id)
+            await self.tap_interface.delete()
+            raise
+
+    async def start(self):
+        logger.debug(f"starting vm {self.vm_id}")
+        if not self.fvm:
+            raise ValueError("No VM found. Call setup() before start()")
+
+        if self.enable_console:
+            self.fvm.start_printing_logs()
+
+        logger.debug(f"started fvm {self.vm_id}")
+
+    async def configure(self, volumes: Optional[List[Volume]], interface: Optional[Interface]):
+        """Configure the VM by sending configuration info to it's init"""
+        volumes = volumes or [
+            Volume(
+                mount=volume.mount,
+                device=self.fvm.drives[index].drive_id,
+                read_only=volume.read_only,
+            )
+            for index, volume in enumerate(self.resources.volumes)
+        ]
+
+        # The ip and route should not contain the network mask in order to maintain
+        # compatibility with the existing runtimes.
+        ip = self.tap_interface.guest_ip.with_prefixlen.split("/", 1)[0]
+        route = str(self.tap_interface.host_ip).split("/", 1)[0]
+
+        self.vm_configuration = VMConfiguration(
+            ip=ip if self.enable_networking else None,
+            route=route if self.enable_networking else None,
+            dns_servers=settings.DNS_NAMESERVERS,
+            vm_hash=self.vm_hash,
+            volumes=volumes,
+            variables=self.resources.message_content.variables,
+            interface=interface,
+        )
+
+    async def start_guest_api(self):
+        logger.debug(f"starting guest API for {self.vm_id}")
+        vsock_path = f"{self.fvm.vsock_path}_53"
+        vm_hash = self.vm_hash
+        self.guest_api_process = Process(
+            target=run_guest_api, args=(vsock_path, vm_hash)
+        )
+        self.guest_api_process.start()
+        while not exists(vsock_path):
+            await asyncio.sleep(0.01)
+        subprocess.run(f"chown jailman:jailman {vsock_path}", shell=True, check=True)
+        logger.debug(f"started guest API for {self.vm_id}")
+
+    async def stop_guest_api(self):
+        if self.guest_api_process and self.guest_api_process._popen:
+            self.guest_api_process.terminate()
+
+    async def teardown(self):
+        if self.fvm:
+            await self.fvm.teardown()
+            teardown_nftables_for_vm(self.vm_id)
+            await self.tap_interface.delete()
+        await self.stop_guest_api()
diff --git a/vm_supervisor/vm/firecracker_program.py b/vm_supervisor/vm/firecracker_program.py
index 2b2c450d9..fbe832d0e 100644
--- a/vm_supervisor/vm/firecracker_program.py
+++ b/vm_supervisor/vm/firecracker_program.py
@@ -2,11 +2,9 @@
 import dataclasses
 import logging
 import os.path
-import subprocess
 from dataclasses import dataclass, field
-from enum import Enum
 from multiprocessing import Process, set_start_method
-from os.path import exists, isfile
+from os.path import isfile
 from pathlib import Path
 from typing import Dict, List, Optional
 
@@ -17,7 +15,6 @@
 except ImportError:
     psutil = None
 from aiohttp import ClientResponseError
-from aleph_message.models import ProgramContent
 from aleph_message.models.execution.base import Encoding
 from aleph_message.models.execution.environment import MachineResources
 
@@ -30,13 +27,13 @@
     Vsock,
 )
 from firecracker.microvm import MicroVM, setfacl
-from guest_api.__main__ import run_guest_api
 
+from .firecracker_microvm import AlephFirecrackerVM, AlephFirecrackerResources, VmSetupError, VmInitNotConnected, \
+    Interface, Volume
 from ..conf import settings
 from ..models import ExecutableContent
-from ..network.firewall import teardown_nftables_for_vm
 from ..network.interfaces import TapInterface
-from ..storage import get_code_path, get_data_path, get_runtime_path, get_volume_path, create_devmapper
+from ..storage import get_code_path, get_data_path, get_runtime_path
 
 logger = logging.getLogger(__name__)
 set_start_method("spawn")
@@ -67,33 +64,14 @@ def __init__(self, error: ClientResponseError):
         )
 
 
-class Interface(str, Enum):
-    asgi = "asgi"
-    executable = "executable"
-
-
-@dataclass
-class Volume:
-    mount: str
-    device: str
-    read_only: bool
-
-
-@dataclass
-class HostVolume:
-    mount: str
-    path_on_host: Path
-    read_only: bool
-
-
 @dataclass
 class ConfigurationPayload:
     input_data: bytes
     interface: Interface
     vm_hash: str
-    code: Optional[bytes] = None
-    encoding: Optional[Encoding] = None
-    entrypoint: Optional[str] = None
+    code: bytes = None
+    encoding: Encoding = None
+    entrypoint: str = None
     ip: Optional[str] = None
     route: Optional[str] = None
     dns_servers: List[str] = field(default_factory=list)
@@ -119,23 +97,15 @@ def as_msgpack(self) -> bytes:
         return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True)
 
 
-class AlephFirecrackerResources:
-
-    message_content: ExecutableContent
+class AlephFunctionResources(AlephFirecrackerResources):
 
-    kernel_image_path: Path
-    code_path: Optional[Path]
-    code_encoding: Optional[Encoding]
-    code_entrypoint: Optional[str]
-    rootfs_path: Path
-    volumes: List[HostVolume]
-    volume_paths: Dict[str, Path]
+    code_path: Path
+    code_encoding: Encoding
+    code_entrypoint: str
     data_path: Optional[Path]
-    namespace: str
 
     def __init__(self, message_content: ExecutableContent, namespace: str):
-        self.message_content = message_content
-        self.namespace = namespace
+        super().__init__(message_content, namespace)
         if hasattr(message_content, "code"):
             self.code_encoding = message_content.code.encoding
             self.code_entrypoint = message_content.code.entrypoint
@@ -144,17 +114,7 @@ def __init__(self, message_content: ExecutableContent, namespace: str):
             self.code_encoding = None
             self.code_entrypoint = None
 
-    def to_dict(self):
-        return self.__dict__
-
-    async def download_kernel(self):
-        # Assumes kernel is already present on the host
-        self.kernel_image_path = Path(settings.LINUX_PATH)
-        assert isfile(self.kernel_image_path)
-
     async def download_code(self):
-        if not hasattr(self.message_content, "code"):
-            return
         code_ref: str = self.message_content.code.ref
         try:
             self.code_path = await get_code_path(code_ref)
@@ -163,43 +123,21 @@ async def download_code(self):
         assert isfile(self.code_path), f"Code not found on '{self.code_path}'"
 
     async def download_runtime(self):
-        if hasattr(self.message_content, "rootfs"):
-            self.rootfs_path = await create_devmapper(self.message_content.rootfs, self.namespace)
-            assert self.rootfs_path.is_block_device(), f"Runtime not found on {self.rootfs_path}"
-        else:
-            runtime_ref: str = self.message_content.runtime.ref
+        runtime_ref: str = self.message_content.runtime.ref
+        try:
+            self.rootfs_path = await get_runtime_path(runtime_ref)
+        except ClientResponseError as error:
+            raise ResourceDownloadError(error)
+        assert isfile(self.rootfs_path), f"Runtime not found on {self.rootfs_path}"
+
+    async def download_data(self):
+        if self.message_content.data:
+            data_ref: str = self.message_content.data.ref
             try:
-                self.rootfs_path = await get_runtime_path(runtime_ref)
+                self.data_path = await get_data_path(data_ref)
             except ClientResponseError as error:
                 raise ResourceDownloadError(error)
-            assert isfile(self.rootfs_path), f"Runtime not found on {self.rootfs_path}"
-
-    async def download_data(self):
-        if hasattr(self.message_content, "data"):
-            if self.message_content.data:
-                data_ref: str = self.message_content.data.ref
-                try:
-                    self.data_path = await get_data_path(data_ref)
-                except ClientResponseError as error:
-                    raise ResourceDownloadError(error)
-                assert isfile(self.data_path)
-        else:
-            self.data_path = None
-
-    async def download_volumes(self):
-        volumes = []
-        # TODO: Download in parallel
-        for volume in self.message_content.volumes:
-            volumes.append(
-                HostVolume(
-                    mount=volume.mount,
-                    path_on_host=(
-                        await get_volume_path(volume=volume, namespace=self.namespace)
-                    ),
-                    read_only=volume.is_read_only(),
-                )
-            )
-        self.volumes = volumes
+            assert isfile(self.data_path)
 
     async def download_all(self):
         await asyncio.gather(
@@ -211,18 +149,10 @@ async def download_all(self):
         )
 
 
-class VmSetupError(Exception):
-    pass
-
-
-class VmInitNotConnected(Exception):
-    pass
-
-
-class AlephFirecrackerVM:
+class AlephFirecrackerFunction(AlephFirecrackerVM):
     vm_id: int
     vm_hash: str
-    resources: AlephFirecrackerResources
+    resources: AlephFunctionResources
     enable_console: bool
     enable_networking: bool
     is_instance: bool
@@ -235,86 +165,41 @@ def __init__(
         self,
         vm_id: int,
         vm_hash: str,
-        resources: AlephFirecrackerResources,
+        resources: AlephFunctionResources,
         enable_networking: bool = False,
         enable_console: Optional[bool] = None,
         hardware_resources: MachineResources = MachineResources(),
-        tap_interface: Optional[TapInterface] = None,
-        is_instance: bool = False,
+        tap_interface: Optional[TapInterface] = None
     ):
-        self.vm_id = vm_id
-        self.vm_hash = vm_hash
-        self.resources = resources
-        self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING
-        if enable_console is None:
-            enable_console = settings.PRINT_SYSTEM_LOGS
-        self.enable_console = enable_console
-        self.hardware_resources = hardware_resources
-        self.tap_interface = tap_interface
-        self.is_instance = is_instance
-
-    def to_dict(self):
-        if self.fvm.proc and psutil:
-            try:
-                p = psutil.Process(self.fvm.proc.pid)
-                pid_info = {
-                    "status": p.status(),
-                    "create_time": p.create_time(),
-                    "cpu_times": p.cpu_times(),
-                    "cpu_percent": p.cpu_percent(),
-                    "memory_info": p.memory_info(),
-                    "io_counters": p.io_counters(),
-                    "open_files": p.open_files(),
-                    "connections": p.connections(),
-                    "num_threads": p.num_threads(),
-                    "num_ctx_switches": p.num_ctx_switches(),
-                }
-            except psutil.NoSuchProcess:
-                logger.warning("Cannot read process metrics (process not found)")
-                pid_info = None
-        else:
-            pid_info = None
-
-        return {
-            "process": pid_info,
-            **self.__dict__,
-        }
+        super().__init__(vm_id, vm_hash, resources, enable_networking, enable_console, hardware_resources, tap_interface)
+        self.is_instance = False
 
     async def setup(self):
         logger.debug("setup started")
         await setfacl()
 
-        fvm = MicroVM(
-            vm_id=self.vm_id,
-            firecracker_bin_path=settings.FIRECRACKER_PATH,
-            use_jailer=settings.USE_JAILER,
-            jailer_bin_path=settings.JAILER_PATH,
-            init_timeout=settings.INIT_TIMEOUT,
-        )
-        fvm.prepare_jailer()
-
         config = FirecrackerConfig(
             boot_source=BootSource(
                 kernel_image_path=Path(
-                    fvm.enable_kernel(self.resources.kernel_image_path)
+                    self.fvm.enable_kernel(self.resources.kernel_image_path)
                 ),
                 boot_args=BootSource.args(enable_console=self.enable_console, writable=self.is_instance),
             ),
             drives=[
                 Drive(
                     drive_id="rootfs",
-                    path_on_host=fvm.enable_rootfs(self.resources.rootfs_path),
+                    path_on_host=self.fvm.enable_rootfs(self.resources.rootfs_path),
                     is_root_device=True,
-                    is_read_only=not self.is_instance,
+                    is_read_only=True,
                 ),
             ]
             + (
-                [fvm.enable_drive(self.resources.code_path)]
+                [self.fvm.enable_drive(self.resources.code_path)]
                 if hasattr(self.resources, "code_encoding") and self.resources.code_encoding == Encoding.squashfs
                 else []
             )
             + [
-                fvm.enable_drive(volume.path_on_host, read_only=volume.read_only)
+                self.fvm.enable_drive(volume.path_on_host, read_only=volume.read_only)
                 for volume in self.resources.volumes
             ],
             machine_config=MachineConfig(
@@ -331,30 +216,7 @@ async def setup(self):
             else [],
         )
 
-        logger.debug(config.json(by_alias=True, exclude_none=True, indent=4))
-
-        try:
-            await fvm.start(config)
-            logger.debug("setup done")
-            self.fvm = fvm
-        except Exception:
-            await fvm.teardown()
-            teardown_nftables_for_vm(self.vm_id)
-            await self.tap_interface.delete()
-            raise
-
-    async def start(self):
-        logger.debug(f"starting vm {self.vm_id}")
-        if not self.fvm:
-            raise ValueError("No VM found. Call setup() before start()")
-
-        fvm = self.fvm
-
-        if self.enable_console:
-            fvm.start_printing_logs()
-
-        await fvm.wait_for_init()
-        logger.debug(f"started fvm {self.vm_id}")
+        await super().setup(config)
 
     async def configure(self):
         """Configure the VM by sending configuration info to it's init"""
@@ -369,11 +231,7 @@ async def configure(self):
         input_data: bytes = load_file_content(self.resources.data_path) if \
             hasattr(self.resources, "data_path") else None
 
-        interface = (
-            Interface.asgi
-            if self.resources.code_entrypoint and ":" in self.resources.code_entrypoint
-            else Interface.executable
-        )
+        interface = Interface.asgi
 
         volumes: List[Volume]
         if self.resources.code_encoding == Encoding.squashfs:
@@ -406,6 +264,7 @@ async def configure(self):
                 for index, volume in enumerate(self.resources.volumes)
             ]
 
+        await super().configure(volumes, interface)
         reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
 
         # The ip and route should not contain the network mask in order to maintain
@@ -438,30 +297,6 @@ async def configure(self):
             logger.exception(response.traceback)
             raise VmSetupError(response.error)
 
-    async def start_guest_api(self):
-        logger.debug(f"starting guest API for {self.vm_id}")
-        vsock_path = f"{self.fvm.vsock_path}_53"
-        vm_hash = self.vm_hash
-        self.guest_api_process = Process(
-            target=run_guest_api, args=(vsock_path, vm_hash)
-        )
-        self.guest_api_process.start()
-        while not exists(vsock_path):
-            await asyncio.sleep(0.01)
-        subprocess.run(f"chown jailman:jailman {vsock_path}", shell=True, check=True)
-        logger.debug(f"started guest API for {self.vm_id}")
-
-    async def stop_guest_api(self):
-        if self.guest_api_process and self.guest_api_process._popen:
-            self.guest_api_process.terminate()
-
-    async def teardown(self):
-        if self.fvm:
-            await self.fvm.teardown()
-            teardown_nftables_for_vm(self.vm_id)
-            await self.tap_interface.delete()
-        await self.stop_guest_api()
-
     async def run_code(
         self,
         scope: Optional[dict] = None,

From 99c516828a6ad31605bd1cb255fc1b6238939c47 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 12 Jun 2023 23:20:02 +0200
Subject: [PATCH 393/990] Refactor: Interface only allowed programs on
 firecracker

Solution: Refactor in order to split generic methods shared by all firecracker virtual machines from program specific code.

This makes it possible to implement other types of execution on top of Firecracker such as Instances.

Co-authored-by: nesitor 
---
 vm_supervisor/__main__.py                  |  11 +-
 vm_supervisor/models.py                    |  46 +++--
 vm_supervisor/run.py                       |   2 +-
 vm_supervisor/vm/__init__.py               |  12 +-
 vm_supervisor/vm/firecracker_executable.py | 131 ++++---------
 vm_supervisor/vm/firecracker_program.py    | 211 ++++++++++++---------
 6 files changed, 211 insertions(+), 202 deletions(-)

diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index acf34ae24..ced8344db 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -16,13 +16,14 @@
 except ImportError:
     sentry_sdk = None
 
-from . import supervisor, metrics
-from .conf import settings, make_db_url
+import alembic.command
+import alembic.config
+
+from . import metrics, supervisor
+from .conf import make_db_url, settings
 from .models import VmHash
 from .pubsub import PubSub
-from .run import run_code_on_request, run_code_on_event
-import alembic.config
-import alembic.command
+from .run import run_code_on_event, run_code_on_request
 
 logger = logging.getLogger(__name__)
 
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 5fe3f763d..c37a7cf6f 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -5,22 +5,31 @@
 from asyncio import Task
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Dict, NewType, Optional
+from typing import Dict, Optional
 
-from aleph_message.models import ExecutableContent, InstanceContent
-from aleph_message.models.execution.base import MachineType
+from aleph_message.models import (
+    ExecutableContent,
+    InstanceContent,
+    ItemHash,
+    ProgramContent,
+)
 
 from .conf import settings
 from .metrics import ExecutionRecord, save_execution_data, save_record
 from .network.interfaces import TapInterface
 from .pubsub import PubSub
 from .utils import create_task_log_exceptions, dumps_for_json
-from .vm import AlephFirecrackerVM
-from .vm.firecracker_program import AlephFirecrackerResources
+from .vm import AlephFirecrackerExecutable
+from .vm.firecracker_program import (
+    AlephFirecrackerProgram,
+    AlephFirecrackerResources,
+    AlephProgramResources,
+)
 
 logger = logging.getLogger(__name__)
 
-VmHash = NewType("VmHash", str)
+VmHash = ItemHash
+
 
 @dataclass
 class VmExecutionTimes:
@@ -48,7 +57,7 @@ class VmExecution:
     original: ExecutableContent
     message: ExecutableContent
     resources: Optional[AlephFirecrackerResources] = None
-    vm: Optional[AlephFirecrackerVM] = None
+    vm: Optional[AlephFirecrackerExecutable] = None
 
     times: VmExecutionTimes
 
@@ -59,12 +68,19 @@ class VmExecution:
     update_task: Optional[asyncio.Task] = None
 
     persistent: bool = False
-    is_instance: bool = False
 
     @property
     def is_running(self):
         return self.times.starting_at and not self.times.stopping_at
 
+    @property
+    def is_program(self):
+        return isinstance(self.message, ProgramContent)
+
+    @property
+    def is_instance(self):
+        return isinstance(self.message, InstanceContent)
+
     @property
     def becomes_ready(self):
         return self.ready_event.wait
@@ -84,7 +100,6 @@ def __init__(
         self.ready_event = asyncio.Event()
         self.concurrent_runs = 0
         self.runs_done_event = asyncio.Event()
-        self.is_instance = isinstance(self.message, InstanceContent)
 
     def to_dict(self) -> Dict:
         return {
@@ -98,25 +113,30 @@ def to_json(self, indent: Optional[int] = None) -> str:
     async def prepare(self):
         """Download VM required files"""
         self.times.preparing_at = datetime.now()
-        resources = AlephFirecrackerResources(self.message, namespace=self.vm_hash)
+        if self.is_program:
+            resources = AlephProgramResources(self.message, namespace=self.vm_hash)
+        elif self.is_instance:
+            # resources = AlephInstanceResources(self.message, namespace=self.vm_hash)
+            pass  # TODO
+        else:
+            raise ValueError("Unknown executable message type")
         await resources.download_all()
         self.times.prepared_at = datetime.now()
         self.resources = resources
 
     async def create(
         self, vm_id: int, tap_interface: TapInterface
-    ) -> AlephFirecrackerVM:
+    ) -> AlephFirecrackerExecutable:
         if not self.resources:
             raise ValueError("Execution resources must be configured first")
         self.times.starting_at = datetime.now()
-        self.vm = vm = AlephFirecrackerVM(
+        self.vm = vm = AlephFirecrackerProgram(
             vm_id=vm_id,
             vm_hash=self.vm_hash,
             resources=self.resources,
             enable_networking=self.message.environment.internet,
             hardware_resources=self.message.resources,
             tap_interface=tap_interface,
-            is_instance=self.is_instance,
         )
         try:
             await vm.setup()
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 8aac6549c..a5021bf88 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -145,7 +145,7 @@ async def run_code_on_request(
         headers.update(
             {
                 "Aleph-Program-ItemHash": execution.vm_hash,
-                "Aleph-Program-Code-Ref": execution.message.code.ref if not execution.is_instance else None,
+                "Aleph-Program-Code-Ref": execution.message.code.ref
                 # "Aleph-Compute-Vm-Id": str(execution.vm.vm_id),
             }
         )
diff --git a/vm_supervisor/vm/__init__.py b/vm_supervisor/vm/__init__.py
index e263ec434..1210d172b 100644
--- a/vm_supervisor/vm/__init__.py
+++ b/vm_supervisor/vm/__init__.py
@@ -1,4 +1,10 @@
-from . import firecracker_program
-from .firecracker_program import AlephFirecrackerVM
+from . import firecracker_executable, firecracker_program
+from .firecracker_executable import AlephFirecrackerExecutable
+from .firecracker_program import AlephFirecrackerProgram
 
-__all__ = ("firecracker_program", "AlephFirecrackerVM")
+__all__ = (
+    "firecracker_executable",
+    "AlephFirecrackerExecutable",
+    "firecracker_program",
+    "AlephFirecrackerProgram",
+)
diff --git a/vm_supervisor/vm/firecracker_executable.py b/vm_supervisor/vm/firecracker_executable.py
index 13cc2cebe..61ce9e4df 100644
--- a/vm_supervisor/vm/firecracker_executable.py
+++ b/vm_supervisor/vm/firecracker_executable.py
@@ -1,15 +1,21 @@
+"""
+This module contains abstract class for executables (programs and instances) running inside Firecracker MicroVMs.
+"""
+
 import asyncio
 import dataclasses
 import logging
 import subprocess
 from dataclasses import dataclass, field
-from enum import Enum
 from multiprocessing import Process, set_start_method
 from os.path import exists, isfile
 from pathlib import Path
 from typing import Dict, List, Optional
 
 import msgpack
+from aleph_message.models import ItemHash
+
+from .firecracker_program import Interface
 
 try:
     import psutil as psutil
@@ -18,15 +24,8 @@
 from aiohttp import ClientResponseError
 from aleph_message.models.execution.environment import MachineResources
 
-from firecracker.config import (
-    BootSource,
-    Drive,
-    FirecrackerConfig,
-    MachineConfig,
-    NetworkInterface,
-    Vsock,
-)
-from firecracker.microvm import MicroVM, setfacl
+from firecracker.config import FirecrackerConfig
+from firecracker.microvm import MicroVM
 from guest_api.__main__ import run_guest_api
 
 from ..conf import settings
@@ -38,6 +37,7 @@
 logger = logging.getLogger(__name__)
 set_start_method("spawn")
 
+
 class ResourceDownloadError(ClientResponseError):
     """An error occurred while downloading a VM resource file"""
 
@@ -51,11 +51,6 @@ def __init__(self, error: ClientResponseError):
         )
 
 
-class Interface(str, Enum):
-    asgi = "asgi"
-    executable = "executable"
-
-
 @dataclass
 class Volume:
     mount: str
@@ -73,7 +68,7 @@ class HostVolume:
 @dataclass
 class VMConfiguration:
     interface: Interface
-    vm_hash: str
+    vm_hash: ItemHash
     ip: Optional[str] = None
     route: Optional[str] = None
     dns_servers: List[str] = field(default_factory=list)
@@ -92,6 +87,7 @@ class ConfigurationResponse:
 
 
 class AlephFirecrackerResources:
+    """Resources required to start a Firecracker VM"""
 
     message_content: ExecutableContent
 
@@ -143,24 +139,24 @@ class VmInitNotConnected(Exception):
     pass
 
 
-class AlephFirecrackerVM:
+class AlephFirecrackerExecutable:
     vm_id: int
-    vm_hash: str
+    vm_hash: ItemHash
     resources: AlephFirecrackerResources
     enable_console: bool
     enable_networking: bool
-    is_instance: bool
     hardware_resources: MachineResources
-    vm_configuration: VMConfiguration
-    fvm: Optional[MicroVM] = None
-    guest_api_process: Optional[Process] = None
     tap_interface: Optional[TapInterface] = None
     fvm: MicroVM
+    vm_configuration: Optional[VMConfiguration]
+    guest_api_process: Optional[Process] = None
+    is_instance: bool
+    _firecracker_config: Optional[FirecrackerConfig] = None
 
     def __init__(
         self,
         vm_id: int,
-        vm_hash: str,
+        vm_hash: ItemHash,
         resources: AlephFirecrackerResources,
         enable_networking: bool = False,
         enable_console: Optional[bool] = None,
@@ -170,10 +166,10 @@ def __init__(
         self.vm_id = vm_id
         self.vm_hash = vm_hash
         self.resources = resources
-        self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING
         if enable_console is None:
             enable_console = settings.PRINT_SYSTEM_LOGS
         self.enable_console = enable_console
+        self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING
         self.hardware_resources = hardware_resources
         self.tap_interface = tap_interface
 
@@ -186,8 +182,15 @@ def __init__(
         )
         self.fvm.prepare_jailer()
 
+        # These properties are set later in the setup and configuration.
+        self.vm_configuration = None
+        self.guest_api_process = None
+        self._firecracker_config = None
+
     def to_dict(self):
+        """Dict representation of the virtual machine. Used to record resource usage and for JSON serialization."""
         if self.fvm.proc and psutil:
+            # The firecracker process is still running and process information can be obtained from `psutil`.
             try:
                 p = psutil.Process(self.fvm.proc.pid)
                 pid_info = {
@@ -213,89 +216,33 @@ def to_dict(self):
             **self.__dict__,
         }
 
-    async def setup(self, config: FirecrackerConfig):
-        logger.debug("setup started")
-        await setfacl()
-
-        config = config or FirecrackerConfig(
-            boot_source=BootSource(
-                kernel_image_path=Path(
-                    self.fvm.enable_kernel(self.resources.kernel_image_path)
-                ),
-                boot_args=BootSource.args(enable_console=self.enable_console, writable=self.is_instance),
-            ),
-            drives=[
-                Drive(
-                    drive_id="rootfs",
-                    path_on_host=self.fvm.enable_rootfs(self.resources.rootfs_path),
-                    is_root_device=True,
-                    is_read_only=True,
-                ),
-            ]
-            + [
-                self.fvm.enable_drive(volume.path_on_host, read_only=volume.read_only)
-                for volume in self.resources.volumes
-            ],
-            machine_config=MachineConfig(
-                vcpu_count=self.hardware_resources.vcpus,
-                mem_size_mib=self.hardware_resources.memory,
-            ),
-            vsock=Vsock(),
-            network_interfaces=[
-                NetworkInterface(
-                    iface_id="eth0", host_dev_name=self.tap_interface.device_name
-                )
-            ]
-            if self.enable_networking
-            else [],
-        )
+    async def setup(self):
+        # self._firecracker_config = FirecrackerConfig(...)
+        raise NotImplementedError()
+
+    async def start(self):
+        logger.debug(f"Starting VM={self.vm_id}")
 
-        logger.debug(config.json(by_alias=True, exclude_none=True, indent=4))
+        if not self.fvm:
+            raise ValueError("No VM found. Call setup() before start()")
 
         try:
-            await self.fvm.start(config)
+            await self.fvm.start(self._firecracker_config)
             logger.debug("setup done")
         except Exception:
+            # Stop the VM and clear network interfaces in case any error prevented the start of the virtual machine.
             await self.fvm.teardown()
             teardown_nftables_for_vm(self.vm_id)
             await self.tap_interface.delete()
             raise
 
-    async def start(self):
-        logger.debug(f"starting vm {self.vm_id}")
-        if not self.fvm:
-            raise ValueError("No VM found. Call setup() before start()")
-
         if self.enable_console:
             self.fvm.start_printing_logs()
 
         logger.debug(f"started fvm {self.vm_id}")
 
-    async def configure(self, volumes: Optional[List[Volume]], interface: Optional[Interface]):
-        """Configure the VM by sending configuration info to it's init"""
-        volumes = volumes or [
-            Volume(
-                mount=volume.mount,
-                device=self.fvm.drives[index].drive_id,
-                read_only=volume.read_only,
-            )
-            for index, volume in enumerate(self.resources.volumes)
-        ]
-
-        # The ip and route should not contain the network mask in order to maintain
-        # compatibility with the existing runtimes.
-        ip = self.tap_interface.guest_ip.with_prefixlen.split("/", 1)[0]
-        route = str(self.tap_interface.host_ip).split("/", 1)[0]
-
-        self.vm_configuration = VMConfiguration(
-            ip=ip if self.enable_networking else None,
-            route=route if self.enable_networking else None,
-            dns_servers=settings.DNS_NAMESERVERS,
-            vm_hash=self.vm_hash,
-            volumes=volumes,
-            variables=self.resources.message_content.variables,
-            interface=interface,
-        )
+    async def configure(self):
+        raise NotImplementedError()
 
     async def start_guest_api(self):
         logger.debug(f"starting guest API for {self.vm_id}")
diff --git a/vm_supervisor/vm/firecracker_program.py b/vm_supervisor/vm/firecracker_program.py
index fbe832d0e..ddcd0a393 100644
--- a/vm_supervisor/vm/firecracker_program.py
+++ b/vm_supervisor/vm/firecracker_program.py
@@ -3,12 +3,13 @@
 import logging
 import os.path
 from dataclasses import dataclass, field
-from multiprocessing import Process, set_start_method
-from os.path import isfile
+from enum import Enum
+from multiprocessing import set_start_method
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Tuple
 
 import msgpack
+from aleph_message.models import ItemHash
 
 try:
     import psutil as psutil
@@ -26,27 +27,24 @@
     NetworkInterface,
     Vsock,
 )
-from firecracker.microvm import MicroVM, setfacl
+from firecracker.microvm import setfacl
 
-from .firecracker_microvm import AlephFirecrackerVM, AlephFirecrackerResources, VmSetupError, VmInitNotConnected, \
-    Interface, Volume
 from ..conf import settings
 from ..models import ExecutableContent
 from ..network.interfaces import TapInterface
 from ..storage import get_code_path, get_data_path, get_runtime_path
+from .firecracker_executable import (
+    AlephFirecrackerExecutable,
+    AlephFirecrackerResources,
+    VmInitNotConnected,
+    VmSetupError,
+    Volume,
+)
 
 logger = logging.getLogger(__name__)
 set_start_method("spawn")
 
 
-def load_file_content(path: Path) -> bytes:
-    if path:
-        with open(path, "rb") as fd:
-            return fd.read()
-    else:
-        return b""
-
-
 class FileTooLargeError(Exception):
     pass
 
@@ -64,8 +62,35 @@ def __init__(self, error: ClientResponseError):
         )
 
 
+def read_input_data(path_to_data: Path) -> Optional[bytes]:
+    if not path_to_data:
+        return None
+
+    if os.path.getsize(path_to_data) > settings.MAX_DATA_ARCHIVE_SIZE:
+        raise FileTooLargeError(f"Data file too large to pass as an inline zip")
+
+    return path_to_data.read_bytes()
+
+
+class Interface(str, Enum):
+    asgi = "asgi"
+    executable = "executable"
+
+    @classmethod
+    def from_entrypoint(cls, entrypoint: str):
+        """Determine the interface type (Python ASGI or executable HTTP service) from the entrypoint of the program."""
+        # Only Python ASGI entrypoints contain a column `:` in their name.
+        # We use this to differentiate Python ASGI programs from executable HTTP service mode.
+        if ":" in entrypoint:
+            return cls.asgi
+        else:
+            return cls.executable
+
+
 @dataclass
 class ConfigurationPayload:
+    """Configuration passed to the init of the virtual machine in order to start the program."""
+
     input_data: bytes
     interface: Interface
     vm_hash: str
@@ -84,6 +109,8 @@ def as_msgpack(self) -> bytes:
 
 @dataclass
 class ConfigurationResponse:
+    """Response received from the virtual machine in response to a request."""
+
     success: bool
     error: Optional[str] = None
     traceback: Optional[str] = None
@@ -91,13 +118,17 @@ class ConfigurationResponse:
 
 @dataclass
 class RunCodePayload:
+    """Information passed to the init of the virtual machine to launch a function/path of the program."""
+
     scope: Dict
 
     def as_msgpack(self) -> bytes:
         return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True)
 
 
-class AlephFunctionResources(AlephFirecrackerResources):
+class AlephProgramResources(AlephFirecrackerResources):
+    """Resources required by the virtual machine in order to launch the program.
+    Extends the resources required by all Firecracker VMs."""
 
     code_path: Path
     code_encoding: Encoding
@@ -106,13 +137,8 @@ class AlephFunctionResources(AlephFirecrackerResources):
 
     def __init__(self, message_content: ExecutableContent, namespace: str):
         super().__init__(message_content, namespace)
-        if hasattr(message_content, "code"):
-            self.code_encoding = message_content.code.encoding
-            self.code_entrypoint = message_content.code.entrypoint
-        else:
-            self.code_path = None
-            self.code_encoding = None
-            self.code_entrypoint = None
+        self.code_encoding = message_content.code.encoding
+        self.code_entrypoint = message_content.code.entrypoint
 
     async def download_code(self):
         code_ref: str = self.message_content.code.ref
@@ -120,7 +146,7 @@ async def download_code(self):
             self.code_path = await get_code_path(code_ref)
         except ClientResponseError as error:
             raise ResourceDownloadError(error)
-        assert isfile(self.code_path), f"Code not found on '{self.code_path}'"
+        assert self.code_path.is_file(), f"Code not found on '{self.code_path}'"
 
     async def download_runtime(self):
         runtime_ref: str = self.message_content.runtime.ref
@@ -128,7 +154,7 @@ async def download_runtime(self):
             self.rootfs_path = await get_runtime_path(runtime_ref)
         except ClientResponseError as error:
             raise ResourceDownloadError(error)
-        assert isfile(self.rootfs_path), f"Runtime not found on {self.rootfs_path}"
+        assert self.rootfs_path.is_file(), f"Runtime not found on {self.rootfs_path}"
 
     async def download_data(self):
         if self.message_content.data:
@@ -137,7 +163,7 @@ async def download_data(self):
                 self.data_path = await get_data_path(data_ref)
             except ClientResponseError as error:
                 raise ResourceDownloadError(error)
-            assert isfile(self.data_path)
+            assert self.data_path.is_file(), f"Data nout found on {self.data_path}"
 
     async def download_all(self):
         await asyncio.gather(
@@ -149,41 +175,73 @@ async def download_all(self):
         )
 
 
-class AlephFirecrackerFunction(AlephFirecrackerVM):
-    vm_id: int
-    vm_hash: str
-    resources: AlephFunctionResources
-    enable_console: bool
-    enable_networking: bool
-    is_instance: bool
-    hardware_resources: MachineResources
-    fvm: Optional[MicroVM] = None
-    guest_api_process: Optional[Process] = None
-    tap_interface: Optional[TapInterface] = None
+def get_volumes_for_program(
+    resources: AlephProgramResources, drives: List[Drive]
+) -> Tuple[Optional[bytes], List[Volume]]:
+    if resources.code_encoding == Encoding.squashfs:
+        code = b""
+        volumes = [Volume(mount="/opt/code", device="vdb", read_only=True)] + [
+            Volume(
+                mount=volume.mount,
+                device=drives[index + 1].drive_id,
+                read_only=volume.read_only,
+            )
+            for index, volume in enumerate(resources.volumes)
+        ]
+    else:
+        if os.path.getsize(resources.code_path) > settings.MAX_PROGRAM_ARCHIVE_SIZE:
+            raise FileTooLargeError(f"Program file too large to pass as an inline zip")
+
+        code: Optional[bytes] = (
+            resources.code_path.read_bytes() if resources.code_path else None
+        )
+        volumes = [
+            Volume(
+                mount=volume.mount,
+                device=drives[index].drive_id,
+                read_only=volume.read_only,
+            )
+            for index, volume in enumerate(resources.volumes)
+        ]
+    return code, volumes
+
+
+class AlephFirecrackerProgram(AlephFirecrackerExecutable):
+    resources: AlephProgramResources
+    is_instance = False
 
     def __init__(
         self,
         vm_id: int,
-        vm_hash: str,
-        resources: AlephFunctionResources,
+        vm_hash: ItemHash,
+        resources: AlephProgramResources,
         enable_networking: bool = False,
         enable_console: Optional[bool] = None,
         hardware_resources: MachineResources = MachineResources(),
-        tap_interface: Optional[TapInterface] = None
+        tap_interface: Optional[TapInterface] = None,
     ):
-        super().__init__(vm_id, vm_hash, resources, enable_networking, enable_console, hardware_resources, tap_interface)
-        self.is_instance = False
+        super().__init__(
+            vm_id,
+            vm_hash,
+            resources,
+            enable_networking,
+            enable_console,
+            hardware_resources,
+            tap_interface,
+        )
 
     async def setup(self):
-        logger.debug("setup started")
+        logger.debug(f"Setup started for VM={self.vm_id}")
         await setfacl()
 
-        config = FirecrackerConfig(
+        self._firecracker_config = FirecrackerConfig(
             boot_source=BootSource(
                 kernel_image_path=Path(
                     self.fvm.enable_kernel(self.resources.kernel_image_path)
                 ),
-                boot_args=BootSource.args(enable_console=self.enable_console, writable=self.is_instance),
+                boot_args=BootSource.args(
+                    enable_console=self.enable_console, writable=False
+                ),
             ),
             drives=[
                 Drive(
@@ -195,7 +253,8 @@ async def setup(self):
             ]
             + (
                 [self.fvm.enable_drive(self.resources.code_path)]
-                if hasattr(self.resources, "code_encoding") and self.resources.code_encoding == Encoding.squashfs
+                if hasattr(self.resources, "code_encoding")
+                and self.resources.code_encoding == Encoding.squashfs
                 else []
             )
             + [
@@ -216,55 +275,31 @@ async def setup(self):
             else [],
         )
 
-        await super().setup(config)
-
     async def configure(self):
         """Configure the VM by sending configuration info to it's init"""
 
-        if (
-            hasattr(self.resources, "data_path") and self.resources.data_path
-            and os.path.getsize(self.resources.data_path)
-            > settings.MAX_DATA_ARCHIVE_SIZE
-        ):
-            raise FileTooLargeError(f"Data file too large to pass as an inline zip")
-
-        input_data: bytes = load_file_content(self.resources.data_path) if \
-            hasattr(self.resources, "data_path") else None
-
-        interface = Interface.asgi
-
+        code: Optional[bytes]
         volumes: List[Volume]
-        if self.resources.code_encoding == Encoding.squashfs:
-            code = b""
-            volumes = [Volume(mount="/opt/code", device="vdb", read_only=True)] + [
-                Volume(
-                    mount=volume.mount,
-                    device=self.fvm.drives[index + 1].drive_id,
-                    read_only=volume.read_only,
-                )
-                for index, volume in enumerate(self.resources.volumes)
-            ]
-        else:
-            if (
-                hasattr(self.resources, "data_path") and self.resources.data_path
-                and os.path.getsize(self.resources.code_path)
-                > settings.MAX_PROGRAM_ARCHIVE_SIZE
-            ):
-                raise FileTooLargeError(
-                    f"Program file too large to pass as an inline zip"
-                )
 
-            code: Optional[bytes] = load_file_content(self.resources.code_path) if self.resources.code_path else None
-            volumes = [
-                Volume(
-                    mount=volume.mount,
-                    device=self.fvm.drives[index].drive_id,
-                    read_only=volume.read_only,
-                )
-                for index, volume in enumerate(self.resources.volumes)
-            ]
+        code, volumes = get_volumes_for_program(
+            resources=self.resources, drives=self.fvm.drives
+        )
+        interface: Interface = Interface.from_entrypoint(self.resources.code_entrypoint)
+        input_data: Optional[bytes] = read_input_data(self.resources.data_path)
+
+        self._setup_configuration(
+            code=code, input_data=input_data, interface=interface, volumes=volumes
+        )
 
-        await super().configure(volumes, interface)
+    def _setup_configuration(
+        self,
+        code: Optional[bytes],
+        input_data: Optional[bytes],
+        interface: Interface,
+        volumes: List[Volume],
+    ):
+        """Set up the VM configuration. The program mode uses a VSOCK connection to the custom init of the virtual
+        machine to send this configuration. Other modes may use Cloud-init, ..."""
         reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
 
         # The ip and route should not contain the network mask in order to maintain

From 413f3ebbe47fefaaa04d5af371baef453285aa78 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 12 Jun 2023 23:20:41 +0200
Subject: [PATCH 394/990] Cleanup: `black` and `isort` were not applied

---
 vm_supervisor/messages.py                     | 14 +++--
 vm_supervisor/migrations/env.py               | 15 +++---
 .../0001_bbb12a12372e_execution_records.py    |  2 +-
 vm_supervisor/network/interfaces.py           |  2 +-
 vm_supervisor/pool.py                         |  2 +-
 vm_supervisor/reactor.py                      |  2 +-
 vm_supervisor/storage.py                      | 53 ++++++++++---------
 vm_supervisor/tasks.py                        |  2 +-
 8 files changed, 52 insertions(+), 40 deletions(-)

diff --git a/vm_supervisor/messages.py b/vm_supervisor/messages.py
index 505427a16..3df538767 100644
--- a/vm_supervisor/messages.py
+++ b/vm_supervisor/messages.py
@@ -56,17 +56,25 @@ async def update_message(message: ExecutableMessage):
             update_with_latest_ref(message.content.runtime),
             update_with_latest_ref(message.content.code),
             update_with_latest_ref(message.content.data),
-            *(update_with_latest_ref(volume) for volume in (message.content.volumes or [])),
+            *(
+                update_with_latest_ref(volume)
+                for volume in (message.content.volumes or [])
+            ),
         )
     else:
         assert message.type == MessageType.instance
         await asyncio.gather(
             update_with_latest_ref(message.content.rootfs.parent),
-            *(update_with_latest_ref(volume) for volume in (message.content.volumes or [])),
+            *(
+                update_with_latest_ref(volume)
+                for volume in (message.content.volumes or [])
+            ),
         )
 
 
-async def load_updated_message(ref: VmHash) -> Tuple[ExecutableMessage, ExecutableMessage]:
+async def load_updated_message(
+    ref: VmHash,
+) -> Tuple[ExecutableMessage, ExecutableMessage]:
     original_message = await try_get_message(ref)
     message = copy.deepcopy(original_message)
     await update_message(message)
diff --git a/vm_supervisor/migrations/env.py b/vm_supervisor/migrations/env.py
index 9db9eb318..71577f27e 100644
--- a/vm_supervisor/migrations/env.py
+++ b/vm_supervisor/migrations/env.py
@@ -1,9 +1,11 @@
-from logging.config import fileConfig
-
 from alembic import context
-from vm_supervisor.conf import make_db_url
 from sqlalchemy import create_engine
 
+from vm_supervisor.conf import make_db_url
+
+# Auto-generate migrations
+from vm_supervisor.metrics import Base
+
 # # this is the Alembic Config object, which provides
 # # access to the values within the .ini file in use.
 # config = context.config
@@ -13,8 +15,7 @@
 # if config.config_file_name is not None:
 #     fileConfig(config.config_file_name)
 
-# Auto-generate migrations
-from vm_supervisor.metrics import Base
+
 target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,
@@ -56,9 +57,7 @@ def run_migrations_online() -> None:
     """
     connectable = create_engine(make_db_url())
     with connectable.connect() as connection:
-        context.configure(
-            connection=connection, target_metadata=target_metadata
-        )
+        context.configure(connection=connection, target_metadata=target_metadata)
 
         with context.begin_transaction():
             context.run_migrations()
diff --git a/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py b/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py
index b210e8cf9..ddb1e55ff 100644
--- a/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py
+++ b/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py
@@ -1,7 +1,7 @@
 """execution records
 
 Revision ID: bbb12a12372e
-Revises: 
+Revises:
 Create Date: 2022-09-28 18:52:16.431200
 
 """
diff --git a/vm_supervisor/network/interfaces.py b/vm_supervisor/network/interfaces.py
index a121c6356..b520e1008 100644
--- a/vm_supervisor/network/interfaces.py
+++ b/vm_supervisor/network/interfaces.py
@@ -1,10 +1,10 @@
 import asyncio
 import logging
+import shutil
 from ipaddress import IPv4Interface
 from subprocess import run
 
 from .ipaddresses import IPv4NetworkWithInterfaces
-import shutil
 
 logger = logging.getLogger(__name__)
 
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index e7ad57a47..2772cb308 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -7,7 +7,7 @@
 from vm_supervisor.network.hostnetwork import Network
 
 from .conf import settings
-from .models import VmExecution, VmHash, ExecutableContent
+from .models import ExecutableContent, VmExecution, VmHash
 
 logger = logging.getLogger(__name__)
 
diff --git a/vm_supervisor/reactor.py b/vm_supervisor/reactor.py
index 217bc2f2d..b977c4ae4 100644
--- a/vm_supervisor/reactor.py
+++ b/vm_supervisor/reactor.py
@@ -1,7 +1,7 @@
 import logging
 from typing import Coroutine, List
 
-from aleph_message.models import ProgramMessage, AlephMessage
+from aleph_message.models import AlephMessage
 from aleph_message.models.execution.environment import Subscription
 
 from .pubsub import PubSub
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index b1defe02c..779819294 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -10,24 +10,28 @@
 import logging
 import os
 import re
-import sys
 import subprocess
+import sys
 from os.path import isfile, join
 from pathlib import Path
 from shutil import make_archive
 from typing import Union
 
 import aiohttp
-from aleph_message.models import ExecutableMessage, InstanceMessage, ProgramMessage, MessageType
+from aleph_message.models import (
+    ExecutableMessage,
+    InstanceMessage,
+    MessageType,
+    ProgramMessage,
+)
+from aleph_message.models.execution.instance import RootfsVolume
+from aleph_message.models.execution.program import Encoding
 from aleph_message.models.execution.volume import (
     ImmutableVolume,
     MachineVolume,
-    ImmutableVolume,
     PersistentVolume,
     VolumePersistence,
 )
-from aleph_message.models.execution.program import Encoding
-from aleph_message.models.execution.instance import RootfsVolume
 
 from .conf import settings
 
@@ -166,7 +170,9 @@ def create_ext4(path: Path, size_mib: int) -> bool:
     return True
 
 
-async def create_volume_file(volume: Union[PersistentVolume, RootfsVolume], namespace: str) -> Path:
+async def create_volume_file(
+    volume: Union[PersistentVolume, RootfsVolume], namespace: str
+) -> Path:
     volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
     path = Path(settings.PERSISTENT_VOLUMES_DIR) / namespace / f"{volume_name}.ext4"
     if not path.is_file():
@@ -178,19 +184,13 @@ async def create_volume_file(volume: Union[PersistentVolume, RootfsVolume], name
 
 
 async def create_loopback_device(path: Path, read_only: bool = False) -> str:
-    command_args = [
-        "losetup",
-        "--find",
-        "--show"
-    ]
+    command_args = ["losetup", "--find", "--show"]
     if read_only:
         command_args.append("--read-only")
     command_args.append(str(path))
     loop_device = subprocess.run(
-        command_args,
-        check=True,
-        capture_output=True,
-        encoding="UTF-8").stdout.strip()
+        command_args, check=True, capture_output=True, encoding="UTF-8"
+    ).stdout.strip()
     return loop_device
 
 
@@ -199,18 +199,21 @@ def get_block_size(device_path: Path) -> str:
         ["blockdev", "--getsz", device_path],
         check=True,
         capture_output=True,
-        encoding="UTF-8").stdout.strip()
+        encoding="UTF-8",
+    ).stdout.strip()
     return block_size
 
 
 def create_mapped_device(device_name: str, table_command: str) -> None:
-    subprocess.run(f"dmsetup create {device_name}",
-                   input=table_command,
-                   text=True,
-                   shell=True,
-                   check=True,
-                   stdout=subprocess.PIPE,
-                   stderr=subprocess.PIPE)
+    subprocess.run(
+        f"dmsetup create {device_name}",
+        input=table_command,
+        text=True,
+        shell=True,
+        check=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
 
 
 def e2fs_check_and_resize(device_path: Path) -> None:
@@ -218,7 +221,9 @@ def e2fs_check_and_resize(device_path: Path) -> None:
     os.system(f"resize2fs {device_path}")
 
 
-async def create_devmapper(volume: Union[PersistentVolume, RootfsVolume], namespace: str) -> Path:
+async def create_devmapper(
+    volume: Union[PersistentVolume, RootfsVolume], namespace: str
+) -> Path:
     """It creates a /dev/mapper/DEVICE inside the VM, that is an extended mapped device of the volume specified.
     We follow the steps described here: https://community.aleph.im/t/deploying-mutable-vm-instances-on-aleph/56/2"""
     volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py
index 42b29f8ec..32ac0f673 100644
--- a/vm_supervisor/tasks.py
+++ b/vm_supervisor/tasks.py
@@ -8,9 +8,9 @@
 import aiohttp
 import pydantic
 from aiohttp import web
+from aleph_message.models import AlephMessage, ProgramMessage, parse_message
 from yarl import URL
 
-from aleph_message.models import AlephMessage, ProgramMessage, InstanceMessage, parse_message
 from .conf import settings
 from .messages import load_updated_message
 from .models import VmHash

From 1407ea4d70f446844125b18dfbffef7e4687cf45 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 13 Jun 2023 12:08:58 +0200
Subject: [PATCH 395/990] Feature: Instances were not supported

Solution:
1. Add support for running Instances with Firecracker.
2. Refactor all firecracker virtual machine code into a subpackage `vm_supervisor.vm.firecracker` to avoid namespace in filenames and in preparation for Qemu support.

Co-authored-by: nesitor 
---
 vm_supervisor/models.py                       |   4 +-
 vm_supervisor/vm/__init__.py                  |  10 +-
 vm_supervisor/vm/firecracker/__init__.py      |   4 +
 .../executable.py}                            |  25 ++--
 vm_supervisor/vm/firecracker/instance.py      | 113 ++++++++++++++++++
 .../program.py}                               |  40 ++++---
 6 files changed, 156 insertions(+), 40 deletions(-)
 create mode 100644 vm_supervisor/vm/firecracker/__init__.py
 rename vm_supervisor/vm/{firecracker_executable.py => firecracker/executable.py} (93%)
 create mode 100644 vm_supervisor/vm/firecracker/instance.py
 rename vm_supervisor/vm/{firecracker_program.py => firecracker/program.py} (93%)

diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index c37a7cf6f..cde768966 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -19,8 +19,8 @@
 from .network.interfaces import TapInterface
 from .pubsub import PubSub
 from .utils import create_task_log_exceptions, dumps_for_json
-from .vm import AlephFirecrackerExecutable
-from .vm.firecracker_program import (
+from .vm.firecracker.executable import AlephFirecrackerExecutable
+from .vm.firecracker.program import (
     AlephFirecrackerProgram,
     AlephFirecrackerResources,
     AlephProgramResources,
diff --git a/vm_supervisor/vm/__init__.py b/vm_supervisor/vm/__init__.py
index 1210d172b..4b2995d53 100644
--- a/vm_supervisor/vm/__init__.py
+++ b/vm_supervisor/vm/__init__.py
@@ -1,10 +1,8 @@
-from . import firecracker_executable, firecracker_program
-from .firecracker_executable import AlephFirecrackerExecutable
-from .firecracker_program import AlephFirecrackerProgram
+from .firecracker import AlephFirecrackerProgram
+from .firecracker import AlephFirecrackerInstance
+
 
 __all__ = (
-    "firecracker_executable",
-    "AlephFirecrackerExecutable",
-    "firecracker_program",
     "AlephFirecrackerProgram",
+    "AlephFirecrackerInstance",
 )
diff --git a/vm_supervisor/vm/firecracker/__init__.py b/vm_supervisor/vm/firecracker/__init__.py
new file mode 100644
index 000000000..a28769581
--- /dev/null
+++ b/vm_supervisor/vm/firecracker/__init__.py
@@ -0,0 +1,4 @@
+from .instance import AlephFirecrackerInstance
+from .program import AlephFirecrackerProgram
+
+__all__ = ("AlephFirecrackerProgram", "AlephFirecrackerInstance")
diff --git a/vm_supervisor/vm/firecracker_executable.py b/vm_supervisor/vm/firecracker/executable.py
similarity index 93%
rename from vm_supervisor/vm/firecracker_executable.py
rename to vm_supervisor/vm/firecracker/executable.py
index 61ce9e4df..d1ab6d135 100644
--- a/vm_supervisor/vm/firecracker_executable.py
+++ b/vm_supervisor/vm/firecracker/executable.py
@@ -3,20 +3,17 @@
 """
 
 import asyncio
-import dataclasses
 import logging
 import subprocess
 from dataclasses import dataclass, field
 from multiprocessing import Process, set_start_method
 from os.path import exists, isfile
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Any
 
-import msgpack
 from aleph_message.models import ItemHash
 
-from .firecracker_program import Interface
-
+psutil: Optional[Any]
 try:
     import psutil as psutil
 except ImportError:
@@ -28,11 +25,11 @@
 from firecracker.microvm import MicroVM
 from guest_api.__main__ import run_guest_api
 
-from ..conf import settings
-from ..models import ExecutableContent
-from ..network.firewall import teardown_nftables_for_vm
-from ..network.interfaces import TapInterface
-from ..storage import get_volume_path
+from vm_supervisor.conf import settings
+from vm_supervisor.models import ExecutableContent
+from vm_supervisor.network.firewall import teardown_nftables_for_vm
+from vm_supervisor.network.interfaces import TapInterface
+from vm_supervisor.storage import get_volume_path
 
 logger = logging.getLogger(__name__)
 set_start_method("spawn")
@@ -66,8 +63,7 @@ class HostVolume:
 
 
 @dataclass
-class VMConfiguration:
-    interface: Interface
+class VmConfiguration:
     vm_hash: ItemHash
     ip: Optional[str] = None
     route: Optional[str] = None
@@ -75,9 +71,6 @@ class VMConfiguration:
     volumes: List[Volume] = field(default_factory=list)
     variables: Optional[Dict[str, str]] = None
 
-    def as_msgpack(self) -> bytes:
-        return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True)
-
 
 @dataclass
 class ConfigurationResponse:
@@ -148,7 +141,7 @@ class AlephFirecrackerExecutable:
     hardware_resources: MachineResources
     tap_interface: Optional[TapInterface] = None
     fvm: MicroVM
-    vm_configuration: Optional[VMConfiguration]
+    vm_configuration: Optional[VmConfiguration]
     guest_api_process: Optional[Process] = None
     is_instance: bool
     _firecracker_config: Optional[FirecrackerConfig] = None
diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
new file mode 100644
index 000000000..33d3577d3
--- /dev/null
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -0,0 +1,113 @@
+import asyncio
+import logging
+from multiprocessing import set_start_method
+from pathlib import Path
+from typing import Optional
+
+from aleph_message.models import ItemHash
+from aleph_message.models.execution.environment import MachineResources
+
+from firecracker.config import (
+    BootSource,
+    Drive,
+    FirecrackerConfig,
+    MachineConfig,
+    NetworkInterface,
+    Vsock,
+)
+from firecracker.microvm import setfacl
+from vm_supervisor.network.interfaces import TapInterface
+from vm_supervisor.storage import create_devmapper
+from .executable import (
+    AlephFirecrackerExecutable,
+    AlephFirecrackerResources,
+)
+
+logger = logging.getLogger(__name__)
+set_start_method("spawn")
+
+
+class AlephInstanceResources(AlephFirecrackerResources):
+    async def download_runtime(self):
+        self.rootfs_path = await create_devmapper(
+            self.message_content.rootfs, self.namespace
+        )
+        assert (
+            self.rootfs_path.is_block_device()
+        ), f"Runtime not found on {self.rootfs_path}"
+
+    async def download_all(self):
+        await asyncio.gather(
+            self.download_kernel(),
+            self.download_runtime(),
+            self.download_volumes(),
+        )
+
+
+class AlephFirecrackerInstance(AlephFirecrackerExecutable):
+    resources: AlephInstanceResources
+    is_instance = True
+
+    def __init__(
+        self,
+        vm_id: int,
+        vm_hash: ItemHash,
+        resources: AlephInstanceResources,
+        enable_networking: bool = False,
+        enable_console: Optional[bool] = None,
+        hardware_resources: MachineResources = MachineResources(),
+        tap_interface: Optional[TapInterface] = None,
+    ):
+        super().__init__(
+            vm_id,
+            vm_hash,
+            resources,
+            enable_networking,
+            enable_console,
+            hardware_resources,
+            tap_interface,
+        )
+
+    async def setup(self):
+        logger.debug("instance setup started")
+        await setfacl()
+
+        self._firecracker_config = FirecrackerConfig(
+            boot_source=BootSource(
+                kernel_image_path=Path(
+                    self.fvm.enable_kernel(self.resources.kernel_image_path)
+                ),
+                boot_args=BootSource.args(
+                    enable_console=self.enable_console, writable=True
+                ),
+            ),
+            drives=[
+                Drive(
+                    drive_id="rootfs",
+                    path_on_host=self.fvm.enable_rootfs(self.resources.rootfs_path),
+                    is_root_device=True,
+                    is_read_only=False,
+                ),
+            ]
+            + [
+                self.fvm.enable_drive(volume.path_on_host, read_only=volume.read_only)
+                for volume in self.resources.volumes
+            ],
+            machine_config=MachineConfig(
+                vcpu_count=self.hardware_resources.vcpus,
+                mem_size_mib=self.hardware_resources.memory,
+            ),
+            vsock=Vsock(),
+            network_interfaces=[
+                NetworkInterface(
+                    iface_id="eth0", host_dev_name=self.tap_interface.device_name
+                )
+            ]
+            if self.enable_networking
+            else [],
+        )
+
+    async def configure(self):
+        """Configure the VM by sending configuration info to it's init"""
+        # TODO: Implement Cloud-init interface
+        raise NotImplementedError()
diff --git a/vm_supervisor/vm/firecracker_program.py b/vm_supervisor/vm/firecracker/program.py
similarity index 93%
rename from vm_supervisor/vm/firecracker_program.py
rename to vm_supervisor/vm/firecracker/program.py
index ddcd0a393..4e2beafbb 100644
--- a/vm_supervisor/vm/firecracker_program.py
+++ b/vm_supervisor/vm/firecracker/program.py
@@ -9,13 +9,8 @@
 from typing import Dict, List, Optional, Tuple
 
 import msgpack
-from aleph_message.models import ItemHash
-
-try:
-    import psutil as psutil
-except ImportError:
-    psutil = None
 from aiohttp import ClientResponseError
+from aleph_message.models import ItemHash
 from aleph_message.models.execution.base import Encoding
 from aleph_message.models.execution.environment import MachineResources
 
@@ -28,18 +23,16 @@
     Vsock,
 )
 from firecracker.microvm import setfacl
-
-from ..conf import settings
-from ..models import ExecutableContent
-from ..network.interfaces import TapInterface
-from ..storage import get_code_path, get_data_path, get_runtime_path
-from .firecracker_executable import (
+from vm_supervisor.conf import settings
+from vm_supervisor.models import ExecutableContent
+from vm_supervisor.network.interfaces import TapInterface
+from vm_supervisor.storage import get_code_path, get_data_path, get_runtime_path
+from .executable import (
     AlephFirecrackerExecutable,
     AlephFirecrackerResources,
     VmInitNotConnected,
     VmSetupError,
-    Volume,
-)
+    Volume, )
 
 logger = logging.getLogger(__name__)
 set_start_method("spawn")
@@ -87,6 +80,20 @@ def from_entrypoint(cls, entrypoint: str):
             return cls.executable
 
 
+@dataclass
+class ProgramVmConfiguration:
+    interface: Interface
+    vm_hash: ItemHash
+    ip: Optional[str] = None
+    route: Optional[str] = None
+    dns_servers: List[str] = field(default_factory=list)
+    volumes: List[Volume] = field(default_factory=list)
+    variables: Optional[Dict[str, str]] = None
+
+    def as_msgpack(self) -> bytes:
+        return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True)
+
+
 @dataclass
 class ConfigurationPayload:
     """Configuration passed to the init of the virtual machine in order to start the program."""
@@ -207,6 +214,7 @@ def get_volumes_for_program(
 
 
 class AlephFirecrackerProgram(AlephFirecrackerExecutable):
+    vm_configuration: Optional[ProgramVmConfiguration]
     resources: AlephProgramResources
     is_instance = False
 
@@ -287,11 +295,11 @@ async def configure(self):
         interface: Interface = Interface.from_entrypoint(self.resources.code_entrypoint)
         input_data: Optional[bytes] = read_input_data(self.resources.data_path)
 
-        self._setup_configuration(
+        await self._setup_configuration(
             code=code, input_data=input_data, interface=interface, volumes=volumes
         )
 
-    def _setup_configuration(
+    async def _setup_configuration(
         self,
         code: Optional[bytes],
         input_data: Optional[bytes],

From ed2d04a3ccb6a50fd078c362741ecbbb7634e009 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 13 Jun 2023 12:29:06 +0200
Subject: [PATCH 396/990] Fix: Many typing and consistency errors

---
 firecracker/microvm.py                     |  4 +-
 vm_supervisor/__main__.py                  |  4 +-
 vm_supervisor/conf.py                      |  6 +--
 vm_supervisor/messages.py                  |  5 +--
 vm_supervisor/models.py                    | 51 ++++++++++++++-------
 vm_supervisor/network/ipaddresses.py       |  4 +-
 vm_supervisor/pool.py                      | 18 +++++---
 vm_supervisor/resources.py                 |  9 ++--
 vm_supervisor/run.py                       | 15 ++++---
 vm_supervisor/tasks.py                     |  5 +--
 vm_supervisor/views.py                     | 14 +++---
 vm_supervisor/vm/__init__.py               |  4 +-
 vm_supervisor/vm/firecracker/executable.py |  9 +++-
 vm_supervisor/vm/firecracker/instance.py   | 15 ++++---
 vm_supervisor/vm/firecracker/program.py    | 52 +++++++++++++---------
 15 files changed, 126 insertions(+), 89 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index b6001dafc..5b1a5e0c4 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -206,11 +206,11 @@ async def start_jailed_firecracker(
         logger.debug(
             " ".join(
                 (
-                    self.jailer_bin_path,
+                    str(self.jailer_bin_path),
                     "--id",
                     str(self.vm_id),
                     "--exec-file",
-                    self.firecracker_bin_path,
+                    str(self.firecracker_bin_path),
                     "--uid",
                     uid,
                     "--gid",
diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index ced8344db..8858c5f9f 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -18,10 +18,10 @@
 
 import alembic.command
 import alembic.config
+from aleph_message.models import ItemHash
 
 from . import metrics, supervisor
 from .conf import make_db_url, settings
-from .models import VmHash
 from .pubsub import PubSub
 from .run import run_code_on_event, run_code_on_request
 
@@ -132,7 +132,7 @@ async def benchmark(runs: int):
     engine = metrics.setup_engine()
     metrics.create_tables(engine)
 
-    ref = VmHash("fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-hash")
+    ref = ItemHash("fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-hash")
     settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM
 
     FakeRequest: Request
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 7022510ba..0f5b15bfc 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -95,9 +95,9 @@ class Settings(BaseSettings):
     DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf
     DNS_NAMESERVERS: Optional[List[str]] = None
 
-    FIRECRACKER_PATH = "/opt/firecracker/firecracker"
-    JAILER_PATH = "/opt/firecracker/jailer"
-    LINUX_PATH = "/opt/firecracker/vmlinux.bin"
+    FIRECRACKER_PATH = Path("/opt/firecracker/firecracker")
+    JAILER_PATH = Path("/opt/firecracker/jailer")
+    LINUX_PATH = Path("/opt/firecracker/vmlinux.bin")
     INIT_TIMEOUT: float = 20.0
 
     CONNECTOR_URL = Url("http://localhost:4021")
diff --git a/vm_supervisor/messages.py b/vm_supervisor/messages.py
index 3df538767..c5167a1d0 100644
--- a/vm_supervisor/messages.py
+++ b/vm_supervisor/messages.py
@@ -4,9 +4,8 @@
 
 from aiohttp import ClientConnectorError, ClientResponseError
 from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable
-from aleph_message.models import ExecutableMessage, MessageType
+from aleph_message.models import ExecutableMessage, ItemHash, MessageType
 
-from .models import VmHash
 from .storage import get_latest_amend, get_message
 
 
@@ -73,7 +72,7 @@ async def update_message(message: ExecutableMessage):
 
 
 async def load_updated_message(
-    ref: VmHash,
+    ref: ItemHash,
 ) -> Tuple[ExecutableMessage, ExecutableMessage]:
     original_message = await try_get_message(ref)
     message = copy.deepcopy(original_message)
diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index cde768966..3e2d9541a 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -5,7 +5,7 @@
 from asyncio import Task
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
 
 from aleph_message.models import (
     ExecutableContent,
@@ -19,7 +19,9 @@
 from .network.interfaces import TapInterface
 from .pubsub import PubSub
 from .utils import create_task_log_exceptions, dumps_for_json
+from .vm import AlephFirecrackerInstance
 from .vm.firecracker.executable import AlephFirecrackerExecutable
+from .vm.firecracker.instance import AlephInstanceResources
 from .vm.firecracker.program import (
     AlephFirecrackerProgram,
     AlephFirecrackerResources,
@@ -28,8 +30,6 @@
 
 logger = logging.getLogger(__name__)
 
-VmHash = ItemHash
-
 
 @dataclass
 class VmExecutionTimes:
@@ -53,7 +53,7 @@ class VmExecution:
     """
 
     uuid: uuid.UUID  # Unique identifier of this execution
-    vm_hash: VmHash
+    vm_hash: ItemHash
     original: ExecutableContent
     message: ExecutableContent
     resources: Optional[AlephFirecrackerResources] = None
@@ -90,7 +90,7 @@ def vm_id(self) -> Optional[int]:
         return self.vm.vm_id if self.vm else None
 
     def __init__(
-        self, vm_hash: VmHash, message: ExecutableContent, original: ExecutableContent
+        self, vm_hash: ItemHash, message: ExecutableContent, original: ExecutableContent
     ):
         self.uuid = uuid.uuid1()  # uuid1() includes the hardware address and timestamp
         self.vm_hash = vm_hash
@@ -116,8 +116,7 @@ async def prepare(self):
         if self.is_program:
             resources = AlephProgramResources(self.message, namespace=self.vm_hash)
         elif self.is_instance:
-            # resources = AlephInstanceResources(self.message, namespace=self.vm_hash)
-            pass  # TODO
+            resources = AlephInstanceResources(self.message, namespace=self.vm_hash)
         else:
             raise ValueError("Unknown executable message type")
         await resources.download_all()
@@ -125,19 +124,34 @@ async def prepare(self):
         self.resources = resources
 
     async def create(
-        self, vm_id: int, tap_interface: TapInterface
+        self, vm_id: int, tap_interface: Optional[TapInterface] = None
     ) -> AlephFirecrackerExecutable:
         if not self.resources:
             raise ValueError("Execution resources must be configured first")
         self.times.starting_at = datetime.now()
-        self.vm = vm = AlephFirecrackerProgram(
-            vm_id=vm_id,
-            vm_hash=self.vm_hash,
-            resources=self.resources,
-            enable_networking=self.message.environment.internet,
-            hardware_resources=self.message.resources,
-            tap_interface=tap_interface,
-        )
+
+        vm: Union[AlephFirecrackerProgram, AlephFirecrackerInstance]
+        if self.is_program:
+            assert isinstance(self.resources, AlephProgramResources)
+            self.vm = vm = AlephFirecrackerProgram(
+                vm_id=vm_id,
+                vm_hash=self.vm_hash,
+                resources=self.resources,
+                enable_networking=self.message.environment.internet,
+                hardware_resources=self.message.resources,
+                tap_interface=tap_interface,
+            )
+        else:
+            assert self.is_instance
+            assert isinstance(self.resources, AlephInstanceResources)
+            self.vm = vm = AlephFirecrackerInstance(
+                vm_id=vm_id,
+                vm_hash=self.vm_hash,
+                resources=self.resources,
+                enable_networking=self.message.environment.internet,
+                hardware_resources=self.message.resources,
+                tap_interface=tap_interface,
+            )
         try:
             await vm.setup()
             await vm.start()
@@ -293,6 +307,11 @@ async def record_usage(self):
     async def run_code(self, scope: Optional[dict] = None) -> bytes:
         if not self.vm:
             raise ValueError("The VM has not been created yet")
+
+        if not self.is_program:
+            raise ValueError("Code can ony be run on programs")
+        assert isinstance(self.vm, AlephFirecrackerProgram)
+
         self.concurrent_runs += 1
         self.runs_done_event.clear()
         try:
diff --git a/vm_supervisor/network/ipaddresses.py b/vm_supervisor/network/ipaddresses.py
index c445129d6..8a7c038b0 100644
--- a/vm_supervisor/network/ipaddresses.py
+++ b/vm_supervisor/network/ipaddresses.py
@@ -1,9 +1,9 @@
 from ipaddress import IPv4Interface, IPv4Network
-from typing import Iterable
+from typing import Iterator
 
 
 class IPv4NetworkWithInterfaces(IPv4Network):
-    def hosts(self) -> Iterable[IPv4Interface]:
+    def hosts(self) -> Iterator[IPv4Interface]:
         network = int(self.network_address)
         broadcast = int(self.broadcast_address)
         for x in range(network + 1, broadcast):
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 2772cb308..02606ba0f 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -2,12 +2,12 @@
 import logging
 from typing import Dict, Iterable, Optional
 
-from aleph_message.models import ExecutableMessage
+from aleph_message.models import ExecutableMessage, ItemHash
 
 from vm_supervisor.network.hostnetwork import Network
 
 from .conf import settings
-from .models import ExecutableContent, VmExecution, VmHash
+from .models import ExecutableContent, VmExecution
 
 logger = logging.getLogger(__name__)
 
@@ -22,7 +22,7 @@ class VmPool:
     """
 
     counter: int  # Used to provide distinct ids to network interfaces
-    executions: Dict[VmHash, VmExecution]
+    executions: Dict[ItemHash, VmExecution]
     message_cache: Dict[str, ExecutableMessage] = {}
     network: Optional[Network]
 
@@ -40,7 +40,7 @@ def __init__(self):
         )
 
     async def create_a_vm(
-        self, vm_hash: VmHash, message: ExecutableContent, original: ExecutableContent
+        self, vm_hash: ItemHash, message: ExecutableContent, original: ExecutableContent
     ) -> VmExecution:
         """Create a new Aleph Firecracker VM from an Aleph function message."""
         execution = VmExecution(vm_hash=vm_hash, message=message, original=original)
@@ -48,7 +48,11 @@ async def create_a_vm(
         await execution.prepare()
         vm_id = self.get_unique_vm_id()
 
-        tap_interface = await self.network.create_tap(vm_id)
+        if self.network:
+            tap_interface = await self.network.create_tap(vm_id)
+        else:
+            tap_interface = None
+
         await execution.create(vm_id=vm_id, tap_interface=tap_interface)
         return execution
 
@@ -82,7 +86,7 @@ def get_unique_vm_id(self) -> int:
             else:
                 raise ValueError("No available value for vm_id.")
 
-    async def get_running_vm(self, vm_hash: VmHash) -> Optional[VmExecution]:
+    async def get_running_vm(self, vm_hash: ItemHash) -> Optional[VmExecution]:
         """Return a running VM or None. Disables the VM expiration task."""
         execution = self.executions.get(vm_hash)
         if execution and execution.is_running:
@@ -91,7 +95,7 @@ async def get_running_vm(self, vm_hash: VmHash) -> Optional[VmExecution]:
         else:
             return None
 
-    def forget_vm(self, vm_hash: VmHash) -> None:
+    def forget_vm(self, vm_hash: ItemHash) -> None:
         """Remove a VM from the executions pool.
 
         Used after self.create_a_vm(...) raised an error in order to
diff --git a/vm_supervisor/resources.py b/vm_supervisor/resources.py
index d914240f8..07425e0b3 100644
--- a/vm_supervisor/resources.py
+++ b/vm_supervisor/resources.py
@@ -1,3 +1,4 @@
+import math
 from datetime import datetime, timezone
 from functools import lru_cache
 from typing import Optional, Set, Tuple
@@ -100,8 +101,8 @@ async def about_system_usage(request: web.Request):
             core_frequencies=CoreFrequencies.from_psutil(psutil.cpu_freq()),
         ),
         mem=MemoryUsage(
-            total_kB=psutil.virtual_memory().total / 1000,
-            available_kB=psutil.virtual_memory().available / 1000,
+            total_kB=math.ceil(psutil.virtual_memory().total / 1000),
+            available_kB=math.floor(psutil.virtual_memory().available / 1000),
         ),
         disk=DiskUsage(
             total_kB=psutil.disk_usage(str(settings.PERSISTENT_VOLUMES_DIR)).total
@@ -122,6 +123,6 @@ async def about_system_usage(request: web.Request):
 
 class Allocation(BaseModel):
     persistent_vms: Set[str]
-    instances: Optional[Set[str]] = None
-    on_demand_vms: Optional[Set[str]] = None
+    instances: Set[str] = set()
+    on_demand_vms: Set[str] = set()
     jobs: Optional[Set] = None
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index a5021bf88..9beea492a 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -5,16 +5,17 @@
 import msgpack
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError
+from aleph_message.models import ItemHash
 from msgpack import UnpackValueError
 
 from firecracker.microvm import MicroVMFailedInit
 
 from .conf import settings
 from .messages import load_updated_message
-from .models import VmExecution, VmHash
+from .models import VmExecution
 from .pool import VmPool
 from .pubsub import PubSub
-from .vm.firecracker_program import (
+from .vm.firecracker.program import (
     FileTooLargeError,
     ResourceDownloadError,
     VmSetupError,
@@ -45,7 +46,7 @@ async def build_event_scope(event) -> Dict[str, Any]:
     }
 
 
-async def create_vm_execution(vm_hash: VmHash) -> VmExecution:
+async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
     message, original_message = await load_updated_message(vm_hash)
     pool.message_cache[vm_hash] = message
 
@@ -77,7 +78,7 @@ async def create_vm_execution(vm_hash: VmHash) -> VmExecution:
 
 
 async def run_code_on_request(
-    vm_hash: VmHash, path: str, request: web.Request
+    vm_hash: ItemHash, path: str, request: web.Request
 ) -> web.Response:
     """
     Execute the code corresponding to the 'code id' in the path.
@@ -167,7 +168,7 @@ async def run_code_on_request(
             await execution.stop()
 
 
-async def run_code_on_event(vm_hash: VmHash, event, pubsub: PubSub):
+async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub):
     """
     Execute code in response to an event.
     """
@@ -217,7 +218,7 @@ async def run_code_on_event(vm_hash: VmHash, event, pubsub: PubSub):
             await execution.stop()
 
 
-async def start_persistent_vm(vm_hash: VmHash, pubsub: PubSub) -> VmExecution:
+async def start_persistent_vm(vm_hash: ItemHash, pubsub: PubSub) -> VmExecution:
     execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
@@ -236,7 +237,7 @@ async def start_persistent_vm(vm_hash: VmHash, pubsub: PubSub) -> VmExecution:
     return execution
 
 
-async def stop_persistent_vm(vm_hash: VmHash) -> Optional[VmExecution]:
+async def stop_persistent_vm(vm_hash: ItemHash) -> Optional[VmExecution]:
     logger.info(f"Stopping persistent VM {vm_hash}")
     execution = await pool.get_running_vm(vm_hash)
     if execution:
diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py
index 32ac0f673..697ca31ca 100644
--- a/vm_supervisor/tasks.py
+++ b/vm_supervisor/tasks.py
@@ -8,12 +8,11 @@
 import aiohttp
 import pydantic
 from aiohttp import web
-from aleph_message.models import AlephMessage, ProgramMessage, parse_message
+from aleph_message.models import AlephMessage, ItemHash, ProgramMessage, parse_message
 from yarl import URL
 
 from .conf import settings
 from .messages import load_updated_message
-from .models import VmHash
 from .pubsub import PubSub
 from .reactor import Reactor
 from .utils import create_task_log_exceptions
@@ -114,7 +113,7 @@ async def start_watch_for_messages_task(app: web.Application):
     # Register an hardcoded initial program
     # TODO: Register all programs with subscriptions
     sample_message, _ = await load_updated_message(
-        ref=VmHash("cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e")
+        ref=ItemHash("cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e")
     )
     if isinstance(sample_message, ProgramMessage):
         assert sample_message.content.on.message, sample_message
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index fd1f8606b..23309b17d 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -9,6 +9,7 @@
 import aiohttp
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPNotFound
+from aleph_message.models import ItemHash
 from pydantic import ValidationError
 
 from packaging.version import InvalidVersion, Version
@@ -16,7 +17,6 @@
 from . import status
 from .conf import settings
 from .metrics import get_execution_records
-from .models import VmHash
 from .pubsub import PubSub
 from .resources import Allocation
 from .run import pool, run_code_on_request, start_persistent_vm
@@ -35,7 +35,7 @@ def run_code_from_path(request: web.Request) -> Awaitable[web.Response]:
     path = request.match_info["suffix"]
     path = path if path.startswith("/") else f"/{path}"
 
-    message_ref = VmHash(request.match_info["ref"])
+    message_ref = ItemHash(request.match_info["ref"])
     return run_code_on_request(message_ref, path, request)
 
 
@@ -62,16 +62,16 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
 
     message_ref_base32 = request.host.split(".")[0]
     if settings.FAKE_DATA_PROGRAM:
-        message_ref = VmHash("fake-hash")
+        message_ref = ItemHash("fake-hash")
     else:
         try:
-            message_ref = VmHash(b32_to_b16(message_ref_base32).decode())
+            message_ref = ItemHash(b32_to_b16(message_ref_base32).decode())
             logger.debug(
                 f"Using base32 message id from hostname to obtain '{message_ref}"
             )
         except binascii.Error:
             try:
-                message_ref = VmHash(
+                message_ref = ItemHash(
                     await get_ref_from_dns(domain=f"_aleph-id.{request.host}")
                 )
                 logger.debug(f"Using DNS TXT record to obtain '{message_ref}'")
@@ -204,7 +204,7 @@ async def update_allocations(request: web.Request):
 
     # Start VMs
     for vm_hash in allocation.persistent_vms:
-        vm_hash = VmHash(vm_hash)
+        vm_hash = ItemHash(vm_hash)
         logger.info(f"Starting long running VM {vm_hash}")
         await start_persistent_vm(vm_hash, pubsub)
 
@@ -217,7 +217,7 @@ async def update_allocations(request: web.Request):
 
     # Start Instances
     for instance_hash in allocation.instances:
-        instance_hash = VmHash(instance_hash)
+        instance_hash = ItemHash(instance_hash)
         logger.info(f"Starting instance {instance_hash}")
         await start_persistent_vm(instance_hash, pubsub)
 
diff --git a/vm_supervisor/vm/__init__.py b/vm_supervisor/vm/__init__.py
index 4b2995d53..5cab3b7b3 100644
--- a/vm_supervisor/vm/__init__.py
+++ b/vm_supervisor/vm/__init__.py
@@ -1,6 +1,4 @@
-from .firecracker import AlephFirecrackerProgram
-from .firecracker import AlephFirecrackerInstance
-
+from .firecracker import AlephFirecrackerInstance, AlephFirecrackerProgram
 
 __all__ = (
     "AlephFirecrackerProgram",
diff --git a/vm_supervisor/vm/firecracker/executable.py b/vm_supervisor/vm/firecracker/executable.py
index d1ab6d135..2389b51d5 100644
--- a/vm_supervisor/vm/firecracker/executable.py
+++ b/vm_supervisor/vm/firecracker/executable.py
@@ -9,7 +9,7 @@
 from multiprocessing import Process, set_start_method
 from os.path import exists, isfile
 from pathlib import Path
-from typing import Dict, List, Optional, Any
+from typing import Any, Dict, List, Optional
 
 from aleph_message.models import ItemHash
 
@@ -24,7 +24,6 @@
 from firecracker.config import FirecrackerConfig
 from firecracker.microvm import MicroVM
 from guest_api.__main__ import run_guest_api
-
 from vm_supervisor.conf import settings
 from vm_supervisor.models import ExecutableContent
 from vm_supervisor.network.firewall import teardown_nftables_for_vm
@@ -232,8 +231,14 @@ async def start(self):
         if self.enable_console:
             self.fvm.start_printing_logs()
 
+        await self.wait_for_init()
         logger.debug(f"started fvm {self.vm_id}")
 
+    async def wait_for_init(self) -> None:
+        """Wait for the init process of the virtual machine to be ready.
+        May be empty."""
+        return
+
     async def configure(self):
         raise NotImplementedError()
 
diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index 33d3577d3..4c6379d45 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -1,6 +1,5 @@
 import asyncio
 import logging
-from multiprocessing import set_start_method
 from pathlib import Path
 from typing import Optional
 
@@ -18,13 +17,10 @@
 from firecracker.microvm import setfacl
 from vm_supervisor.network.interfaces import TapInterface
 from vm_supervisor.storage import create_devmapper
-from .executable import (
-    AlephFirecrackerExecutable,
-    AlephFirecrackerResources,
-)
+
+from .executable import AlephFirecrackerExecutable, AlephFirecrackerResources
 
 logger = logging.getLogger(__name__)
-set_start_method("spawn")
 
 
 class AlephInstanceResources(AlephFirecrackerResources):
@@ -107,7 +103,12 @@ async def setup(self):
             else [],
         )
 
+    async def wait_for_init(self) -> None:
+        """Wait for the init process of the instance to be ready."""
+        # TODO: Check availability via ping ?
+        return
+
     async def configure(self):
         """Configure the VM by sending configuration info to it's init"""
         # TODO: Implement Cloud-init interface
-        raise NotImplementedError()
+        pass
diff --git a/vm_supervisor/vm/firecracker/program.py b/vm_supervisor/vm/firecracker/program.py
index 4e2beafbb..5568d4261 100644
--- a/vm_supervisor/vm/firecracker/program.py
+++ b/vm_supervisor/vm/firecracker/program.py
@@ -4,7 +4,6 @@
 import os.path
 from dataclasses import dataclass, field
 from enum import Enum
-from multiprocessing import set_start_method
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
@@ -27,15 +26,16 @@
 from vm_supervisor.models import ExecutableContent
 from vm_supervisor.network.interfaces import TapInterface
 from vm_supervisor.storage import get_code_path, get_data_path, get_runtime_path
+
 from .executable import (
     AlephFirecrackerExecutable,
     AlephFirecrackerResources,
     VmInitNotConnected,
     VmSetupError,
-    Volume, )
+    Volume,
+)
 
 logger = logging.getLogger(__name__)
-set_start_method("spawn")
 
 
 class FileTooLargeError(Exception):
@@ -55,7 +55,7 @@ def __init__(self, error: ClientResponseError):
         )
 
 
-def read_input_data(path_to_data: Path) -> Optional[bytes]:
+def read_input_data(path_to_data: Optional[Path]) -> Optional[bytes]:
     if not path_to_data:
         return None
 
@@ -98,12 +98,12 @@ def as_msgpack(self) -> bytes:
 class ConfigurationPayload:
     """Configuration passed to the init of the virtual machine in order to start the program."""
 
-    input_data: bytes
+    input_data: Optional[bytes]
     interface: Interface
     vm_hash: str
-    code: bytes = None
-    encoding: Encoding = None
-    entrypoint: str = None
+    encoding: Encoding
+    entrypoint: str
+    code: Optional[bytes] = None
     ip: Optional[str] = None
     route: Optional[str] = None
     dns_servers: List[str] = field(default_factory=list)
@@ -147,7 +147,7 @@ def __init__(self, message_content: ExecutableContent, namespace: str):
         self.code_encoding = message_content.code.encoding
         self.code_entrypoint = message_content.code.entrypoint
 
-    async def download_code(self):
+    async def download_code(self) -> None:
         code_ref: str = self.message_content.code.ref
         try:
             self.code_path = await get_code_path(code_ref)
@@ -155,7 +155,7 @@ async def download_code(self):
             raise ResourceDownloadError(error)
         assert self.code_path.is_file(), f"Code not found on '{self.code_path}'"
 
-    async def download_runtime(self):
+    async def download_runtime(self) -> None:
         runtime_ref: str = self.message_content.runtime.ref
         try:
             self.rootfs_path = await get_runtime_path(runtime_ref)
@@ -163,7 +163,7 @@ async def download_runtime(self):
             raise ResourceDownloadError(error)
         assert self.rootfs_path.is_file(), f"Runtime not found on {self.rootfs_path}"
 
-    async def download_data(self):
+    async def download_data(self) -> None:
         if self.message_content.data:
             data_ref: str = self.message_content.data.ref
             try:
@@ -185,6 +185,8 @@ async def download_all(self):
 def get_volumes_for_program(
     resources: AlephProgramResources, drives: List[Drive]
 ) -> Tuple[Optional[bytes], List[Volume]]:
+    code: Optional[bytes]
+    volumes: List[Volume]
     if resources.code_encoding == Encoding.squashfs:
         code = b""
         volumes = [Volume(mount="/opt/code", device="vdb", read_only=True)] + [
@@ -197,11 +199,9 @@ def get_volumes_for_program(
         ]
     else:
         if os.path.getsize(resources.code_path) > settings.MAX_PROGRAM_ARCHIVE_SIZE:
-            raise FileTooLargeError(f"Program file too large to pass as an inline zip")
+            raise FileTooLargeError("Program file too large to pass as an inline zip")
 
-        code: Optional[bytes] = (
-            resources.code_path.read_bytes() if resources.code_path else None
-        )
+        code = resources.code_path.read_bytes() if resources.code_path else None
         volumes = [
             Volume(
                 mount=volume.mount,
@@ -283,7 +283,11 @@ async def setup(self):
             else [],
         )
 
-    async def configure(self):
+    async def wait_for_init(self) -> None:
+        """Wait for the custom init inside the virtual machine to signal it is ready."""
+        await self.fvm.wait_for_init()
+
+    async def configure(self) -> None:
         """Configure the VM by sending configuration info to it's init"""
 
         code: Optional[bytes]
@@ -312,12 +316,18 @@ async def _setup_configuration(
 
         # The ip and route should not contain the network mask in order to maintain
         # compatibility with the existing runtimes.
-        ip = self.tap_interface.guest_ip.with_prefixlen.split("/", 1)[0]
-        route = str(self.tap_interface.host_ip).split("/", 1)[0]
+        if self.enable_networking and self.tap_interface:
+            ip = self.tap_interface.guest_ip.with_prefixlen.split("/", 1)[0]
+            route = str(self.tap_interface.host_ip).split("/", 1)[0]
+        else:
+            ip, route = None, None
+
+        if not settings.DNS_NAMESERVERS:
+            raise ValueError("Invalid configuration: DNS nameservers missing")
 
         config = ConfigurationPayload(
-            ip=ip if self.enable_networking else None,
-            route=route if self.enable_networking else None,
+            ip=ip,
+            route=route,
             dns_servers=settings.DNS_NAMESERVERS,
             code=code,
             encoding=self.resources.code_encoding,
@@ -349,7 +359,7 @@ async def run_code(
         logger.debug("running code")
         scope = scope or {}
 
-        async def communicate(reader, writer, scope):
+        async def communicate(reader, writer, scope) -> bytes:
             payload = RunCodePayload(scope=scope)
 
             writer.write(b"CONNECT 52\n" + payload.as_msgpack())

From 1169ec5fa907f80ac7b8aa7c60a5b95204c5703b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 13 Jun 2023 19:39:49 +0200
Subject: [PATCH 397/990] Refactoring: Subprocesses and path manipulation were
 not consistent

Solution:
1. Call subprocesses asynchronously using new `run_in_subprocess` function.
2. Use `pathlib.Path` instead of functions from `import os`.
3. Fix typing all around.
---
 firecracker/microvm.py                     |  22 +--
 vm_supervisor/metrics.py                   |  26 ++--
 vm_supervisor/network/hostnetwork.py       |  14 +-
 vm_supervisor/reactor.py                   |   1 -
 vm_supervisor/status.py                    |   1 -
 vm_supervisor/storage.py                   | 154 ++++++++++-----------
 vm_supervisor/tasks.py                     |   1 -
 vm_supervisor/utils.py                     |  23 ++-
 vm_supervisor/views.py                     |  22 ++-
 vm_supervisor/vm/firecracker/executable.py |  13 +-
 vm_supervisor/vm/firecracker/instance.py   |   7 +-
 vm_supervisor/vm/firecracker/program.py    |   2 +-
 12 files changed, 152 insertions(+), 134 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 5b1a5e0c4..c35ec1c47 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -166,9 +166,9 @@ async def start_firecracker(
         logger.debug(
             " ".join(
                 (
-                    self.firecracker_bin_path,
+                    str(self.firecracker_bin_path),
                     "--api-sock",
-                    self.socket_path,
+                    str(self.socket_path),
                     "--config-file",
                     config_file.name,
                 )
@@ -253,15 +253,18 @@ def enable_kernel(self, kernel_image_path: Path) -> Path:
         if self.use_jailer:
             kernel_filename = kernel_image_path.name
             jailer_kernel_image_path = f"/opt/{kernel_filename}"
-            os.link(kernel_image_path, f"{self.jailer_path}{jailer_kernel_image_path}")
-            kernel_image_path = jailer_kernel_image_path
-        return kernel_image_path
+            kernel_image_path.link_to(f"{self.jailer_path}{jailer_kernel_image_path}")
+            return Path(jailer_kernel_image_path)
+        else:
+            return kernel_image_path
 
     def enable_rootfs(self, path_on_host: Path) -> Path:
         if path_on_host.is_file():
             return self.enable_file_rootfs(path_on_host)
         elif path_on_host.is_block_device():
             return self.enable_device_mapper_rootfs(path_on_host)
+        else:
+            raise ValueError(f"Not a file or a block device: {path_on_host}")
 
     def enable_file_rootfs(self, path_on_host: Path) -> Path:
         """Make a rootfs available to the VM.
@@ -310,11 +313,12 @@ def enable_drive(self, drive_path: Path, read_only: bool = True) -> Drive:
         """
         index = len(self.drives)
         device_name = self.compute_device_name(index)
+
         if self.use_jailer:
             drive_filename = drive_path.name
             jailer_path_on_host = f"/opt/{drive_filename}"
-            os.link(drive_path, f"{self.jailer_path}/{jailer_path_on_host}")
-            drive_path = jailer_path_on_host
+            drive_path.link_to(f"{self.jailer_path}/{jailer_path_on_host}")
+            drive_path = Path(jailer_path_on_host)
 
         drive = Drive(
             drive_id=device_name,
@@ -370,7 +374,7 @@ async def unix_client_connected(*_):
             logger.warning("Never received signal from init")
             raise MicroVMFailedInit()
 
-    async def shutdown(self):
+    async def shutdown(self) -> None:
         logger.debug(f"Shutdown vm={self.vm_id}")
         try:
             reader, writer = await asyncio.open_unix_connection(path=self.vsock_path)
@@ -400,7 +404,7 @@ async def shutdown(self):
             logger.debug(f"msg2={msg2!r}")
 
             if msg2 != b"STOPZ\n":
-                logger.warning(f"Unexpected response from VM: {msg2[:20]}")
+                logger.warning(f"Unexpected response from VM: {msg2[:20]!r}")
         except ConnectionResetError as error:
             logger.warning(
                 f"ConnectionResetError in shutdown of {self.vm_id}: {error.args}"
diff --git a/vm_supervisor/metrics.py b/vm_supervisor/metrics.py
index 3d397ff6c..a5687bbb7 100644
--- a/vm_supervisor/metrics.py
+++ b/vm_supervisor/metrics.py
@@ -1,7 +1,6 @@
 import logging
-import os
-from os.path import join
-from typing import Iterable, Optional
+from pathlib import Path
+from typing import Any, Iterable
 from uuid import UUID
 
 from sqlalchemy import Column, DateTime, Float, Integer, String, create_engine
@@ -15,7 +14,7 @@
 
 logger = logging.getLogger(__name__)
 
-Base = declarative_base()
+Base: Any = declarative_base()
 
 
 def setup_engine():
@@ -40,13 +39,13 @@ class ExecutionRecord(Base):
     time_started = Column(DateTime)
     time_stopping = Column(DateTime)
 
-    cpu_time_user: Optional[float] = Column(Float, nullable=True)
-    cpu_time_system: Optional[float] = Column(Float, nullable=True)
+    cpu_time_user = Column(Float, nullable=True)
+    cpu_time_system = Column(Float, nullable=True)
 
-    io_read_count: Optional[int] = Column(Integer, nullable=True)
-    io_write_count: Optional[int] = Column(Integer, nullable=True)
-    io_read_bytes: Optional[int] = Column(Integer, nullable=True)
-    io_write_bytes: Optional[int] = Column(Integer, nullable=True)
+    io_read_count = Column(Integer, nullable=True)
+    io_write_count = Column(Integer, nullable=True)
+    io_read_bytes = Column(Integer, nullable=True)
+    io_write_bytes = Column(Integer, nullable=True)
 
     vcpus = Column(Integer, nullable=False)
     memory = Column(Integer, nullable=False)
@@ -61,10 +60,9 @@ def to_dict(self):
 
 async def save_execution_data(execution_uuid: UUID, execution_data: str):
     """Save the execution data in a file on disk"""
-    os.makedirs(settings.EXECUTION_LOG_DIRECTORY, exist_ok=True)
-    filepath = join(settings.EXECUTION_LOG_DIRECTORY, f"{execution_uuid}.json")
-    with open(filepath, "w") as fd:
-        fd.write(execution_data)
+    directory = Path(settings.EXECUTION_LOG_DIRECTORY)
+    directory.mkdir(exist_ok=True)
+    (directory / f"{execution_uuid}.json").write_text(execution_data)
 
 
 async def save_record(record: ExecutionRecord):
diff --git a/vm_supervisor/network/hostnetwork.py b/vm_supervisor/network/hostnetwork.py
index fee59abb7..81ecd4b7c 100644
--- a/vm_supervisor/network/hostnetwork.py
+++ b/vm_supervisor/network/hostnetwork.py
@@ -1,4 +1,5 @@
 import logging
+from pathlib import Path
 
 from .firewall import initialize_nftables, setup_nftables_for_vm, teardown_nftables
 from .interfaces import TapInterface
@@ -9,8 +10,7 @@
 
 def get_ipv4_forwarding_state() -> int:
     """Reads the current ipv4 forwarding setting from the hosts, converts it to int and returns it"""
-    with open("/proc/sys/net/ipv4/ip_forward") as f:
-        return int(f.read())
+    return int(Path("/proc/sys/net/ipv4/ip_forward").read_text())
 
 
 class Network:
@@ -25,18 +25,18 @@ def get_network_for_tap(self, vm_id: int) -> IPv4NetworkWithInterfaces:
 
     def enable_ipv4_forwarding(self) -> None:
         """Saves the hosts IPv4 forwarding state, and if it was disabled, enables it"""
-        logger.debug(f"Enabling IPv4 forwarding")
+        logger.debug("Enabling IPv4 forwarding")
         self.ipv4_forward_state_before_setup = get_ipv4_forwarding_state()
         if not self.ipv4_forward_state_before_setup:
-            with open("/proc/sys/net/ipv4/ip_forward", "w") as f:
-                f.write("1")
+            Path("/proc/sys/net/ipv4/ip_forward").write_text("1")
 
     def reset_ipv4_forwarding_state(self) -> None:
         """Returns the hosts IPv4 forwarding state how it was before we enabled it"""
         logger.debug("Resetting IPv4 forwarding state to state before we enabled it")
         if self.ipv4_forward_state_before_setup != get_ipv4_forwarding_state():
-            with open("/proc/sys/net/ipv4/ip_forward", "w") as f:
-                f.write(str(self.ipv4_forward_state_before_setup))
+            Path("/proc/sys/net/ipv4/ip_forward").write_text(
+                str(self.ipv4_forward_state_before_setup)
+            )
 
     def __init__(
         self, vm_address_pool_range: str, vm_network_size: int, external_interface: str
diff --git a/vm_supervisor/reactor.py b/vm_supervisor/reactor.py
index b977c4ae4..4c0bbe284 100644
--- a/vm_supervisor/reactor.py
+++ b/vm_supervisor/reactor.py
@@ -36,7 +36,6 @@ def subscription_matches(subscription: Subscription, message: AlephMessage) -> b
 
 
 class Reactor:
-
     pubsub: PubSub
     listeners: List[AlephMessage]
 
diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py
index 02c462ef2..e5482bddf 100644
--- a/vm_supervisor/status.py
+++ b/vm_supervisor/status.py
@@ -131,7 +131,6 @@ async def check_error_raised(session: ClientSession) -> bool:
 
 
 async def check_crash_and_restart(session: ClientSession) -> bool:
-
     # Crash the VM init.
     async with session.get(f"{CHECK_VM_URL}/crash") as resp:
         if resp.status != 502:
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 779819294..f8b2d6b01 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -4,15 +4,11 @@
 In this prototype, it returns a hardcoded example.
 In the future, it should connect to an Aleph node and retrieve the code from there.
 """
-import asyncio
 import hashlib
 import json
 import logging
-import os
 import re
-import subprocess
 import sys
-from os.path import isfile, join
 from pathlib import Path
 from shutil import make_archive
 from typing import Union
@@ -34,19 +30,28 @@
 )
 
 from .conf import settings
+from .utils import run_in_subprocess
 
 logger = logging.getLogger(__name__)
 
 DEVICE_MAPPER_DIRECTORY = "/dev/mapper"
 
 
+async def chown_to_jailman(path: Path) -> None:
+    """Changes ownership of the target when running firecracker inside jailer isolation."""
+    if not path.exists():
+        raise FileNotFoundError("No such file to change ownership from", path)
+    if settings.USE_JAILER:
+        await run_in_subprocess(["chown", "jailman:jailman", str(path)])
+
+
 async def download_file(url: str, local_path: Path) -> None:
     # TODO: Limit max size of download to the message specification
-    if isfile(local_path):
+    if local_path.is_file():
         logger.debug(f"File already exists: {local_path}")
         return
 
-    tmp_path = f"{local_path}.part"
+    tmp_path = Path(f"{local_path}.part")
     logger.debug(f"Downloading {url} -> {tmp_path}")
     async with aiohttp.ClientSession() as session:
         resp = await session.get(url)
@@ -64,14 +69,11 @@ async def download_file(url: str, local_path: Path) -> None:
                         sys.stdout.write(".")
                         sys.stdout.flush()
 
-            os.rename(tmp_path, local_path)
+            tmp_path.rename(local_path)
             logger.debug(f"Download complete, moved {tmp_path} -> {local_path}")
         except Exception:
             # Ensure no partial file is left
-            try:
-                os.remove(tmp_path)
-            except FileNotFoundError:
-                pass
+            tmp_path.unlink(missing_ok=True)
             raise
 
 
@@ -92,7 +94,7 @@ async def get_message(ref: str) -> ExecutableMessage:
     if settings.FAKE_DATA_PROGRAM:
         cache_path = settings.FAKE_DATA_MESSAGE
     else:
-        cache_path = Path(join(settings.MESSAGE_CACHE, ref) + ".json")
+        cache_path = (Path(settings.MESSAGE_CACHE) / ref).with_suffix(".json")
         url = f"{settings.CONNECTOR_URL}/download/message/{ref}"
         await download_file(url, cache_path)
 
@@ -110,25 +112,28 @@ async def get_message(ref: str) -> ExecutableMessage:
 
 async def get_code_path(ref: str) -> Path:
     if settings.FAKE_DATA_PROGRAM:
-        archive_path = settings.FAKE_DATA_PROGRAM
+        archive_path = Path(settings.FAKE_DATA_PROGRAM)
 
         encoding: Encoding = (
             await get_message(ref="fake-message")
         ).content.code.encoding
         if encoding == Encoding.squashfs:
-            if os.path.exists(f"{archive_path}.squashfs"):
-                os.remove(f"{archive_path}.squashfs")
-            os.system(f"mksquashfs {archive_path} {archive_path}.squashfs")
-            logger.debug(f"Squashfs generated on {archive_path}.squashfs")
-            return Path(f"{archive_path}.squashfs")
+            squashfs_path = Path(archive_path.name + ".squashfs")
+            squashfs_path.unlink(missing_ok=True)
+            await run_in_subprocess(
+                ["mksquashfs", str(archive_path), str(squashfs_path)]
+            )
+            logger.debug(f"Squashfs generated on {squashfs_path}")
+            return squashfs_path
         elif encoding == Encoding.zip:
             make_archive(str(archive_path), "zip", root_dir=archive_path)
-            logger.debug(f"Zip generated on {archive_path}.zip")
-            return Path(f"{archive_path}.zip")
+            zip_path = Path(f"{archive_path}.zip")
+            logger.debug(f"Zip generated on {zip_path}")
+            return zip_path
         else:
             raise ValueError(f"Unsupported encoding: {encoding}")
 
-    cache_path = Path(join(settings.CODE_CACHE, ref))
+    cache_path = Path(settings.CODE_CACHE) / ref
     url = f"{settings.CONNECTOR_URL}/download/code/{ref}"
     await download_file(url, cache_path)
     return cache_path
@@ -140,7 +145,7 @@ async def get_data_path(ref: str) -> Path:
         make_archive(str(data_dir), "zip", data_dir)
         return Path(f"{data_dir}.zip")
 
-    cache_path = Path(join(settings.DATA_CACHE, ref))
+    cache_path = Path(settings.DATA_CACHE) / ref
     url = f"{settings.CONNECTOR_URL}/download/data/{ref}"
     await download_file(url, cache_path)
     return cache_path
@@ -150,23 +155,24 @@ async def get_runtime_path(ref: str) -> Path:
     if settings.FAKE_DATA_PROGRAM:
         return Path(settings.FAKE_DATA_RUNTIME)
 
-    cache_path = Path(join(settings.RUNTIME_CACHE, ref))
+    cache_path = Path(settings.RUNTIME_CACHE) / ref
     url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}"
     await download_file(url, cache_path)
-    if settings.USE_JAILER:
-        os.system(f"chown jailman:jailman {cache_path}")
+    await chown_to_jailman(cache_path)
     return cache_path
 
 
-def create_ext4(path: Path, size_mib: int) -> bool:
-    if os.path.isfile(path):
+async def create_ext4(path: Path, size_mib: int) -> bool:
+    if path.is_file():
+        logger.debug("File already exists, skipping ext4 creation", path)
         return False
     tmp_path = f"{path}.tmp"
-    os.system(f"dd if=/dev/zero of={tmp_path} bs=1M count={size_mib}")
-    os.system(f"mkfs.ext4 {tmp_path}")
-    if settings.USE_JAILER:
-        os.system(f"chown jailman:jailman {tmp_path}")
-    os.rename(tmp_path, path)
+    await run_in_subprocess(
+        ["dd", "if=/dev/zero", f"of={tmp_path}", "bs=1M", f"count={size_mib}"]
+    )
+    await run_in_subprocess(["mkfs.ext4", tmp_path])
+    await chown_to_jailman(Path(tmp_path))
+    Path(tmp_path).rename(path)
     return True
 
 
@@ -177,9 +183,10 @@ async def create_volume_file(
     path = Path(settings.PERSISTENT_VOLUMES_DIR) / namespace / f"{volume_name}.ext4"
     if not path.is_file():
         logger.debug(f"Creating {volume.size_mib}MB volume")
-        os.system(f"dd if=/dev/zero of={path} bs=1M count={volume.size_mib}")
-        if settings.USE_JAILER:
-            os.system(f"chown jailman:jailman {path}")
+        await run_in_subprocess(
+            ["dd", "if=/dev/zero", f"of={path}", "bs=1M", f"count={volume.size_mib}"]
+        )
+        await chown_to_jailman(path)
     return path
 
 
@@ -188,44 +195,34 @@ async def create_loopback_device(path: Path, read_only: bool = False) -> str:
     if read_only:
         command_args.append("--read-only")
     command_args.append(str(path))
-    loop_device = subprocess.run(
-        command_args, check=True, capture_output=True, encoding="UTF-8"
-    ).stdout.strip()
+    stdout = await run_in_subprocess(command_args)
+    loop_device = stdout.strip().decode()
     return loop_device
 
 
-def get_block_size(device_path: Path) -> str:
-    block_size = subprocess.run(
-        ["blockdev", "--getsz", device_path],
-        check=True,
-        capture_output=True,
-        encoding="UTF-8",
-    ).stdout.strip()
+async def get_block_size(device_path: Path) -> int:
+    command = ["blockdev", "--getsz", str(device_path)]
+    stdout = await run_in_subprocess(command)
+    block_size = int(stdout.decode("UTF-8").strip())
     return block_size
 
 
-def create_mapped_device(device_name: str, table_command: str) -> None:
-    subprocess.run(
-        f"dmsetup create {device_name}",
-        input=table_command,
-        text=True,
-        shell=True,
-        check=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
+async def create_mapped_device(device_name: str, table_command: str) -> None:
+    command = ["dmsetup", "create", device_name]
+    await run_in_subprocess(command, stdin_input=table_command.encode())
 
 
-def e2fs_check_and_resize(device_path: Path) -> None:
-    os.system(f"e2fsck -fy {device_path}")
-    os.system(f"resize2fs {device_path}")
+async def e2fs_check_and_resize(device_path: Path) -> None:
+    await run_in_subprocess(["e2fsck", "-fy", str(device_path)])
+    await run_in_subprocess(["resize2fs", str(device_path)])
 
 
 async def create_devmapper(
     volume: Union[PersistentVolume, RootfsVolume], namespace: str
 ) -> Path:
     """It creates a /dev/mapper/DEVICE inside the VM, that is an extended mapped device of the volume specified.
-    We follow the steps described here: https://community.aleph.im/t/deploying-mutable-vm-instances-on-aleph/56/2"""
+    We follow the steps described here: https://community.aleph.im/t/deploying-mutable-vm-instances-on-aleph/56/2
+    """
     volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
     mapped_volume_name = f"{namespace}_{volume_name}"
     path_mapped_volume_name = Path(DEVICE_MAPPER_DIRECTORY) / mapped_volume_name
@@ -237,23 +234,22 @@ async def create_devmapper(
     parent_path = await get_runtime_path(volume.parent.ref)
 
     base_loop_device = await create_loopback_device(parent_path, read_only=True)
-    base_block_size = get_block_size(parent_path)
+    base_block_size: int = await get_block_size(parent_path)
     extended_loop_device = await create_loopback_device(volume_path)
-    extended_block_size = get_block_size(volume_path)
+    extended_block_size: int = await get_block_size(volume_path)
 
     base_table_command = f"0 {base_block_size} linear {base_loop_device} 0\n{base_block_size} {extended_block_size} zero"
     base_volume_name = volume.parent.ref
     path_base_device_name = Path(DEVICE_MAPPER_DIRECTORY) / base_volume_name
     if not path_base_device_name.is_block_device():
-        create_mapped_device(base_volume_name, base_table_command)
+        await create_mapped_device(base_volume_name, base_table_command)
 
     snapshot_table_command = f"0 {extended_block_size} snapshot {path_base_device_name} {extended_loop_device} P 8"
-    create_mapped_device(mapped_volume_name, snapshot_table_command)
+    await create_mapped_device(mapped_volume_name, snapshot_table_command)
 
-    e2fs_check_and_resize(path_mapped_volume_name)
-    if settings.USE_JAILER:
-        os.system(f"chown jailman:jailman {path_base_device_name}")
-        os.system(f"chown jailman:jailman {path_mapped_volume_name}")
+    await e2fs_check_and_resize(path_mapped_volume_name)
+    await chown_to_jailman(path_base_device_name)
+    await chown_to_jailman(path_mapped_volume_name)
     return path_mapped_volume_name
 
 
@@ -261,11 +257,10 @@ async def get_existing_file(ref: str) -> Path:
     if settings.FAKE_DATA_PROGRAM and settings.FAKE_DATA_VOLUME:
         return Path(settings.FAKE_DATA_VOLUME)
 
-    cache_path = Path(join(settings.DATA_CACHE, ref))
+    cache_path = Path(settings.DATA_CACHE) / ref
     url = f"{settings.CONNECTOR_URL}/download/data/{ref}"
     await download_file(url, cache_path)
-    if settings.USE_JAILER:
-        os.system(f"chown jailman:jailman {cache_path}")
+    await chown_to_jailman(cache_path)
     return cache_path
 
 
@@ -274,24 +269,21 @@ async def get_volume_path(volume: MachineVolume, namespace: str) -> Path:
         ref = volume.ref
         return await get_existing_file(ref)
     elif isinstance(volume, PersistentVolume) or isinstance(volume, RootfsVolume):
-        volume_name = volume.name if isinstance(volume, RootfsVolume) else "rootfs"
+        volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
         if volume.persistence != VolumePersistence.host:
             raise NotImplementedError("Only 'host' persistence is supported")
         if not re.match(r"^[\w\-_/]+$", volume_name):
             raise ValueError(f"Invalid value for volume name: {volume_name}")
-        os.makedirs(join(settings.PERSISTENT_VOLUMES_DIR, namespace), exist_ok=True)
+        (Path(settings.PERSISTENT_VOLUMES_DIR) / namespace).mkdir(exist_ok=True)
         if volume.parent:
-            device_path = await asyncio.get_event_loop().run_in_executor(
-                None, create_devmapper, volume, namespace
-            )
-            return device_path
+            return await create_devmapper(volume, namespace)
         else:
-            volume_path = Path(
-                join(settings.PERSISTENT_VOLUMES_DIR, namespace, f"{volume_name}.ext4")
-            )
-            await asyncio.get_event_loop().run_in_executor(
-                None, create_ext4, volume_path, volume.size_mib
+            volume_path = (
+                Path(settings.PERSISTENT_VOLUMES_DIR)
+                / namespace
+                / f"{volume_name}.ext4"
             )
+            await create_ext4(volume_path, volume.size_mib)
             return volume_path
     else:
         raise NotImplementedError("Only immutable volumes are supported")
diff --git a/vm_supervisor/tasks.py b/vm_supervisor/tasks.py
index 697ca31ca..53b483d13 100644
--- a/vm_supervisor/tasks.py
+++ b/vm_supervisor/tasks.py
@@ -92,7 +92,6 @@ async def watch_for_messages(dispatcher: PubSub, reactor: Reactor):
     )
 
     async for message in retry_generator(subscribe_via_ws(url)):
-
         # Dispatch update to running VMs
         await dispatcher.publish(key=message.item_hash, value=message)
         if hasattr(message.content, "ref") and message.content.ref:
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index 219d97289..9da13e1a6 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -1,10 +1,11 @@
 import asyncio
 import json
 import logging
+import subprocess
 from base64 import b16encode, b32decode
 from dataclasses import asdict as dataclass_as_dict
 from dataclasses import is_dataclass
-from typing import Any, Coroutine, Optional
+from typing import Any, Coroutine, List, Optional
 
 import aiodns
 
@@ -52,3 +53,23 @@ async def run_and_log_exception(coro: Coroutine):
 def create_task_log_exceptions(coro: Coroutine, *, name=None):
     """Ensure that exceptions running in coroutines are logged."""
     return asyncio.create_task(run_and_log_exception(coro), name=name)
+
+
+async def run_in_subprocess(
+    command: List[str], check: bool = True, stdin_input: Optional[bytes] = None
+) -> bytes:
+    """Run the specified command in a subprocess, returns the stdout of the process."""
+    process = await asyncio.create_subprocess_exec(
+        *command,
+        stdin=asyncio.subprocess.PIPE,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await process.communicate(input=stdin_input)
+
+    if check and process.returncode:
+        raise subprocess.CalledProcessError(
+            process.returncode, str(command), stderr.decode()
+        )
+
+    return stdout
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index 23309b17d..db547293d 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -1,7 +1,7 @@
 import binascii
 import logging
-import os.path
 from hashlib import sha256
+from pathlib import Path
 from string import Template
 from typing import Awaitable, Optional
 
@@ -123,17 +123,15 @@ async def about_execution_records(request: web.Request):
 
 async def index(request: web.Request):
     assert request.method == "GET"
-    path = os.path.join(os.path.dirname(__file__), "templates/index.html")
-    with open(path, "r") as template:
-        body = template.read()
-        s = Template(body)
-        body = s.substitute(
-            public_url=f"https://{settings.DOMAIN_NAME}/",
-            multiaddr_dns4=f"/dns4/{settings.DOMAIN_NAME}/tcp/443/https",
-            multiaddr_dns6=f"/dns6/{settings.DOMAIN_NAME}/tcp/443/https",
-            check_fastapi_vm_id=settings.CHECK_FASTAPI_VM_ID,
-            version=__version__,
-        )
+    body = (Path(__file__).parent.absolute() / "templates/index.html").read_text()
+    s = Template(body)
+    body = s.substitute(
+        public_url=f"https://{settings.DOMAIN_NAME}/",
+        multiaddr_dns4=f"/dns4/{settings.DOMAIN_NAME}/tcp/443/https",
+        multiaddr_dns6=f"/dns6/{settings.DOMAIN_NAME}/tcp/443/https",
+        check_fastapi_vm_id=settings.CHECK_FASTAPI_VM_ID,
+        version=__version__,
+    )
     return web.Response(content_type="text/html", body=body)
 
 
diff --git a/vm_supervisor/vm/firecracker/executable.py b/vm_supervisor/vm/firecracker/executable.py
index 2389b51d5..4f6944895 100644
--- a/vm_supervisor/vm/firecracker/executable.py
+++ b/vm_supervisor/vm/firecracker/executable.py
@@ -9,13 +9,13 @@
 from multiprocessing import Process, set_start_method
 from os.path import exists, isfile
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Generic, List, Optional, TypeVar
 
 from aleph_message.models import ItemHash
 
 psutil: Optional[Any]
 try:
-    import psutil as psutil
+    import psutil  # type: ignore [no-redef]
 except ImportError:
     psutil = None
 from aiohttp import ClientResponseError
@@ -62,7 +62,7 @@ class HostVolume:
 
 
 @dataclass
-class VmConfiguration:
+class BaseConfiguration:
     vm_hash: ItemHash
     ip: Optional[str] = None
     route: Optional[str] = None
@@ -131,7 +131,10 @@ class VmInitNotConnected(Exception):
     pass
 
 
-class AlephFirecrackerExecutable:
+ConfigurationType = TypeVar("ConfigurationType")
+
+
+class AlephFirecrackerExecutable(Generic[ConfigurationType]):
     vm_id: int
     vm_hash: ItemHash
     resources: AlephFirecrackerResources
@@ -140,7 +143,7 @@ class AlephFirecrackerExecutable:
     hardware_resources: MachineResources
     tap_interface: Optional[TapInterface] = None
     fvm: MicroVM
-    vm_configuration: Optional[VmConfiguration]
+    vm_configuration: Optional[ConfigurationType]
     guest_api_process: Optional[Process] = None
     is_instance: bool
     _firecracker_config: Optional[FirecrackerConfig] = None
diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index 4c6379d45..2d85b9898 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -18,7 +18,11 @@
 from vm_supervisor.network.interfaces import TapInterface
 from vm_supervisor.storage import create_devmapper
 
-from .executable import AlephFirecrackerExecutable, AlephFirecrackerResources
+from .executable import (
+    AlephFirecrackerExecutable,
+    AlephFirecrackerResources,
+    BaseConfiguration,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -41,6 +45,7 @@ async def download_all(self):
 
 
 class AlephFirecrackerInstance(AlephFirecrackerExecutable):
+    vm_configuration: BaseConfiguration
     resources: AlephInstanceResources
     is_instance = True
 
diff --git a/vm_supervisor/vm/firecracker/program.py b/vm_supervisor/vm/firecracker/program.py
index 5568d4261..c05787727 100644
--- a/vm_supervisor/vm/firecracker/program.py
+++ b/vm_supervisor/vm/firecracker/program.py
@@ -213,7 +213,7 @@ def get_volumes_for_program(
     return code, volumes
 
 
-class AlephFirecrackerProgram(AlephFirecrackerExecutable):
+class AlephFirecrackerProgram(AlephFirecrackerExecutable[ProgramVmConfiguration]):
     vm_configuration: Optional[ProgramVmConfiguration]
     resources: AlephProgramResources
     is_instance = False

From 117a1634ca5be848fc795443c728f4685a29b319 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 13 Jun 2023 20:01:14 +0200
Subject: [PATCH 398/990] CI: Code quality tools were absent from CI

Solution: Start adding `black`, `isort` and `mypy` in GitHub Actions.
---
 .github/workflows/code-quality.yml | 40 ++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 .github/workflows/code-quality.yml

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
new file mode 100644
index 000000000..0f0bcf823
--- /dev/null
+++ b/.github/workflows/code-quality.yml
@@ -0,0 +1,40 @@
+name: Test code quality
+
+on: push
+
+jobs:
+  code-quality:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
+      - name: Install required system packages only for Ubuntu Linux
+        run: |
+          sudo apt-get update
+          sudo apt-get -y upgrade
+          sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema
+
+      - name: Install required Python packages
+        run: |
+          python3 -m pip install mypy pytest black isort
+
+      - name: Test with Black
+        run: |
+          black --check ./vm_supervisor
+
+      - name: Test with isort
+        run: |
+          isort --check-only --profile=black ./vm_supervisor
+
+      - name: Test with MyPy
+        run: |
+          mypy --ignore-missing-imports ./vm_supervisor
+#          mypy --config-file ./mypy.ini ./vm_supervisor
+
+#      - name: Test with flake8
+#        run: |
+#          flake8 ./vm_supervisor

From 9d67c16de50934f85fd366902644836654fec931 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 14 Jun 2023 16:18:17 +0200
Subject: [PATCH 399/990] Fix: Program could not run without a data volume

---
 vm_supervisor/vm/firecracker/program.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/vm/firecracker/program.py b/vm_supervisor/vm/firecracker/program.py
index c05787727..79a79a54e 100644
--- a/vm_supervisor/vm/firecracker/program.py
+++ b/vm_supervisor/vm/firecracker/program.py
@@ -170,7 +170,9 @@ async def download_data(self) -> None:
                 self.data_path = await get_data_path(data_ref)
             except ClientResponseError as error:
                 raise ResourceDownloadError(error)
-            assert self.data_path.is_file(), f"Data nout found on {self.data_path}"
+            assert self.data_path.is_file(), f"Data not found on {self.data_path}"
+        else:
+            self.data_path = None
 
     async def download_all(self):
         await asyncio.gather(

From c96ac0b4e07f541df8e6aeba5946be83589a59ee Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 13 Jun 2023 17:21:25 +0200
Subject: [PATCH 400/990] Fix: Developers could not start an instance from
 command-line

Solution: Add option `--run-test-instance` to the command-line of the VM supervisor that starts an instance instead of starting the supervisor.
---
 ...h.json => program_message_from_aleph.json} |  0
 packaging/Makefile                            |  3 +-
 vm_supervisor/__main__.py                     | 39 +++++++++++++++--
 vm_supervisor/conf.py                         |  9 +++-
 vm_supervisor/run.py                          |  6 ++-
 vm_supervisor/storage.py                      | 43 +++++++++++--------
 vm_supervisor/utils.py                        | 18 +++++++-
 7 files changed, 91 insertions(+), 27 deletions(-)
 rename examples/{message_from_aleph.json => program_message_from_aleph.json} (100%)

diff --git a/examples/message_from_aleph.json b/examples/program_message_from_aleph.json
similarity index 100%
rename from examples/message_from_aleph.json
rename to examples/program_message_from_aleph.json
diff --git a/packaging/Makefile b/packaging/Makefile
index daa3bad6a..08d26f501 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -13,7 +13,8 @@ debian-package-code:
 	# Fake data for diagnostic and benchmarks
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/
 	cp -r ../examples/example_fastapi ./aleph-vm/opt/aleph-vm/examples/example_fastapi
-	cp ../examples/message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/message_from_aleph.json
+	cp ../examples/program_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/program_message_from_aleph.json
+	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
 	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.0a2'
diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index 8858c5f9f..d739238d4 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -23,7 +23,7 @@
 from . import metrics, supervisor
 from .conf import make_db_url, settings
 from .pubsub import PubSub
-from .run import run_code_on_event, run_code_on_request
+from .run import run_code_on_event, run_code_on_request, start_persistent_vm
 
 logger = logging.getLogger(__name__)
 
@@ -122,17 +122,25 @@ def parse_args(args):
         default=None,
         help="Path to project containing fake data",
     )
+    parser.add_argument(
+        "-k",
+        "--run-test-instance",
+        dest="run_test_instance",
+        action="store_true",
+        default=False,
+        help="Run a test instance instead of starting the entire supervisor",
+    )
     return parser.parse_args(args)
 
 
 async def benchmark(runs: int):
-    """Measure performance by immediately running the supervisor
+    """Measure program performance by immediately running the supervisor
     with fake requests.
     """
     engine = metrics.setup_engine()
     metrics.create_tables(engine)
 
-    ref = ItemHash("fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-hash")
+    ref = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")
     settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM
 
     FakeRequest: Request
@@ -209,6 +217,17 @@ async def fake_read() -> bytes:
     print("Event result", result)
 
 
+async def run_instance(item_hash: ItemHash):
+    """Run an instance from an InstanceMessage."""
+
+    # The main program uses a singleton pubsub instance in order to watch for updates.
+    # We create another instance here since that singleton is not initialized yet.
+    # Watching for updates on this instance will therefore not work.
+    dummy_pubsub = PubSub()
+
+    await start_persistent_vm(item_hash, dummy_pubsub)
+
+
 @contextlib.contextmanager
 def change_dir(directory: Path):
     current_directory = Path.cwd()
@@ -283,9 +302,21 @@ def main():
 
     if args.benchmark > 0:
         asyncio.run(benchmark(runs=args.benchmark), debug=args.debug_asyncio)
-        print("Finished")
+        logger.info("Finished")
+        sys.exit(0)
     elif args.do_not_run:
         logger.info("Option --do-not-run, exiting")
+    elif args.run_test_instance:
+        logger.info("Running test instance virtual machine")
+        instance_item_hash = settings.FAKE_INSTANCE_ID
+
+        async def forever():
+            await run_instance(item_hash=instance_item_hash)
+            await asyncio.sleep(1000)
+
+        asyncio.run(forever())
+        logger.info("Finished")
+        sys.exit(0)
     else:
         supervisor.run()
 
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 0f5b15bfc..9aa1f3511 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -129,7 +129,7 @@ class Settings(BaseSettings):
     )
 
     FAKE_DATA_MESSAGE = Path(
-        abspath(join(__file__, "../../examples/message_from_aleph.json"))
+        abspath(join(__file__, "../../examples/program_message_from_aleph.json"))
     )
     FAKE_DATA_DATA: Optional[Path] = Path(
         abspath(join(__file__, "../../examples/data/"))
@@ -141,6 +141,13 @@ class Settings(BaseSettings):
         abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs"))
     )
 
+    FAKE_INSTANCE_ID = (
+        "decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca"
+    )
+    FAKE_INSTANCE_MESSAGE = Path(
+        abspath(join(__file__, "../../examples/instance_message_from_aleph.json"))
+    )
+
     CHECK_FASTAPI_VM_ID = (
         "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
     )
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 9beea492a..d05b09eeb 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -50,6 +50,10 @@ async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
     message, original_message = await load_updated_message(vm_hash)
     pool.message_cache[vm_hash] = message
 
+    logger.debug(
+        f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}"
+    )
+
     try:
         execution = await pool.create_a_vm(
             vm_hash=vm_hash,
@@ -222,7 +226,7 @@ async def start_persistent_vm(vm_hash: ItemHash, pubsub: PubSub) -> VmExecution:
     execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
-        logger.info(f"Starting persistent VM {vm_hash}")
+        logger.info(f"Starting persistent virtual machine with id: {vm_hash}")
         execution = await create_vm_execution(vm_hash=vm_hash)
     # If the VM was already running in lambda mode, it should not expire
     # as long as it is also scheduled as long-running
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index f8b2d6b01..e293164c7 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -4,7 +4,6 @@
 In this prototype, it returns a hardcoded example.
 In the future, it should connect to an Aleph node and retrieve the code from there.
 """
-import hashlib
 import json
 import logging
 import re
@@ -14,12 +13,7 @@
 from typing import Union
 
 import aiohttp
-from aleph_message.models import (
-    ExecutableMessage,
-    InstanceMessage,
-    MessageType,
-    ProgramMessage,
-)
+from aleph_message.models import InstanceMessage, ProgramMessage, parse_message
 from aleph_message.models.execution.instance import RootfsVolume
 from aleph_message.models.execution.program import Encoding
 from aleph_message.models.execution.volume import (
@@ -30,7 +24,7 @@
 )
 
 from .conf import settings
-from .utils import run_in_subprocess
+from .utils import fix_message_validation, run_in_subprocess
 
 logger = logging.getLogger(__name__)
 
@@ -69,6 +63,9 @@ async def download_file(url: str, local_path: Path) -> None:
                         sys.stdout.write(".")
                         sys.stdout.flush()
 
+            sys.stdout.write("\n")
+            sys.stdout.flush()
+
             tmp_path.rename(local_path)
             logger.debug(f"Download complete, moved {tmp_path} -> {local_path}")
         except Exception:
@@ -90,8 +87,11 @@ async def get_latest_amend(item_hash: str) -> str:
             return result or item_hash
 
 
-async def get_message(ref: str) -> ExecutableMessage:
-    if settings.FAKE_DATA_PROGRAM:
+async def get_message(ref: str) -> Union[ProgramMessage, InstanceMessage]:
+    if ref == settings.FAKE_INSTANCE_ID:
+        logger.debug("Using the fake instance message since the ref matches")
+        cache_path = settings.FAKE_INSTANCE_MESSAGE
+    elif settings.FAKE_DATA_PROGRAM:
         cache_path = settings.FAKE_DATA_MESSAGE
     else:
         cache_path = (Path(settings.MESSAGE_CACHE) / ref).with_suffix(".json")
@@ -100,14 +100,16 @@ async def get_message(ref: str) -> ExecutableMessage:
 
     with open(cache_path, "r") as cache_file:
         msg = json.load(cache_file)
-        if settings.FAKE_DATA_PROGRAM:
-            msg["item_content"] = json.dumps(msg["content"])
-            msg["item_hash"] = hashlib.sha256(
-                msg["item_content"].encode("utf-8")
-            ).hexdigest()
-        if msg["type"] == MessageType.program:
-            return ProgramMessage.parse_obj(msg)
-        return InstanceMessage.parse_obj(msg)
+
+        if cache_path in (settings.FAKE_DATA_MESSAGE, settings.FAKE_INSTANCE_MESSAGE):
+            # Ensure validation passes while tweaking message content
+            msg = fix_message_validation(msg)
+
+        result = parse_message(message_dict=msg)
+        assert isinstance(result, ProgramMessage) or isinstance(
+            result, InstanceMessage
+        ), "Parsed message is not executable"
+        return result
 
 
 async def get_code_path(ref: str) -> Path:
@@ -164,7 +166,7 @@ async def get_runtime_path(ref: str) -> Path:
 
 async def create_ext4(path: Path, size_mib: int) -> bool:
     if path.is_file():
-        logger.debug("File already exists, skipping ext4 creation", path)
+        logger.debug(f"File already exists, skipping ext4 creation on {path}")
         return False
     tmp_path = f"{path}.tmp"
     await run_in_subprocess(
@@ -183,6 +185,9 @@ async def create_volume_file(
     path = Path(settings.PERSISTENT_VOLUMES_DIR) / namespace / f"{volume_name}.ext4"
     if not path.is_file():
         logger.debug(f"Creating {volume.size_mib}MB volume")
+        # Ensure that the parent directory exists
+        path.parent.mkdir(exist_ok=True)
+        # Create an empty file the right size
         await run_in_subprocess(
             ["dd", "if=/dev/zero", f"of={path}", "bs=1M", f"count={volume.size_mib}"]
         )
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index 9da13e1a6..43376361c 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -1,11 +1,12 @@
 import asyncio
+import hashlib
 import json
 import logging
 import subprocess
 from base64 import b16encode, b32decode
 from dataclasses import asdict as dataclass_as_dict
 from dataclasses import is_dataclass
-from typing import Any, Coroutine, List, Optional
+from typing import Any, Coroutine, Dict, List, Optional
 
 import aiodns
 
@@ -68,8 +69,23 @@ async def run_in_subprocess(
     stdout, stderr = await process.communicate(input=stdin_input)
 
     if check and process.returncode:
+        logger.error(
+            f"Command failed with error code {process.returncode}:\n"
+            f"    stdin = {stdin_input!r}\n"
+            f"    command = {command}\n"
+            f"    stdout = {stderr!r}"
+        )
         raise subprocess.CalledProcessError(
             process.returncode, str(command), stderr.decode()
         )
 
     return stdout
+
+
+def fix_message_validation(message: Dict) -> Dict:
+    """Patch a fake message program to pass validation."""
+    message["item_content"] = json.dumps(message["content"])
+    message["item_hash"] = hashlib.sha256(
+        message["item_content"].encode("utf-8")
+    ).hexdigest()
+    return message

From 409f78c69e8b4ec070afcd8d8d399dee9031ed52 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Tue, 20 Jun 2023 16:09:41 +0200
Subject: [PATCH 401/990] Fix: When the users runs a test instance, it cannot
 use fake runtime Solution: Update general settings adding FAKE_DATA_INSTANCE
 variable to use the fake rootfs or the real one.

---
 vm_supervisor/__main__.py | 3 +++
 vm_supervisor/conf.py     | 1 +
 vm_supervisor/storage.py  | 2 +-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index d739238d4..15fe6204b 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -309,6 +309,9 @@ def main():
     elif args.run_test_instance:
         logger.info("Running test instance virtual machine")
         instance_item_hash = settings.FAKE_INSTANCE_ID
+        settings.update(
+            FAKE_DATA_INSTANCE=True,
+        )
 
         async def forever():
             await run_instance(item_hash=instance_item_hash)
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 9aa1f3511..0706769f1 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -124,6 +124,7 @@ class Settings(BaseSettings):
     )
 
     FAKE_DATA_PROGRAM: Optional[Path] = None
+    FAKE_DATA_INSTANCE: Optional[bool] = False
     BENCHMARK_FAKE_DATA_PROGRAM = Path(
         abspath(join(__file__, "../../examples/example_fastapi"))
     )
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index e293164c7..fe1176ea1 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -154,7 +154,7 @@ async def get_data_path(ref: str) -> Path:
 
 
 async def get_runtime_path(ref: str) -> Path:
-    if settings.FAKE_DATA_PROGRAM:
+    if settings.FAKE_DATA_PROGRAM or settings.FAKE_DATA_INSTANCE:
         return Path(settings.FAKE_DATA_RUNTIME)
 
     cache_path = Path(settings.RUNTIME_CACHE) / ref

From 9facab358253880cd23ee0b4f85d18656c03ff85 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 19 Jun 2023 13:39:57 +0200
Subject: [PATCH 402/990] Fix: Support of Ubuntu 20.04 was broken due to
 NFTables

Incompatibilities in the version of NFTables prevent the current codebase from working correctly on that system.

Since the newer LTS, Ubuntu 22.04, shows no issue and many node operators upgraded to that system without any problem, we decided to drop support for Ubuntu 20.04.
---
 .github/workflows/build-deb-package.yml       |  22 ---
 README.md                                     |   2 +-
 doc/INSTALL-Debian-11.md                      |   2 +-
 doc/INSTALL-Ubuntu-20.04.md                   | 176 +-----------------
 doc/INSTALL.md                                |   2 +-
 packaging/Makefile                            |  28 +--
 .../{focal => jammy}/conf/distributions       |   2 +-
 packaging/ubuntu-20.04.dockerfile             |  18 --
 tutorials/README.md                           |   2 +-
 vm_supervisor/README.md                       |   2 +-
 vm_supervisor/conf.py                         |   2 +-
 11 files changed, 19 insertions(+), 239 deletions(-)
 rename packaging/repositories/{focal => jammy}/conf/distributions (93%)
 delete mode 100644 packaging/ubuntu-20.04.dockerfile

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index efc6516aa..5e552dca8 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -25,28 +25,6 @@ jobs:
           name: aleph-vm.debian-11.deb
           path: packaging/target/aleph-vm.debian-11.deb
 
-  build_deb_ubuntu_20_04:
-    name: "Build Debian Package"
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Unshallow
-        run: |
-          git fetch --prune --unshallow
-          git describe --tags
-
-      - run: |
-          cd packaging && make all-podman-ubuntu-2004 && cd ..
-          ls packaging/target
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: aleph-vm.ubuntu-20.04.deb
-          path: packaging/target/aleph-vm.ubuntu-20.04.deb
-
   build_deb_ubuntu_22_04:
     name: "Build Debian Package"
     runs-on: ubuntu-latest
diff --git a/README.md b/README.md
index ac3253a7c..18854c83f 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@ Writing programs in Python using ASGI compatible frameworks (
 Install Aleph-VM to run an Aleph.im Compute Resource Node easily from official pre-built packages.
 
 - [On Debian 11](./doc/INSTALL-Debian-11.md)
-- [On Ubuntu 20.04](./doc/INSTALL-Ubuntu-20.04.md)
+- [On Ubuntu 22.04](./doc/INSTALL-Ubuntu-22.04.md)
 
 ## 2. Install Aleph-VM from source
 
diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index 23f146eb7..5d2d3f00c 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.5/aleph-vm.debian-11.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.7/aleph-vm.debian-11.deb
 apt install /opt/aleph-vm.debian-11.deb
 ```
 
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
index 239645171..0cd067b36 100644
--- a/doc/INSTALL-Ubuntu-20.04.md
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -1,170 +1,10 @@
-# Installing Aleph-VM on a server / Ubuntu 20.04 Focal Fossa
+# Installing Aleph-VM on a server / Ubuntu 20.04 Focal Fossa (Deprecated) 
 
-## 0. Introduction
+Support for Ubuntu 20.04 was due to compatibility issues with
+the NFTables firewall introduced in version 
+[0.2.6](https://github.com/aleph-im/aleph-vm/releases/tag/0.2.6).
 
-For production using official Debian packages.
-
-## 1. Requirements
-
-- A [supported Linux server](../vm_supervisor/README.md#1-supported-platforms)
-- A public domain name from a registrar and top level domain you trust. 
-
-In order to run an official Aleph.im Compute Resource Node (CRN), you will also need the following resources:
-
-- CPU (2 options):
-  - Min. 8 cores / 16 threads, 3.0 ghz+ CPU (gaming CPU for fast boot-up of microVMs)
-  - Min. 12 core / 24 threads, 2.4ghz+ CPU (datacenter CPU for multiple concurrent loads)
-- RAM: 64GB
-- STORAGE: 1TB (NVMe SSD preferred, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
-- BANDWIDTH: Minimum of 500 MB/s
-
-You will need a public domain name with access to add TXT and wildcard records.
-
-> 💡 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
-
-## 2. Installation
-
-Run the following commands:
-
-First install the [VM-Connector](../vm_connector/README.md) using Docker:
-```shell
-sudo apt update
-sudo apt upgrade
-sudo apt install -y docker.io
-docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
-```
-
-Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
-The procedure is similar for updates.
-```shell
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.5/aleph-vm.ubuntu-20.04.deb
-sudo apt install /opt/aleph-vm.ubuntu-20.04.deb
-```
-
-Reboot if required (new kernel, ...).
-
-### Configuration
-
-#### Hostname
-
-Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
-
-You will want to insert your domain name in the form of:
-```
-ALEPH_VM_DOMAIN_NAME=vm.example.org
-```
-
-#### Network configuration
-
-Ubuntu 20.04 by default uses [systemd-resolved](https://manpages.ubuntu.com/manpages/focal/man8/systemd-resolved.service.8.html)
-for DNS resolution. The following setting configures the VM Supervisor to use it instead of reading the default `/etc/resolv.conf`.
-```
-ALEPH_VM_DNS_RESOLUTION=resolvectl
-```
-
-> 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
-
-On some systems, the default network interface is not `eth0` and you will want to configure the default interface
-by adding:
-```
-ALEPH_VM_NETWORK_INTERFACE=enp0s1
-```
-(don't forget to replace `enp0s1` with the name of your default network interface).
-
-#### Volumes and partitions
-
-Two directories are used to store data from the network:
-- `/var/lib/aleph/vm` contains all the execution and persistent data.
-- `/var/cache/aleph/vm` contains data downloaded from the network.
-
-These two directories must be stored on the same partition.
-That partition must meet the minimum requirements specified for a CRN.
-
-> 💡 This is required due to the software using hard links to optimize performance and disk usage.
-
-#### Applying changes
-
-Finally, restart the service:
-```shell
-sudo systemctl restart aleph-vm-supervisor
-```
-
-## 3. Reverse Proxy
-
-We document how to use Caddy as a reverse proxy since it manages and renews HTTPS certificates automatically.
-
-Any other reverse-proxy (Nginx, HAProxy, Apache2, ...) should do the job as well, just make sure to renew the 
-HTTPS/TLS certificates on time.
-
-First, create a domain name that points to the server on IPv4 (A) and IPv6 (AAAA).
-
-This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CONFIGURE_CADDY.md).
-
-Again, run these commands as `root`:
-```shell
-sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list
-sudo apt update
-sudo apt install caddy
-```
-
-Then, after replacing the domain `vm.example.org` with your own, use configure Caddy:
-```shell
-sudo cat >/etc/caddy/Caddyfile < ![image](https://user-images.githubusercontent.com/404665/150202090-91a02536-4e04-4af2-967f-fe105d116e1f.png)
-
-If you face an issue, check the logs of the different services for errors:
-
-VM-Supervisor:
-```shell
-sudo journalctl -f -u aleph-vm-supervisor.service 
-```
-
-Caddy:
-```shell
-sudo journalctl -f -u caddy.service 
-```
-
-VM-Connector:
-```shell
-sudo docker logs -f vm-connector
-```
-
-### Common errors
-
-#### "Network interface eth0 does not exist"
-
-Did you update the configuration file `/etc/aleph-vm/supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
-the default network interface of your server ?
-
-#### "Aleph Connector unavailable"
-
-Investigate the installation of the VM-Connector using Docker in step 2.
+We recommend upgrading to the newest Ubuntu LTS version 
+and then use the 
+[following instructions on Ubuntu 22.04](./INSTALL-Ubuntu-22.04.md)
+).
diff --git a/doc/INSTALL.md b/doc/INSTALL.md
index 3ede0680a..79e8d18cf 100644
--- a/doc/INSTALL.md
+++ b/doc/INSTALL.md
@@ -1,4 +1,4 @@
 # Installing Aleph-VM
 
 - [On Debian 11](./INSTALL-Debian-11.md)
-- [On Ubuntu 20.04](./INSTALL-Ubuntu-20.04.md)
+- [On Ubuntu 22.04](./INSTALL-Ubuntu-22.04.md)
diff --git a/packaging/Makefile b/packaging/Makefile
index 08d26f501..4bd076703 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -68,17 +68,6 @@ all-podman-debian-11: version
 	file target/aleph-vm.deb
 	mv target/aleph-vm.deb target/aleph-vm.debian-11.deb
 
-all-podman-ubuntu-2004: version
-	cd .. && podman build -t localhost/aleph-vm-packaging-ubuntu-2004:latest -f ./packaging/ubuntu-20.04.dockerfile .
-	mkdir -p ./target
-	podman run --rm -ti \
-		-w /opt/packaging \
-		-v ./target:/opt/packaging/target \
-		localhost/aleph-vm-packaging-ubuntu-2004:latest \
-		make
-	file target/aleph-vm.deb
-	mv target/aleph-vm.deb target/aleph-vm.ubuntu-20.04.deb
-
 all-podman-ubuntu-2204: version
 	cd .. && podman build -t localhost/aleph-vm-packaging-ubuntu-2204:latest -f ./packaging/ubuntu-22.04.dockerfile .
 	mkdir -p ./target
@@ -99,15 +88,6 @@ requirements-debian-11: all-podman-debian-11
 		debian:bullseye \
 		bash -c "/opt/extract_requirements.sh /mnt/requirements-debian-11.txt"
 
-# extract Python requirements from Ubuntu 20.04 container
-requirements-ubuntu-20.04: all-podman-ubuntu-2004
-	podman run --rm -ti \
-		-v ./target/aleph-vm.ubuntu-20.04.deb:/opt/packaging/target/aleph-vm.deb:ro \
-		-v ./extract_requirements.sh:/opt/extract_requirements.sh:ro \
-		-v ./requirements-ubuntu-20.04.txt:/mnt/requirements-ubuntu-20.04.txt \
-		ubuntu:focal \
-		bash -c "/opt/extract_requirements.sh /mnt/requirements-ubuntu-20.04.txt"
-
 # extract Python requirements from Ubuntu 22.04 container
 requirements-ubuntu-22.04: all-podman-ubuntu-2204
 	podman run --rm -ti \
@@ -122,9 +102,9 @@ repository-bullseye:
 	cd ./repositories/bullseye && reprepro -Vb . includedeb bullseye ../../target/aleph-vm.debian-11.deb && cd ..
 
 # run on host in order to sign with GPG
-repository-focal:
-	cd ./repositories/focal && reprepro -Vb . includedeb focal ../../target/aleph-vm.ubuntu-20.04.deb && cd ..
+repository-jammy:
+	cd ./repositories/jammy && reprepro -Vb . includedeb jammy ../../target/aleph-vm.ubuntu-22.04.deb && cd ..
 
-repositories: repository-bullseye repository-focal
+repositories: repository-bullseye repository-jammy
 
-all-podman: all-podman-debian-11 all-podman-ubuntu-2004 repositories
+all-podman: all-podman-debian-11 all-podman-ubuntu-2204 repositories
diff --git a/packaging/repositories/focal/conf/distributions b/packaging/repositories/jammy/conf/distributions
similarity index 93%
rename from packaging/repositories/focal/conf/distributions
rename to packaging/repositories/jammy/conf/distributions
index 577ba950d..2d5872786 100644
--- a/packaging/repositories/focal/conf/distributions
+++ b/packaging/repositories/jammy/conf/distributions
@@ -1,7 +1,7 @@
 Origin: Aleph-IM
 Label: aleph-im
 Suite: stable
-Codename: focal
+Codename: jammy
 Version: 3.0
 Architectures: amd64 source
 Components: contrib
diff --git a/packaging/ubuntu-20.04.dockerfile b/packaging/ubuntu-20.04.dockerfile
deleted file mode 100644
index 794003824..000000000
--- a/packaging/ubuntu-20.04.dockerfile
+++ /dev/null
@@ -1,18 +0,0 @@
-FROM ubuntu:20.04
-
-RUN apt-get update && apt-get -y upgrade && apt-get install -y \
-    make \
-    git \
-    curl \
-    sudo \
-    python3-pip \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /opt
-COPY ../vm_supervisor ./vm_supervisor
-COPY ../guest_api ./guest_api
-COPY ../firecracker ./firecracker
-COPY ../packaging ./packaging
-COPY ../kernels ./kernels
-
-COPY ../examples/ ./examples
diff --git a/tutorials/README.md b/tutorials/README.md
index cf1b87dc6..291bd7ed7 100644
--- a/tutorials/README.md
+++ b/tutorials/README.md
@@ -29,7 +29,7 @@ simple API and the `uvicorn` server to test your program on your desktop before
 Aleph.
 
 First, you need a recent version of Python and [pip](https://pip.pypa.io/en/stable/), 
-preferably running on Debian 11 or Ubuntu Linux 20.04 since we have not tested other platforms yet, 
+preferably running on Debian 11 or Ubuntu Linux 22.04 since we have not tested other platforms yet, 
 but feel free to use the platform of your choice if you have the skills to adapt our instructions to it.
 
 Some cryptographic functionalities of Aleph use curve secp256k1 and require installing [libsecp256k1](https://github.com/bitcoin-core/secp256k1).
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index 846854591..3178dacc1 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -29,7 +29,7 @@ Intel Skylake, Intel Cascade Lake, AMD Zen2 and ARM64 Neoverse N1.
 ### Operating System
 
 These instructions have been tested on Debian 11 Bullseye, and should work on recent versions
-of Ubuntu as well (20.04+).
+of Ubuntu as well (22.04+).
 
 ### Hosting providers
 
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 0706769f1..a1adee0fe 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -32,7 +32,7 @@ def resolvectl_dns_servers(interface: str) -> Iterable[str]:
     """
     Use resolvectl to list available DNS servers (IPv4 and IPv6).
 
-    Note: we used to use systemd-resolve for Ubuntu 20.04 and Debian.
+    Note: we used to use systemd-resolve for Debian 11.
     This command is not available anymore on Ubuntu 22.04 and is actually a symlink
     to resolvectl.
 

From 7c3409c9798bd5d5a57e659c71c2d43bf0a1ea06 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 19 Jun 2023 13:42:29 +0200
Subject: [PATCH 403/990] Fix: Support of Ubuntu 20.04 was broken due to
 NFTables

Incompatibilities in the version of NFTables prevent the current codebase from working correctly on that system.

Since the newer LTS, Ubuntu 22.04, shows no issue and many node operators upgraded to that system without any problem, we decided to drop support for Ubuntu 20.04.
---
 doc/INSTALL-Ubuntu-22.04.md | 170 ++++++++++++++++++++++++++++++++++++
 1 file changed, 170 insertions(+)
 create mode 100644 doc/INSTALL-Ubuntu-22.04.md

diff --git a/doc/INSTALL-Ubuntu-22.04.md b/doc/INSTALL-Ubuntu-22.04.md
new file mode 100644
index 000000000..a7969807b
--- /dev/null
+++ b/doc/INSTALL-Ubuntu-22.04.md
@@ -0,0 +1,170 @@
+# Installing Aleph-VM on a server / Ubuntu 22.04 Jammy Jellyfish
+
+## 0. Introduction
+
+For production using official Debian packages.
+
+## 1. Requirements
+
+- A [supported Linux server](../vm_supervisor/README.md#1-supported-platforms)
+- A public domain name from a registrar and top level domain you trust. 
+
+In order to run an official Aleph.im Compute Resource Node (CRN), you will also need the following resources:
+
+- CPU (2 options):
+  - Min. 8 cores / 16 threads, 3.0 ghz+ CPU (gaming CPU for fast boot-up of microVMs)
+  - Min. 12 core / 24 threads, 2.4ghz+ CPU (datacenter CPU for multiple concurrent loads)
+- RAM: 64GB
+- STORAGE: 1TB (NVMe SSD preferred, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
+- BANDWIDTH: Minimum of 500 MB/s
+
+You will need a public domain name with access to add TXT and wildcard records.
+
+> 💡 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
+
+## 2. Installation
+
+Run the following commands:
+
+First install the [VM-Connector](../vm_connector/README.md) using Docker:
+```shell
+sudo apt update
+sudo apt upgrade
+sudo apt install -y docker.io
+docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
+```
+
+Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
+The procedure is similar for updates.
+```shell
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.7/aleph-vm.ubuntu-22.04.deb
+sudo apt install /opt/aleph-vm.ubuntu-22.04.deb
+```
+
+Reboot if required (new kernel, ...).
+
+### Configuration
+
+#### Hostname
+
+Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
+
+You will want to insert your domain name in the form of:
+```
+ALEPH_VM_DOMAIN_NAME=vm.example.org
+```
+
+#### Network configuration
+
+Ubuntu 22.04 by default uses [systemd-resolved](https://manpages.ubuntu.com/manpages/jammy/man8/systemd-resolved.service.8.html)
+for DNS resolution. The following setting configures the VM Supervisor to use it instead of reading the default `/etc/resolv.conf`.
+```
+ALEPH_VM_DNS_RESOLUTION=resolvectl
+```
+
+> 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
+
+On some systems, the default network interface is not `eth0` and you will want to configure the default interface
+by adding:
+```
+ALEPH_VM_NETWORK_INTERFACE=enp0s1
+```
+(don't forget to replace `enp0s1` with the name of your default network interface).
+
+#### Volumes and partitions
+
+Two directories are used to store data from the network:
+- `/var/lib/aleph/vm` contains all the execution and persistent data.
+- `/var/cache/aleph/vm` contains data downloaded from the network.
+
+These two directories must be stored on the same partition.
+That partition must meet the minimum requirements specified for a CRN.
+
+> 💡 This is required due to the software using hard links to optimize performance and disk usage.
+
+#### Applying changes
+
+Finally, restart the service:
+```shell
+sudo systemctl restart aleph-vm-supervisor
+```
+
+## 3. Reverse Proxy
+
+We document how to use Caddy as a reverse proxy since it manages and renews HTTPS certificates automatically.
+
+Any other reverse-proxy (Nginx, HAProxy, Apache2, ...) should do the job as well, just make sure to renew the 
+HTTPS/TLS certificates on time.
+
+First, create a domain name that points to the server on IPv4 (A) and IPv6 (AAAA).
+
+This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CONFIGURE_CADDY.md).
+
+Again, run these commands as `root`:
+```shell
+sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list
+sudo apt update
+sudo apt install caddy
+```
+
+Then, after replacing the domain `vm.example.org` with your own, use configure Caddy:
+```shell
+sudo cat >/etc/caddy/Caddyfile < ![image](https://user-images.githubusercontent.com/404665/150202090-91a02536-4e04-4af2-967f-fe105d116e1f.png)
+
+If you face an issue, check the logs of the different services for errors:
+
+VM-Supervisor:
+```shell
+sudo journalctl -f -u aleph-vm-supervisor.service 
+```
+
+Caddy:
+```shell
+sudo journalctl -f -u caddy.service 
+```
+
+VM-Connector:
+```shell
+sudo docker logs -f vm-connector
+```
+
+### Common errors
+
+#### "Network interface eth0 does not exist"
+
+Did you update the configuration file `/etc/aleph-vm/supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
+the default network interface of your server ?
+
+#### "Aleph Connector unavailable"
+
+Investigate the installation of the VM-Connector using Docker in step 2.

From 1f2ed276a583943611614539081c630c093f43a1 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Fri, 23 Jun 2023 10:21:24 +0200
Subject: [PATCH 404/990] Implement wait_for_init method on instances (#329)

Fix: Instances missed 'wait_for_init' implementation

Solution: Implement a method to ping a machine and use it on wait_for_init method on instances.

Co-authored-by: Hugo Herter 
---
 vm_supervisor/utils.py                   | 17 +++++++++++++++++
 vm_supervisor/vm/firecracker/instance.py | 22 ++++++++++++++++++++--
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index 43376361c..e4a91ff65 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -89,3 +89,20 @@ def fix_message_validation(message: Dict) -> Dict:
         message["item_content"].encode("utf-8")
     ).hexdigest()
     return message
+
+
+class HostNotFoundError(Exception):
+    pass
+
+
+async def ping(host: str, packets: int, timeout: int):
+    """
+    Waits for a host to respond to a ping request.
+    """
+
+    try:
+        await run_in_subprocess(
+            ["ping", "-c", str(packets), "-W", str(timeout), host], check=True
+        )
+    except subprocess.CalledProcessError as err:
+        raise HostNotFoundError() from err
diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index 2d85b9898..ca6458a84 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -1,5 +1,6 @@
 import asyncio
 import logging
+import subprocess
 from pathlib import Path
 from typing import Optional
 
@@ -17,6 +18,7 @@
 from firecracker.microvm import setfacl
 from vm_supervisor.network.interfaces import TapInterface
 from vm_supervisor.storage import create_devmapper
+from vm_supervisor.utils import ping, HostNotFoundError
 
 from .executable import (
     AlephFirecrackerExecutable,
@@ -110,8 +112,24 @@ async def setup(self):
 
     async def wait_for_init(self) -> None:
         """Wait for the init process of the instance to be ready."""
-        # TODO: Check availability via ping ?
-        return
+        if not self.vm_configuration:
+            raise ValueError("The VM has not been configured yet")
+
+        if not self.vm_configuration.ip:
+            raise ValueError("VM IP address not set")
+
+        attempts = 5
+        timeout_seconds = 1.0
+
+        for attempt in range(attempts):
+            try:
+                await ping(self.vm_configuration.ip, packets=1, timeout=timeout_seconds)
+                return
+            except HostNotFoundError:
+                if attempt < (attempts - 1):
+                    continue
+                else:
+                    raise
 
     async def configure(self):
         """Configure the VM by sending configuration info to it's init"""

From fab40f0824459d4da72efddfaa7329eb81e30a7a Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Wed, 21 Jun 2023 22:59:36 +0200
Subject: [PATCH 405/990] CI: use CodeQL action v2

Problem: v1 is deprecated since 18/1/2023.
---
 .github/workflows/codeql-analysis.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 28c490c57..deeb02846 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -43,7 +43,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

From 5909cf866e638e4ba5514413c21a493912334cc7 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Wed, 21 Jun 2023 23:15:45 +0200
Subject: [PATCH 406/990] CI: fix mypy by upgrading typing-extensions

---
 .github/workflows/code-quality.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
index 0f0bcf823..32434d0a0 100644
--- a/.github/workflows/code-quality.yml
+++ b/.github/workflows/code-quality.yml
@@ -17,6 +17,7 @@ jobs:
           sudo apt-get update
           sudo apt-get -y upgrade
           sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema
+          pip install --upgrade typing-extensions
 
       - name: Install required Python packages
         run: |

From 43a44601d936d74ccde61c82894401d9b515b286 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Tue, 20 Jun 2023 16:32:13 +0200
Subject: [PATCH 407/990] Feature: Instances could not use cloud-init

Problem: Setup an instance using a cloud OS image file was impossible without a custom init program.

Solution: Use the cloud-init tool already integrated in many Linux images for cloud distributions to setup the network and system configuration.

Co-authored-by: Hugo Herter 
---
 examples/instance_message_from_aleph.json  | 13 ++--
 kernels/microvm-kernel-x86_64-5.10.config  |  2 +-
 vm_supervisor/vm/firecracker/executable.py |  6 ++
 vm_supervisor/vm/firecracker/instance.py   | 84 +++++++++++++++++++++-
 vm_supervisor/vm/firecracker/program.py    |  4 +-
 5 files changed, 97 insertions(+), 12 deletions(-)

diff --git a/examples/instance_message_from_aleph.json b/examples/instance_message_from_aleph.json
index 8302d68c7..c2218fbaa 100644
--- a/examples/instance_message_from_aleph.json
+++ b/examples/instance_message_from_aleph.json
@@ -19,7 +19,7 @@
         },
         "resources": {
             "vcpus": 1,
-            "memory": 128,
+            "memory": 512,
             "seconds": 30
         },
         "rootfs": {
@@ -28,14 +28,11 @@
                 "use_latest": true
             },
             "persistence": "host",
-            "size_mib": 20000
-        },
-        "cloud_config": {
-            "password": "password",
-            "chpasswd": {
-                "expire": "False"
-            }
+            "size_mib": 5000
         },
+        "authorized_keys": [
+            "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHlGJRaIv/EzNT0eNqNB5DiGEbii28Fb2zCjuO/bMu7y amolinsdiaz@gmail.com"
+        ],
         "volumes": [
             {
                 "mount": "/opt/venv",
diff --git a/kernels/microvm-kernel-x86_64-5.10.config b/kernels/microvm-kernel-x86_64-5.10.config
index 411e36c27..1f4df1c62 100644
--- a/kernels/microvm-kernel-x86_64-5.10.config
+++ b/kernels/microvm-kernel-x86_64-5.10.config
@@ -2156,7 +2156,7 @@ CONFIG_OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW=y
 #
 # CD-ROM/DVD Filesystems
 #
-# CONFIG_ISO9660_FS is not set
+CONFIG_ISO9660_FS=y
 # CONFIG_UDF_FS is not set
 # end of CD-ROM/DVD Filesystems
 
diff --git a/vm_supervisor/vm/firecracker/executable.py b/vm_supervisor/vm/firecracker/executable.py
index 4f6944895..3421ae248 100644
--- a/vm_supervisor/vm/firecracker/executable.py
+++ b/vm_supervisor/vm/firecracker/executable.py
@@ -182,6 +182,12 @@ def __init__(
         self.guest_api_process = None
         self._firecracker_config = None
 
+    def get_vm_ip(self):
+        return self.tap_interface.guest_ip.with_prefixlen
+
+    def get_vm_route(self):
+        return str(self.tap_interface.host_ip).split("/", 1)[0]
+
     def to_dict(self):
         """Dict representation of the virtual machine. Used to record resource usage and for JSON serialization."""
         if self.fvm.proc and psutil:
diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index ca6458a84..31e03823a 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -2,8 +2,10 @@
 import logging
 import subprocess
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import Optional
 
+import yaml
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
 
@@ -16,10 +18,12 @@
     Vsock,
 )
 from firecracker.microvm import setfacl
+from vm_supervisor.conf import settings
 from vm_supervisor.network.interfaces import TapInterface
 from vm_supervisor.storage import create_devmapper
 from vm_supervisor.utils import ping, HostNotFoundError
 
+from ...utils import run_in_subprocess
 from .executable import (
     AlephFirecrackerExecutable,
     AlephFirecrackerResources,
@@ -75,6 +79,8 @@ async def setup(self):
         logger.debug("instance setup started")
         await setfacl()
 
+        cloud_init_drive = await self._create_cloud_init_drive()
+
         self._firecracker_config = FirecrackerConfig(
             boot_source=BootSource(
                 kernel_image_path=Path(
@@ -91,6 +97,7 @@ async def setup(self):
                     is_root_device=True,
                     is_read_only=False,
                 ),
+                cloud_init_drive,
             ]
             + [
                 self.fvm.enable_drive(volume.path_on_host, read_only=volume.read_only)
@@ -133,5 +140,80 @@ async def wait_for_init(self) -> None:
 
     async def configure(self):
         """Configure the VM by sending configuration info to it's init"""
-        # TODO: Implement Cloud-init interface
+        # Configuration of instances is sent during `self.setup()` by passing it via a volume.
         pass
+
+    def _encode_user_data(self) -> bytes:
+        """Creates user data configuration file for cloud-init tool"""
+
+        ssh_authorized_keys = self.resources.message_content.authorized_keys or []
+
+        config = {
+            "hostname": self.vm_hash,
+            "disable_root": False,
+            "ssh_pwauth": False,
+            "ssh_authorized_keys": ssh_authorized_keys,
+        }
+
+        cloud_config_header = "#cloud-config\n"
+        config_output = yaml.safe_dump(
+            config, default_flow_style=False, sort_keys=False
+        )
+
+        return (cloud_config_header + config_output).encode()
+
+    def _create_network_file(self) -> bytes:
+        """Creates network configuration file for cloud-init tool"""
+
+        assert (
+            self.enable_networking and self.tap_interface
+        ), f"Network not enabled for VM {self.vm_id}"
+
+        ip = self.get_vm_ip()
+        route = self.get_vm_route()
+
+        network = {
+            "network": {
+                "ethernets": {
+                    "eth0": {
+                        "dhcp4": False,
+                        "dhcp6": False,
+                        "addresses": [str(ip)],
+                        "gateway4": route,
+                        "nameservers": {
+                            "addresses": settings.DNS_NAMESERVERS,
+                        },
+                    },
+                },
+                "version": 2,
+            },
+        }
+
+        return yaml.safe_dump(
+            network, default_flow_style=False, sort_keys=False
+        ).encode()
+
+    async def _create_cloud_init_drive(self) -> Drive:
+        """Creates the cloud-init volume to configure and setup the VM"""
+
+        disk_image_path = settings.EXECUTION_ROOT / f"cloud-init-{self.vm_hash}.img"
+
+        with NamedTemporaryFile() as main_config_file:
+            user_data = self._encode_user_data()
+            main_config_file.write(user_data)
+            main_config_file.flush()
+            with NamedTemporaryFile() as network_config_file:
+                network_config = self._create_network_file()
+                network_config_file.write(network_config)
+                network_config_file.flush()
+
+                await run_in_subprocess(
+                    [
+                        "cloud-localds",
+                        f"--network-config={network_config_file.name}",
+                        str(disk_image_path),
+                        main_config_file.name,
+                    ]
+                )
+
+        return self.fvm.enable_drive(disk_image_path, read_only=True)
diff --git a/vm_supervisor/vm/firecracker/program.py b/vm_supervisor/vm/firecracker/program.py
index 79a79a54e..257f04a29 100644
--- a/vm_supervisor/vm/firecracker/program.py
+++ b/vm_supervisor/vm/firecracker/program.py
@@ -319,8 +319,8 @@ async def _setup_configuration(
         # The ip and route should not contain the network mask in order to maintain
         # compatibility with the existing runtimes.
         if self.enable_networking and self.tap_interface:
-            ip = self.tap_interface.guest_ip.with_prefixlen.split("/", 1)[0]
-            route = str(self.tap_interface.host_ip).split("/", 1)[0]
+            ip = self.get_vm_ip().split("/", 1)[0]
+            route = self.get_vm_route()
         else:
             ip, route = None, None
 

From 9dd67de02a82841622215bca84048f12ac4a9bf4 Mon Sep 17 00:00:00 2001
From: "Alie.E" 
Date: Fri, 23 Jun 2023 13:22:05 +0200
Subject: [PATCH 408/990] Fix: Programs could not set multiple cookies

Problem: Setting cookies requires multiple headers with the same name.

Solution: Use a CIMultiDict instead of a normal dict.
---
 vm_supervisor/run.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index d05b09eeb..68c46355f 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -3,10 +3,12 @@
 from typing import Any, Dict, Optional
 
 import msgpack
+from msgpack import UnpackValueError
+
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError
+from multidict import CIMultiDict
 from aleph_message.models import ItemHash
-from msgpack import UnpackValueError
 
 from firecracker.microvm import MicroVMFailedInit
 
@@ -138,9 +140,10 @@ async def run_code_on_request(
                 content_type="text/plain",
             )
 
-        headers = {
-            key.decode(): value.decode() for key, value in result["headers"]["headers"]
-        }
+        # HTTP Headers require specific data structure
+        headers = CIMultiDict([
+            (key.decode().lower(), value.decode()) for key, value in result["headers"]["headers"]
+        ])
         if "content-length" not in headers:
             headers["Content-Length".lower()] = str(len(result["body"]["body"]))
         for header in ["Content-Encoding", "Transfer-Encoding", "Vary"]:

From 2115cd08bab5bf6cdc7edb8e7e6513826fc670a3 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 23 Jun 2023 14:59:25 +0200
Subject: [PATCH 409/990] Cleanup: File init1.py was not formatted with
 black+isort

---
 runtimes/aleph-debian-11-python/init1.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index ed986b449..bdb557c2b 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -10,20 +10,20 @@
 
 logger.debug("Imports starting")
 
-import ctypes
 import asyncio
+import ctypes
 import os
 import socket
-from enum import Enum
 import subprocess
 import sys
 import traceback
 from contextlib import redirect_stdout
 from dataclasses import dataclass, field
+from enum import Enum
 from io import StringIO
 from os import system
 from shutil import make_archive
-from typing import Optional, Dict, Any, Tuple, List, NewType, Union, AsyncIterable
+from typing import Any, AsyncIterable, Dict, List, NewType, Optional, Tuple, Union
 
 import aiohttp
 import msgpack
@@ -127,7 +127,9 @@ def setup_network(
         # Forward compatibility with future supervisors that pass the mask with the IP.
         system(f"ip addr add {ip} dev eth0")
     else:
-        logger.warning("Not passing the mask with the IP is deprecated and will be unsupported")
+        logger.warning(
+            "Not passing the mask with the IP is deprecated and will be unsupported"
+        )
         system(f"ip addr add {ip}/24 dev eth0")
     system("ip link set eth0 up")
 
@@ -236,9 +238,11 @@ def setup_code_executable(
 
 
 def setup_code(
-    code: Optional[bytes], encoding: Optional[Encoding], entrypoint: Optional[str], interface: Interface
+    code: Optional[bytes],
+    encoding: Optional[Encoding],
+    entrypoint: Optional[str],
+    interface: Interface,
 ) -> Union[ASGIApplication, subprocess.Popen]:
-
     if interface == Interface.asgi:
         return setup_code_asgi(code=code, encoding=encoding, entrypoint=entrypoint)
     elif interface == Interface.executable:
@@ -252,7 +256,6 @@ def setup_code(
 async def run_python_code_http(
     application: ASGIApplication, scope: dict
 ) -> Tuple[Dict, Dict, str, Optional[bytes]]:
-
     logger.debug("Running code")
     with StringIO() as buf, redirect_stdout(buf):
         # Execute in the same process, saves ~20ms than a subprocess
@@ -354,7 +357,6 @@ async def process_instruction(
     interface: Interface,
     application: Union[ASGIApplication, subprocess.Popen],
 ) -> AsyncIterable[bytes]:
-
     if instruction == b"halt":
         logger.info("Received halt command")
         system("sync")
@@ -454,7 +456,7 @@ def setup_system(config: ConfigurationPayload):
     # Linux host names are limited to 63 characters. We therefore use the base32 representation
     # of the item_hash instead of its common base16 representation.
     item_hash_binary: bytes = base64.b16decode(config.vm_hash.encode().upper())
-    hostname = base64.b32encode(item_hash_binary).decode().strip('=').lower()
+    hostname = base64.b32encode(item_hash_binary).decode().strip("=").lower()
     setup_hostname(hostname)
 
     setup_variables(config.variables)
@@ -472,7 +474,7 @@ def umount_volumes(volumes: List[Volume]):
         system(f"umount {volume.mount}")
 
 
-async def main():
+async def main() -> None:
     client, addr = s.accept()
 
     logger.debug("Receiving setup...")

From 8fa588af0f297bc2fc5390e68f216d23861b6f48 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 20 Jun 2023 21:05:16 +0200
Subject: [PATCH 410/990] Fix: Some shell commands were not logged

---
 firecracker/microvm.py | 14 ++++++--------
 vm_supervisor/utils.py |  2 ++
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index c35ec1c47..34227dcca 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -4,17 +4,15 @@
 import os.path
 import shutil
 import string
-import subprocess
 from asyncio import Task
 from asyncio.base_events import Server
 from os import getuid
 from pathlib import Path
 from pwd import getpwnam
 from tempfile import NamedTemporaryFile
-from typing import Optional, Tuple, List
+from typing import List, Optional, Tuple
 
-from .config import FirecrackerConfig
-from .config import Drive
+from .config import Drive, FirecrackerConfig
 
 logger = logging.getLogger(__name__)
 
@@ -294,11 +292,11 @@ def enable_device_mapper_rootfs(self, path_on_host: Path) -> Path:
             jailer_device_vm_path.mkdir(exist_ok=True, parents=True)
             rootfs_device = path_on_host.resolve()
             # Copy the /dev/dm-{device_id} special block file that is the real mapping destination on Jailer
-            os.system(f"cp -vap {rootfs_device} {self.jailer_path}/dev/")
+            system(f"cp -vap {rootfs_device} {self.jailer_path}/dev/")
             path_to_mount = jailer_device_vm_path / rootfs_filename
             if not path_to_mount.is_symlink():
                 path_to_mount.symlink_to(rootfs_device)
-            os.system(f"chown -Rh jailman:jailman {self.jailer_path}/dev")
+            system(f"chown -Rh jailman:jailman {self.jailer_path}/dev")
 
         return device_jailer_path
 
@@ -366,7 +364,7 @@ async def unix_client_connected(*_):
         self._unix_socket = await asyncio.start_unix_server(
             unix_client_connected, path=f"{self.vsock_path}_52"
         )
-        os.system(f"chown jailman:jailman {self.vsock_path}_52")
+        system(f"chown jailman:jailman {self.vsock_path}_52")
         try:
             await asyncio.wait_for(queue.get(), timeout=self.init_timeout)
             logger.debug("...signal from init received")
@@ -441,7 +439,7 @@ async def teardown(self):
             logger.debug("Waiting for one second for the VM to shutdown")
             await asyncio.sleep(1)
             root_fs = self.mounted_rootfs.name
-            os.system(f"dmsetup remove {root_fs}")
+            system(f"dmsetup remove {root_fs}")
             if self.use_jailer:
                 shutil.rmtree(self.jailer_path)
 
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index e4a91ff65..e40b34816 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -60,6 +60,8 @@ async def run_in_subprocess(
     command: List[str], check: bool = True, stdin_input: Optional[bytes] = None
 ) -> bytes:
     """Run the specified command in a subprocess, returns the stdout of the process."""
+    logger.debug(f"command: {' '.join(command)}")
+
     process = await asyncio.create_subprocess_exec(
         *command,
         stdin=asyncio.subprocess.PIPE,

From 6888c8f57ca3d38bfed4749566e42f066cb150e3 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Mon, 26 Jun 2023 17:56:02 +0200
Subject: [PATCH 411/990] Feature: Users had no feedback during program load

Problem: When the program in a VM takes a bit long to load, the final user did not see anything, only the loading page.

Solution: Create a loading intermediate page that will be refreshing until the VM is up.

This logic runs inside the VM, and is therefore customizable by program developers.

Co-authored-by: Hugo Herter 
---
 .github/workflows/code-quality.yml            |   2 +-
 .../create_disk_image.sh                      |   1 +
 runtimes/aleph-debian-11-python/init1.py      |  19 +-
 runtimes/aleph-debian-11-python/loading.html  | 346 ++++++++++++++++++
 vm_supervisor/run.py                          |  14 +-
 vm_supervisor/utils.py                        |   2 +-
 vm_supervisor/vm/firecracker/instance.py      |   2 +-
 7 files changed, 376 insertions(+), 10 deletions(-)
 create mode 100644 runtimes/aleph-debian-11-python/loading.html

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
index 32434d0a0..2d9bcd3fd 100644
--- a/.github/workflows/code-quality.yml
+++ b/.github/workflows/code-quality.yml
@@ -17,7 +17,7 @@ jobs:
           sudo apt-get update
           sudo apt-get -y upgrade
           sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema
-          pip install --upgrade typing-extensions
+          pip install --upgrade typing-extensions types-PyYAML
 
       - name: Install required Python packages
         run: |
diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
index c556b1950..a44f78329 100755
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-11-python/create_disk_image.sh
@@ -92,6 +92,7 @@ rm -fr ./rootfs/var/lib/apt/lists/
 # Custom init
 cp ./init0.sh ./rootfs/sbin/init
 cp ./init1.py ./rootfs/root/init1.py
+cp ./loading.html ./rootfs/root/loading.html
 chmod +x ./rootfs/sbin/init
 chmod +x ./rootfs/root/init1.py
 
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index bdb557c2b..8fa639f7b 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3 -OO
 import base64
 import logging
+from pathlib import Path
 
 logging.basicConfig(
     level=logging.DEBUG,
@@ -328,6 +329,22 @@ async def make_request(session, scope):
     return headers, body
 
 
+def show_loading():
+    body = {
+        "body": Path("/root/loading.html").read_text()
+    }
+    headers = {
+        "headers": [
+            [b'Content-Type', b'text/html'],
+            [b'Connection', b'keep-alive'],
+            [b'Keep-Alive', b'timeout=5'],
+            [b'Transfer-Encoding', b'chunked']
+        ],
+        "status": 503,
+    }
+    return headers, body
+
+
 async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[bytes]]:
     logger.debug("Calling localhost")
 
@@ -343,7 +360,7 @@ async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[by
                 headers, body = await make_request(session, scope)
             except aiohttp.ClientConnectorError:
                 if tries > 20:
-                    raise
+                    headers, body = show_loading()
                 await asyncio.sleep(0.05)
 
     output = ""  # Process stdout is not captured per request
diff --git a/runtimes/aleph-debian-11-python/loading.html b/runtimes/aleph-debian-11-python/loading.html
new file mode 100644
index 000000000..da9128c40
--- /dev/null
+++ b/runtimes/aleph-debian-11-python/loading.html
@@ -0,0 +1,346 @@
+
+
+  VM Loading
+  
+
+
+
+
+
    
+  
    
+    
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+    
+  
    
+  
    
+    
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+          
+          
+        
+      
+    
+  
    
+  
    
+    
    
+      
    Whoops!
    
+      
    Seems like your VM is still loading, please wait...
    
+    
    
+
+    Refresh!
+  
    
+
+
    
+
+
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 68c46355f..67aa2d21b 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -3,12 +3,11 @@
 from typing import Any, Dict, Optional
 
 import msgpack
-from msgpack import UnpackValueError
-
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError
-from multidict import CIMultiDict
 from aleph_message.models import ItemHash
+from msgpack import UnpackValueError
+from multidict import CIMultiDict
 
 from firecracker.microvm import MicroVMFailedInit
 
@@ -141,9 +140,12 @@ async def run_code_on_request(
             )
 
         # HTTP Headers require specific data structure
-        headers = CIMultiDict([
-            (key.decode().lower(), value.decode()) for key, value in result["headers"]["headers"]
-        ])
+        headers = CIMultiDict(
+            [
+                (key.decode().lower(), value.decode())
+                for key, value in result["headers"]["headers"]
+            ]
+        )
         if "content-length" not in headers:
             headers["Content-Length".lower()] = str(len(result["body"]["body"]))
         for header in ["Content-Encoding", "Transfer-Encoding", "Vary"]:
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index e40b34816..2f23d414c 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -97,7 +97,7 @@ class HostNotFoundError(Exception):
     pass
 
 
-async def ping(host: str, packets: int, timeout: int):
+async def ping(host: str, packets: int, timeout: float):
     """
     Waits for a host to respond to a ping request.
     """
diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index 31e03823a..5cca68497 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -21,7 +21,7 @@
 from vm_supervisor.conf import settings
 from vm_supervisor.network.interfaces import TapInterface
 from vm_supervisor.storage import create_devmapper
-from vm_supervisor.utils import ping, HostNotFoundError
+from vm_supervisor.utils import HostNotFoundError, ping
 
 from ...utils import run_in_subprocess
 from .executable import (

From b34d8e11b0aa25fddb108d0d4746aa68a7ef9018 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 20 Jun 2023 21:00:05 +0200
Subject: [PATCH 412/990] Fix: Conditions differ for fake path to runtime or
 rootfs

Solution: Fork the function `get_runtime_path` into `get_rootfs_base_path` for instances with specific conditions for handling fake data in development.
---
 vm_supervisor/conf.py    | 20 +++++++++++++++++---
 vm_supervisor/storage.py | 25 ++++++++++++++++++++++---
 2 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index a1adee0fe..668311f2a 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -123,8 +123,9 @@ class Settings(BaseSettings):
         "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda"
     )
 
+    # Tests on programs
+
     FAKE_DATA_PROGRAM: Optional[Path] = None
-    FAKE_DATA_INSTANCE: Optional[bool] = False
     BENCHMARK_FAKE_DATA_PROGRAM = Path(
         abspath(join(__file__, "../../examples/example_fastapi"))
     )
@@ -142,8 +143,21 @@ class Settings(BaseSettings):
         abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs"))
     )
 
-    FAKE_INSTANCE_ID = (
-        "decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca"
+    # Tests on instances
+
+    TEST_INSTANCE_ID: Optional[str] = Field(
+        default=None,  # TODO: Use a valid item_hash here
+        description="Identifier of the instance message used when testing the launch of an instance from the network",
+    )
+
+    USE_FAKE_INSTANCE_BASE = False
+    FAKE_INSTANCE_BASE = Path(
+        abspath(join(__file__, "../../runtimes/instance-debian-rootfs/rootfs.ext4"))
+    )
+    FAKE_INSTANCE_ID: str = Field(
+        default="decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca",
+        description="Identifier used for the 'fake instance' message defined in "
+        "examples/instance_message_from_aleph.json",
     )
     FAKE_INSTANCE_MESSAGE = Path(
         abspath(join(__file__, "../../examples/instance_message_from_aleph.json"))
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index fe1176ea1..439359372 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -13,7 +13,12 @@
 from typing import Union
 
 import aiohttp
-from aleph_message.models import InstanceMessage, ProgramMessage, parse_message
+from aleph_message.models import (
+    InstanceMessage,
+    ItemHash,
+    ProgramMessage,
+    parse_message,
+)
 from aleph_message.models.execution.instance import RootfsVolume
 from aleph_message.models.execution.program import Encoding
 from aleph_message.models.execution.volume import (
@@ -154,7 +159,8 @@ async def get_data_path(ref: str) -> Path:
 
 
 async def get_runtime_path(ref: str) -> Path:
-    if settings.FAKE_DATA_PROGRAM or settings.FAKE_DATA_INSTANCE:
+    """Obtain the runtime used for the rootfs of a program."""
+    if settings.FAKE_DATA_PROGRAM:
         return Path(settings.FAKE_DATA_RUNTIME)
 
     cache_path = Path(settings.RUNTIME_CACHE) / ref
@@ -164,6 +170,19 @@ async def get_runtime_path(ref: str) -> Path:
     return cache_path
 
 
+async def get_rootfs_base_path(ref: ItemHash) -> Path:
+    """Obtain the base partition for the rootfs of an instance."""
+    if settings.USE_FAKE_INSTANCE_BASE and settings.FAKE_INSTANCE_BASE:
+        logger.debug("Using fake instance base")
+        return Path(settings.FAKE_INSTANCE_BASE)
+
+    cache_path = Path(settings.RUNTIME_CACHE) / ref
+    url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}"
+    await download_file(url, cache_path)
+    await chown_to_jailman(cache_path)
+    return cache_path
+
+
 async def create_ext4(path: Path, size_mib: int) -> bool:
     if path.is_file():
         logger.debug(f"File already exists, skipping ext4 creation on {path}")
@@ -236,7 +255,7 @@ async def create_devmapper(
         return path_mapped_volume_name
 
     volume_path = await create_volume_file(volume, namespace)
-    parent_path = await get_runtime_path(volume.parent.ref)
+    parent_path = await get_rootfs_base_path(volume.parent.ref)
 
     base_loop_device = await create_loopback_device(parent_path, read_only=True)
     base_block_size: int = await get_block_size(parent_path)

From b3274e3a7f9869a6dc31964284090ee85cf4041c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 26 Jun 2023 18:41:45 +0200
Subject: [PATCH 413/990] Fix: Typo in CI crashed runtime tests

---
 .github/workflows/test-new-runtime-examples.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-new-runtime-examples.yml b/.github/workflows/test-new-runtime-examples.yml
index de8c917a7..8305aa984 100644
--- a/.github/workflows/test-new-runtime-examples.yml
+++ b/.github/workflows/test-new-runtime-examples.yml
@@ -42,9 +42,9 @@ jobs:
           --enable-ipv6 \
           --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
           aleph-vm-ci-runtime
-      
+
       - name: "Build custom runtime"
-      - run: |
+        run: |
           sudo apt update
           sudo apt install -y debootstrap
           cd runtimes/aleph-debian-11-python && sudo ./create_disk_image.sh && cd ../..

From 0500e60d92a61ecd9e56b730f679eac39d883b11 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 26 Jun 2023 18:53:49 +0200
Subject: [PATCH 414/990] Fix: ItemHash in CI was invalid

This caused the supervisor to refuse the scheduling and raise an error 500.
---
 .github/workflows/test-on-droplet-debian-11.yml    | 2 +-
 .github/workflows/test-on-droplet-ubuntu-22.04.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-on-droplet-debian-11.yml b/.github/workflows/test-on-droplet-debian-11.yml
index e7fc4b970..1327752e0 100644
--- a/.github/workflows/test-on-droplet-debian-11.yml
+++ b/.github/workflows/test-on-droplet-debian-11.yml
@@ -79,7 +79,7 @@ jobs:
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           curl --retry 5 --max-time 10 --fail -X POST -H "Content-Type: application/json" \
             -H "X-Auth-Signature: test" \
-            -d '{"persistent_vms": [], "instances": ["INSTANCE-HASH-TODO-FIXME"]}' \
+            -d '{"persistent_vms": [], "instances": ["67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"]}' \
             "http://${DROPLET_IPV4}:4020/control/allocations"
 
       - name: Cleanup
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index cc9633d57..4860be7b0 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -84,7 +84,7 @@ jobs:
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-ubuntu-22-04 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           curl --retry 5 --max-time 10 --fail -X POST -H "Content-Type: application/json" \
             -H "X-Auth-Signature: test" \
-            -d '{"persistent_vms": [], "instances": ["INSTANCE-HASH-TODO-FIXME"]}' \
+            -d '{"persistent_vms": [], "instances": ["67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"]}' \
             "http://${DROPLET_IPV4}:4020/control/allocations"
 
       - name: Cleanup

From 7701c519462796d227b8b17c067358db11fa970a Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 26 Jun 2023 19:11:05 +0200
Subject: [PATCH 415/990] Fix: ItemHash did not serialize to string

This caused an error when generating cloud-init configuration.
---
 vm_supervisor/vm/firecracker/instance.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index 5cca68497..cb2cfb9db 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -1,9 +1,8 @@
 import asyncio
 import logging
-import subprocess
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Optional
+from typing import Dict, List, Optional, Union
 
 import yaml
 from aleph_message.models import ItemHash
@@ -148,8 +147,8 @@ def _encode_user_data(self) -> bytes:
 
         ssh_authorized_keys = self.resources.message_content.authorized_keys or []
 
-        config = {
-            "hostname": self.vm_hash,
+        config: Dict[str, Union[str, bool, List[str]]] = {
+            "hostname": str(self.vm_hash),
             "disable_root": False,
             "ssh_pwauth": False,
             "ssh_authorized_keys": ssh_authorized_keys,

From 031d64c047d53bc86024b42ca0b9485ae99b851a Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 26 Jun 2023 19:11:53 +0200
Subject: [PATCH 416/990] Fix: Command `cloud-localds` was not found on hosts

Solution: Add package `cloud-image-utils` in package control file.
---
 packaging/aleph-vm/DEBIAN/control | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index 0760e76a0..725e7793d 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils
 Section: aleph-im
 Priority: Extra

From 0f2be3b9037a54ba54379364a1a71edd4a76c23f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 20 Jun 2023 21:08:20 +0200
Subject: [PATCH 417/990] Fix: CLI launch of instances was limited

Solution:

Use 2 commands: `--run-test-instance` to start an instance from the network and `--run-fake-instance` to start an instance from local files.

Add a setting with `--fake-instance-base` to specify the base of the local fake instance root filesystem.
---
 vm_supervisor/__main__.py | 51 ++++++++++++++++++++++++++++-----------
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index 15fe6204b..c660d8fdb 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -123,12 +123,28 @@ def parse_args(args):
         help="Path to project containing fake data",
     )
     parser.add_argument(
-        "-k",
+        "-i",
         "--run-test-instance",
         dest="run_test_instance",
         action="store_true",
         default=False,
-        help="Run a test instance instead of starting the entire supervisor",
+        help="Run a test instance from the network instead of starting the entire supervisor",
+    )
+    parser.add_argument(
+        "-k",
+        "--run-fake-instance",
+        dest="run_fake_instance",
+        action="store_true",
+        default=False,
+        help="Run a fake instance from a local rootfs instead of starting the entire supervisor",
+    )
+    parser.add_argument(
+        "-r",
+        "--fake-instance-base",
+        dest="fake_instance_base",
+        type=str,
+        default=settings.FAKE_INSTANCE_BASE,
+        help="Filesystem path of the base for the rootfs of fake instances. An empty value signals a download instead.",
     )
     return parser.parse_args(args)
 
@@ -217,7 +233,7 @@ async def fake_read() -> bytes:
     print("Event result", result)
 
 
-async def run_instance(item_hash: ItemHash):
+async def start_instance(item_hash: ItemHash) -> None:
     """Run an instance from an InstanceMessage."""
 
     # The main program uses a singleton pubsub instance in order to watch for updates.
@@ -228,6 +244,16 @@ async def run_instance(item_hash: ItemHash):
     await start_persistent_vm(item_hash, dummy_pubsub)
 
 
+async def run_instances(instances: List[ItemHash]) -> None:
+    """Run instances from a list of message identifiers."""
+    logger.info(f"Instances to run: {instances}")
+
+    await asyncio.gather(
+        *[start_instance(item_hash=instance_id) for instance_id in instances]
+    )
+    await asyncio.Event().wait()  # wait forever
+
+
 @contextlib.contextmanager
 def change_dir(directory: Path):
     current_directory = Path.cwd()
@@ -270,7 +296,10 @@ def main():
         ALLOW_VM_NETWORKING=args.allow_vm_networking,
         FAKE_DATA_PROGRAM=args.fake_data_program,
         DEBUG_ASYNCIO=args.debug_asyncio,
+        FAKE_INSTANCE_BASE=args.fake_instance_base,
     )
+    if args.run_fake_instance:
+        settings.USE_FAKE_INSTANCE_BASE = True
 
     if sentry_sdk:
         if settings.SENTRY_DSN:
@@ -307,17 +336,11 @@ def main():
     elif args.do_not_run:
         logger.info("Option --do-not-run, exiting")
     elif args.run_test_instance:
-        logger.info("Running test instance virtual machine")
-        instance_item_hash = settings.FAKE_INSTANCE_ID
-        settings.update(
-            FAKE_DATA_INSTANCE=True,
-        )
-
-        async def forever():
-            await run_instance(item_hash=instance_item_hash)
-            await asyncio.sleep(1000)
-
-        asyncio.run(forever())
+        asyncio.run(run_instances([ItemHash(settings.TEST_INSTANCE_ID)]))
+        logger.info("Finished")
+        sys.exit(0)
+    elif args.run_fake_instance:
+        asyncio.run(run_instances([ItemHash(settings.FAKE_INSTANCE_ID)]))
         logger.info("Finished")
         sys.exit(0)
     else:

From 6b18b42e827f2d9cb149756f3185456c7fef0451 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 27 Jun 2023 15:11:05 +0200
Subject: [PATCH 418/990] Fix: Users reported errors with key `family` absent
 from entry

On Debian 11:

```
python3[1910876]:     if not check_if_table_exists("ip", table):
python3[1910876]:   File "/opt/aleph-vm/vm_supervisor/network/firewall.py", line 91, in check_if_table_exists
python3[1910876]:     and entry["family"] == family
python3[1910876]: KeyError: 'family'
```
---
 vm_supervisor/network/firewall.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/vm_supervisor/network/firewall.py b/vm_supervisor/network/firewall.py
index 454cd5aff..276b8c6dd 100644
--- a/vm_supervisor/network/firewall.py
+++ b/vm_supervisor/network/firewall.py
@@ -88,8 +88,9 @@ def check_if_table_exists(family: str, table: str) -> bool:
         if (
             isinstance(entry, dict)
             and "table" in entry
-            and entry["family"] == family
-            and entry["name"] == table
+            # Key "family" was reported by users as not always present, so we use .get() instead of [].
+            and entry.get("family") == family
+            and entry.get("name") == table
         ):
             return True
     return False

From 8bc38e8c97b76d5a8be43c5d587217c33259ba47 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 29 Jun 2023 16:26:47 +0200
Subject: [PATCH 419/990] Fix: Root login with shared password is insecure

Problem: With the incoming support of IPv6, allowing login into virtual machines using SSH with a password becomes unacceptable.

Solution: Remove SSH password login from runtimes, as well as the root default password.
---
 runtimes/aleph-debian-11-python/create_disk_image.sh | 6 ++++--
 runtimes/instance-debian-rootfs/Dockerfile           | 8 ++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
index a44f78329..243a0fd13 100755
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-11-python/create_disk_image.sh
@@ -39,7 +39,10 @@ pip3 install 'aleph-client>=0.4.6' 'coincurve==15.0.0'
 # Compile all Python bytecode
 python3 -m compileall -f /usr/local/lib/python3.9
 
-echo "root:toor" | /usr/sbin/chpasswd
+echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
+echo "PasswordAuthentication no" >> /etc/ssh/sshd_config
+echo "ChallengeResponseAuthentication no" >> /etc/ssh/sshd_config
+echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
 
 mkdir -p /overlay
 
@@ -48,7 +51,6 @@ ln -s agetty /etc/init.d/agetty.ttyS0
 echo ttyS0 > /etc/securetty
 EOT
 
-echo "PermitRootLogin yes" >> ./rootfs/etc/ssh/sshd_config
 
 # Generate SSH host keys
 #systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
diff --git a/runtimes/instance-debian-rootfs/Dockerfile b/runtimes/instance-debian-rootfs/Dockerfile
index 29f9b5a85..3505f7a61 100644
--- a/runtimes/instance-debian-rootfs/Dockerfile
+++ b/runtimes/instance-debian-rootfs/Dockerfile
@@ -35,7 +35,10 @@ RUN pip3 install 'aleph-client>=0.4.6' 'coincurve==15.0.0'
 RUN python3 -m compileall -f /usr/local/lib/python3.9
 
 # Enable root login by ssh
-RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
+RUN echo "PubkeyAuthentication yes" >> ./rootfs/etc/ssh/sshd_config
+RUN echo "PasswordAuthentication no" >> ./rootfs/etc/ssh/sshd_config
+RUN echo "ChallengeResponseAuthentication no" >> ./rootfs/etc/ssh/sshd_config
+RUN echo "PermitRootLogin yes" >> ./rootfs/etc/ssh/sshd_config
 
 # Generate SSH host keys
 #RUN systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
@@ -47,9 +50,6 @@ RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
 RUN ln -s agetty /etc/init.d/agetty.ttyS0
 RUN echo ttyS0 > /etc/securetty
 
-# Set root password
-RUN echo "root:toor" | /usr/sbin/chpasswd
-
 # Reduce size
 RUN rm -fr /root/.cache
 RUN rm -fr /var/cache

From 3ccc4304241fb1b934ec125d4eb71e4230f7848e Mon Sep 17 00:00:00 2001
From: mhh 
Date: Fri, 16 Jun 2023 09:39:55 +0200
Subject: [PATCH 420/990] add /opt/node_modules to NODE_PATH

---
 runtimes/instance-debian-rootfs/init1.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/runtimes/instance-debian-rootfs/init1.py b/runtimes/instance-debian-rootfs/init1.py
index 99b4a69fa..6c2ae0b9a 100644
--- a/runtimes/instance-debian-rootfs/init1.py
+++ b/runtimes/instance-debian-rootfs/init1.py
@@ -91,6 +91,9 @@ class RunCodePayload:
 os.environ["ALEPH_REMOTE_CRYPTO_HOST"] = "http://localhost"
 os.environ["ALEPH_REMOTE_CRYPTO_UNIX_SOCKET"] = "/tmp/socat-socket"
 
+# Additional node modules from immutable volume
+os.environ["NODE_PATH"] = "/opt/node_modules"
+
 logger.debug("init1.py is launching")
 
 

From d05f31c06ad6820a56af93ad785055834d1e5660 Mon Sep 17 00:00:00 2001
From: mhh 
Date: Fri, 16 Jun 2023 09:40:19 +0200
Subject: [PATCH 421/990] add description to run test mode

---
 vm_supervisor/README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index 3178dacc1..c1a9484c0 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -118,6 +118,10 @@ or in debug mode:
 ```shell
 python3 -m vm_supervisor -vv --system-logs
 ```
+or in test mode to run a single function:
+```shell
+python3 -m vm_supervisor -vv --system-logs -f examples/example_fastapi
+```
 
 Test accessing the service on
 http://localhost:4020/

From cf236981cb51b99fc14f2ce50df41949d017bf43 Mon Sep 17 00:00:00 2001
From: Mike Hukiewitz <70762838+MHHukiewitz@users.noreply.github.com>
Date: Mon, 3 Jul 2023 09:51:01 +0200
Subject: [PATCH 422/990] Revert README.md change

---
 vm_supervisor/README.md | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index c1a9484c0..3178dacc1 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -118,10 +118,6 @@ or in debug mode:
 ```shell
 python3 -m vm_supervisor -vv --system-logs
 ```
-or in test mode to run a single function:
-```shell
-python3 -m vm_supervisor -vv --system-logs -f examples/example_fastapi
-```
 
 Test accessing the service on
 http://localhost:4020/

From a9a19cd5e1fb6ce853285d62505d50ccf7fa18b9 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Tue, 20 Jun 2023 10:21:17 +0200
Subject: [PATCH 423/990] Feature: VMs did not support IPv6

Problem: We want each VM to be reachable on the internet, with
no restriction in terms of ports, protocols, etc.

Solution: Slice up the IPv6 range attributed to each CRN and allocate
a subnet to each VM.

Add the support for NDP proxy for some hosting providers.

IPv6 defaults to local addresses for broad compatibility, opt-in public addresses.

The custom init `init1.py` is updated to support IPv6.

Separate loopback networking and always run it
- Remove duplicate IPv4 setting
- Only setup IPv4 route if IPv4 is set
- Only setup eth0 if ip or ipv6

Co-authored-by: Hugo Herter 
---
 docker/vm_supervisor-dev.dockerfile        |   2 +-
 packaging/aleph-vm/DEBIAN/control          |   2 +-
 runtimes/aleph-debian-11-python/init1.py   |  56 ++++--
 tests/supervisor/test_ipv6_allocator.py    |  23 +++
 vm_supervisor/README.md                    |   2 +-
 vm_supervisor/conf.py                      |  17 ++
 vm_supervisor/network/hostnetwork.py       | 191 +++++++++++++++++++--
 vm_supervisor/network/interfaces.py        |  35 +++-
 vm_supervisor/network/ndp_proxy.py         |  57 ++++++
 vm_supervisor/pool.py                      |  14 +-
 vm_supervisor/views.py                     |   4 +-
 vm_supervisor/vm/firecracker/executable.py |  24 ++-
 vm_supervisor/vm/firecracker/instance.py   |   5 +-
 vm_supervisor/vm/firecracker/program.py    |  20 ++-
 vm_supervisor/vm/vm_type.py                |  21 +++
 15 files changed, 416 insertions(+), 57 deletions(-)
 create mode 100644 tests/supervisor/test_ipv6_allocator.py
 create mode 100644 vm_supervisor/network/ndp_proxy.py
 create mode 100644 vm_supervisor/vm/vm_type.py

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 901bd8c4f..efbe3df24 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -5,7 +5,7 @@ FROM debian:bullseye
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     sudo acl curl squashfs-tools git \
     python3 python3-aiohttp python3-alembic python3-msgpack python3-pip python3-aiodns python3-aioredis\
-    python3-nftables python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging python3-cpuinfo \
+    python3-nftables python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging python3-cpuinfo ndppd \
     && rm -rf /var/lib/apt/lists/*
 
 RUN useradd jailman
diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index 725e7793d..fa4c6972a 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd
 Section: aleph-im
 Priority: Extra
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index 8fa639f7b..aca160f0c 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -65,7 +65,9 @@ class ConfigurationPayload:
     encoding: Encoding = None
     entrypoint: str = None
     ip: Optional[str] = None
+    ipv6: Optional[str] = None
     route: Optional[str] = None
+    ipv6_gateway: Optional[str] = None
     dns_servers: List[str] = field(default_factory=list)
     volumes: List[Volume] = field(default_factory=list)
     variables: Optional[Dict[str, str]] = None
@@ -108,7 +110,11 @@ def setup_variables(variables: Optional[Dict[str, str]]):
 
 
 def setup_network(
-    ip: Optional[str], route: Optional[str], dns_servers: Optional[List[str]] = None
+    ip: Optional[str],
+    ipv6: Optional[str],
+    route: Optional[str],
+    ipv6_gateway: Optional[str],
+    dns_servers: Optional[List[str]] = None,
 ):
     """Setup the system with info from the host."""
     dns_servers = dns_servers or []
@@ -116,29 +122,41 @@ def setup_network(
         logger.info("No network interface eth0")
         return
 
-    if not ip:
+    if not (ip or ipv6):
         logger.info("No network IP")
         return
 
-    logger.debug("Setting up networking")
+    # Configure loopback networking
     system("ip addr add 127.0.0.1/8 dev lo brd + scope host")
     system("ip addr add ::1/128 dev lo")
     system("ip link set lo up")
-    if "/" in ip:
+
+    if ip:
+        logger.debug("Setting up IPv4")
+
         # Forward compatibility with future supervisors that pass the mask with the IP.
+        if "/" not in ip:
+            logger.warning(
+                "Not passing the mask with the IP is deprecated and will be unsupported"
+            )
+            ip = f"{ip}/24"
+
         system(f"ip addr add {ip} dev eth0")
-    else:
-        logger.warning(
-            "Not passing the mask with the IP is deprecated and will be unsupported"
-        )
-        system(f"ip addr add {ip}/24 dev eth0")
-    system("ip link set eth0 up")
 
-    if route:
-        system(f"ip route add default via {route} dev eth0")
-        logger.debug(f"IP and route set: {ip} via {route}")
-    else:
-        logger.warning("IP set with no network route")
+        if route:
+            system(f"ip route add default via {route} dev eth0")
+            logger.debug(f"IP and route set: {ip} via {route}")
+        else:
+            logger.warning("IPv4 set with no network route")
+
+    if ipv6:
+        logger.debug("Setting up IPv6")
+        system(f"ip addr add {ipv6} dev eth0")
+        system(f"ip -6 route add default via {ipv6_gateway} dev eth0")
+        logger.debug(f"IPv6 setup to address {ipv6}")
+
+    if ip or ipv6:
+        system("ip link set eth0 up")
 
     with open("/etc/resolv.conf", "wb") as resolvconf_fd:
         for server in dns_servers:
@@ -478,7 +496,13 @@ def setup_system(config: ConfigurationPayload):
 
     setup_variables(config.variables)
     setup_volumes(config.volumes)
-    setup_network(config.ip, config.route, config.dns_servers)
+    setup_network(
+        ip=config.ip,
+        ipv6=config.ipv6,
+        route=config.route,
+        ipv6_gateway=config.ipv6_gateway,
+        dns_servers=config.dns_servers,
+    )
     setup_input_data(config.input_data)
     logger.debug("Setup finished")
 
diff --git a/tests/supervisor/test_ipv6_allocator.py b/tests/supervisor/test_ipv6_allocator.py
new file mode 100644
index 000000000..cd85ff4d1
--- /dev/null
+++ b/tests/supervisor/test_ipv6_allocator.py
@@ -0,0 +1,23 @@
+import os
+
+# Avoid failures linked to settings when initializing the global VmPool object
+os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
+
+from ipaddress import IPv6Network
+
+from aleph_message.models import ItemHash
+
+from vm_supervisor.network.hostnetwork import StaticIPv6Allocator
+
+
+def test_static_ipv6_allocator():
+    allocator = StaticIPv6Allocator(
+        ipv6_range=IPv6Network("1111:2222:3333:4444::/64"), subnet_prefix=124
+    )
+    ip_subnet = allocator.allocate_vm_ipv6_subnet(
+        vm_id=3,
+        vm_hash=ItemHash(
+            "8920215b2e961a4d4c59a8ceb2803af53f91530ff53d6704273ab4d380bc6446"
+        ),
+    )
+    assert ip_subnet == IPv6Network("1111:2222:3333:4444:0001:8920:215b:2e90/124")
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index 3178dacc1..6d9378805 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -57,7 +57,7 @@ when running the VM Supervisor.
 ```shell
 apt update
 apt install -y git python3 python3-aiohttp python3-msgpack python3-aiodns python3-sqlalchemy python3-setproctitle redis python3-aioredis \
- python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-nftables python3-jsonschema
+ python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-nftables python3-jsonschema ndppd
 useradd jailman
 ```
 
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 668311f2a..8a543ff92 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -20,6 +20,11 @@ class DnsResolver(str, Enum):
     resolvectl = "resolvectl"  # Systemd-resolved, common on Ubuntu
 
 
+class IPv6AllocationPolicy(str, Enum):
+    static = "static"  # Compute the IP address based on the VM item hash.
+    dynamic = "dynamic"  # Assign an available IP address.
+
+
 def etc_resolv_conf_dns_servers():
     with open("/etc/resolv.conf", "r") as resolv_file:
         for line in resolv_file.readlines():
@@ -90,6 +95,18 @@ class Settings(BaseSettings):
         default=24,
         description="Individual VM network prefix length in bits",
     )
+    IPV6_ADDRESS_POOL: str = Field(
+        default="fc00:1:2:3::/64",
+        description="IPv6 address range assigned to the host. Example: 1111:2222:3333:4444::/64. "
+        "Defaults to a local address range for compatibility with hosts not yet configured for IPv6.",
+    )
+    IPV6_ALLOCATION_POLICY: IPv6AllocationPolicy = Field(
+        default=IPv6AllocationPolicy.static
+    )
+    IPV6_SUBNET_PREFIX: int = Field(
+        default=124,
+        description="IPv6 subnet prefix for VMs. Made configurable for testing.",
+    )
     NFTABLES_CHAIN_PREFIX = "aleph"
 
     DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf
diff --git a/vm_supervisor/network/hostnetwork.py b/vm_supervisor/network/hostnetwork.py
index 81ecd4b7c..d363452b1 100644
--- a/vm_supervisor/network/hostnetwork.py
+++ b/vm_supervisor/network/hostnetwork.py
@@ -1,26 +1,160 @@
 import logging
+from ipaddress import IPv6Network
 from pathlib import Path
+from typing import Optional, Protocol
 
+from aleph_message.models import ItemHash
+
+from vm_supervisor.conf import IPv6AllocationPolicy
+
+from ..vm.vm_type import VmType
 from .firewall import initialize_nftables, setup_nftables_for_vm, teardown_nftables
 from .interfaces import TapInterface
 from .ipaddresses import IPv4NetworkWithInterfaces
+from .ndp_proxy import NdpProxy
 
 logger = logging.getLogger(__name__)
 
 
+def _read_file_as_int(config_file: Path) -> int:
+    return int(config_file.read_text())
+
+
 def get_ipv4_forwarding_state() -> int:
-    """Reads the current ipv4 forwarding setting from the hosts, converts it to int and returns it"""
-    return int(Path("/proc/sys/net/ipv4/ip_forward").read_text())
+    """Reads the current IPv4 forwarding setting from the host, converts it to int and returns it"""
+    return _read_file_as_int(Path("/proc/sys/net/ipv4/ip_forward"))
+
+
+def get_ipv6_forwarding_state() -> int:
+    """Reads the current IPv6 forwarding setting from the host, converts it to int and returns it"""
+    return _read_file_as_int(Path("/proc/sys/net/ipv6/conf/all/forwarding"))
+
+
+class IPv6Allocator(Protocol):
+    def allocate_vm_ipv6_subnet(
+        self, vm_id: int, vm_hash: ItemHash, vm_type: VmType
+    ) -> IPv6Network:
+        ...
+
+
+class StaticIPv6Allocator(IPv6Allocator):
+    """
+    Static IPv6 allocator.
+    Computes IPv6 addresses based on the machine type and VM hash. The algorithm works
+    as follows:
+
+    | Component | CRN /64 range | VM type | Item hash prefix | Instance range |
+    |-----------|---------------|---------|------------------|----------------|
+    | Length    | 64 bits       | 16 bits | 44 bits          | 4 bits         |
+    """
+
+    VM_TYPE_PREFIX = {
+        VmType.microvm: "1",
+        VmType.persistent_program: "2",
+        VmType.instance: "3",
+    }
+
+    def __init__(self, ipv6_range: IPv6Network, subnet_prefix: int):
+        if ipv6_range.prefixlen != 64:
+            raise ValueError(
+                "The static IP address allocation scheme requires a /64 subnet"
+            )
+        if subnet_prefix < 124:
+            raise ValueError("The IPv6 subnet prefix cannot be larger than /124.")
+
+        self.ipv6_range = ipv6_range
+        self.subnet_prefix = subnet_prefix
+
+    def allocate_vm_ipv6_subnet(
+        self, vm_id: int, vm_hash: ItemHash, vm_type: VmType
+    ) -> IPv6Network:
+        ipv6_elems = self.ipv6_range.exploded.split(":")[:4]
+        ipv6_elems += [self.VM_TYPE_PREFIX[vm_type]]
+
+        # Add the item hash of the VM as the last 44 bits of the IPv6 address.
+        # The last 4 bits are left for use to the VM owner as a /124 subnet.
+        ipv6_elems += [vm_hash[0:4], vm_hash[4:8], vm_hash[8:11] + "0"]
+
+        return IPv6Network(":".join(ipv6_elems) + "/124")
+
+
+class DynamicIPv6Allocator(IPv6Allocator):
+    """
+    A dynamic allocator, for testing purposes.
+    This allocator slices the input IPv6 address range in subnets of the same size
+    and iterates through them. The first subnet is assumed to be reserved for use by the host,
+    as we use this allocator in situations where the address range is small and the host
+    subnet cannot be isolated from the VM subnets (ex: /124 network on Digital Ocean for the CI).
+    """
+
+    def __init__(self, ipv6_range: IPv6Network, subnet_prefix: int):
+        self.ipv6_range = ipv6_range
+        self.vm_subnet_prefix = subnet_prefix
+
+        self.subnets_generator = ipv6_range.subnets(new_prefix=subnet_prefix)
+        # Assume the first subnet is reserved for the host
+        _ = next(self.subnets_generator)
+
+    def allocate_vm_ipv6_subnet(
+        self, vm_id: int, vm_hash: ItemHash, vm_type: VmType
+    ) -> IPv6Network:
+        return next(self.subnets_generator)
+
+
+def make_ipv6_allocator(
+    allocation_policy: IPv6AllocationPolicy, address_pool: str, subnet_prefix: int
+) -> IPv6Allocator:
+    if allocation_policy == IPv6AllocationPolicy.static:
+        return StaticIPv6Allocator(
+            ipv6_range=IPv6Network(address_pool), subnet_prefix=subnet_prefix
+        )
+
+    return DynamicIPv6Allocator(
+        ipv6_range=IPv6Network(address_pool), subnet_prefix=subnet_prefix
+    )
 
 
 class Network:
-    ipv4_forward_state_before_setup: int
-    address_pool: IPv4NetworkWithInterfaces = IPv4NetworkWithInterfaces("172.16.0.0/12")
+    ipv4_forward_state_before_setup: Optional[int]
+    ipv6_forward_state_before_setup: Optional[int]
+    ipv4_address_pool: IPv4NetworkWithInterfaces = IPv4NetworkWithInterfaces(
+        "172.16.0.0/12"
+    )
+    ipv6_address_pool: IPv6Network
     network_size: int
     external_interface: str
 
+    IPV6_SUBNET_PREFIX: int = 124
+
+    def __init__(
+        self,
+        vm_ipv4_address_pool_range: str,
+        vm_network_size: int,
+        external_interface: str,
+        ipv6_allocator: IPv6Allocator,
+    ) -> None:
+        """Sets up the Network class with some information it needs so future function calls work as expected"""
+        self.ipv4_address_pool = IPv4NetworkWithInterfaces(vm_ipv4_address_pool_range)
+        if not self.ipv4_address_pool.is_private:
+            logger.warning(
+                f"Using a network range that is not private: {self.ipv4_address_pool}"
+            )
+        self.ipv6_allocator = ipv6_allocator
+
+        self.network_size = vm_network_size
+        self.external_interface = external_interface
+        self.ipv4_forward_state_before_setup = None
+        self.ipv6_forward_state_before_setup = None
+
+        self.enable_ipv4_forwarding()
+        self.enable_ipv6_forwarding()
+
+        self.ndp_proxy = NdpProxy(host_network_interface=external_interface)
+
+        initialize_nftables()
+
     def get_network_for_tap(self, vm_id: int) -> IPv4NetworkWithInterfaces:
-        subnets = list(self.address_pool.subnets(new_prefix=self.network_size))
+        subnets = list(self.ipv4_address_pool.subnets(new_prefix=self.network_size))
         return subnets[vm_id]
 
     def enable_ipv4_forwarding(self) -> None:
@@ -33,32 +167,51 @@ def enable_ipv4_forwarding(self) -> None:
     def reset_ipv4_forwarding_state(self) -> None:
         """Returns the hosts IPv4 forwarding state how it was before we enabled it"""
         logger.debug("Resetting IPv4 forwarding state to state before we enabled it")
+        if self.ipv4_forward_state_before_setup is None:
+            return
+
         if self.ipv4_forward_state_before_setup != get_ipv4_forwarding_state():
             Path("/proc/sys/net/ipv4/ip_forward").write_text(
                 str(self.ipv4_forward_state_before_setup)
             )
 
-    def __init__(
-        self, vm_address_pool_range: str, vm_network_size: int, external_interface: str
-    ) -> None:
-        """Sets up the Network class with some information it needs so future function calls work as expected"""
-        self.address_pool = IPv4NetworkWithInterfaces(vm_address_pool_range)
-        if not self.address_pool.is_private:
-            logger.warning(
-                f"Using a network range that is not private: {self.address_pool}"
+    def enable_ipv6_forwarding(self) -> None:
+        """Saves the host IPv6 forwarding state, and if it was disabled, enables it"""
+        logger.debug("Enabling IPv6 forwarding")
+        self.ipv6_forward_state_before_setup = get_ipv6_forwarding_state()
+        if not self.ipv6_forward_state_before_setup:
+            Path("/proc/sys/net/ipv6/conf/all/forwarding").write_text("1")
+
+    def reset_ipv6_forwarding_state(self) -> None:
+        """Returns the host IPv6 forwarding state how it was before we enabled it"""
+        logger.debug("Resetting IPv6 forwarding state to state before we enabled it")
+        if self.ipv6_forward_state_before_setup is None:
+            return
+
+        if self.ipv6_forward_state_before_setup != get_ipv6_forwarding_state():
+            Path("/proc/sys/net/ipv6/conf/all/forwarding").write_text(
+                str(self.ipv6_forward_state_before_setup)
             )
-        self.network_size = vm_network_size
-        self.external_interface = external_interface
-        self.enable_ipv4_forwarding()
-        initialize_nftables()
 
     def teardown(self) -> None:
         teardown_nftables()
         self.reset_ipv4_forwarding_state()
+        self.reset_ipv6_forwarding_state()
 
-    async def create_tap(self, vm_id: int) -> TapInterface:
+    async def create_tap(
+        self, vm_id: int, vm_hash: ItemHash, vm_type: VmType
+    ) -> TapInterface:
         """Create TAP interface to be used by VM"""
-        interface = TapInterface(f"vmtap{vm_id}", self.get_network_for_tap(vm_id))
+        interface = TapInterface(
+            f"vmtap{vm_id}",
+            ip_network=self.get_network_for_tap(vm_id),
+            ipv6_network=self.ipv6_allocator.allocate_vm_ipv6_subnet(
+                vm_id=vm_id,
+                vm_hash=vm_hash,
+                vm_type=vm_type,
+            ),
+            ndp_proxy=self.ndp_proxy,
+        )
         await interface.create()
         setup_nftables_for_vm(vm_id, interface)
         return interface
diff --git a/vm_supervisor/network/interfaces.py b/vm_supervisor/network/interfaces.py
index b520e1008..d20c65a2c 100644
--- a/vm_supervisor/network/interfaces.py
+++ b/vm_supervisor/network/interfaces.py
@@ -1,10 +1,11 @@
 import asyncio
 import logging
 import shutil
-from ipaddress import IPv4Interface
+from ipaddress import IPv4Interface, IPv6Address, IPv6Interface, IPv6Network
 from subprocess import run
 
 from .ipaddresses import IPv4NetworkWithInterfaces
+from .ndp_proxy import NdpProxy
 
 logger = logging.getLogger(__name__)
 
@@ -12,10 +13,19 @@
 class TapInterface:
     device_name: str
     ip_network: IPv4NetworkWithInterfaces
+    ipv6_network: IPv6Network
 
-    def __init__(self, device_name: str, ip_network: IPv4NetworkWithInterfaces):
+    def __init__(
+        self,
+        device_name: str,
+        ip_network: IPv4NetworkWithInterfaces,
+        ipv6_network: IPv6Network,
+        ndp_proxy: NdpProxy,
+    ):
         self.device_name: str = device_name
         self.ip_network: IPv4NetworkWithInterfaces = ip_network
+        self.ipv6_network = ipv6_network
+        self.ndp_proxy = ndp_proxy
 
     @property
     def guest_ip(self) -> IPv4Interface:
@@ -25,6 +35,14 @@ def guest_ip(self) -> IPv4Interface:
     def host_ip(self) -> IPv4Interface:
         return self.ip_network[1]
 
+    @property
+    def guest_ipv6(self) -> IPv6Interface:
+        return IPv6Interface(f"{self.ipv6_network[1]}/{self.ipv6_network.prefixlen}")
+
+    @property
+    def host_ipv6(self) -> IPv6Interface:
+        return IPv6Interface(f"{self.ipv6_network[0]}/{self.ipv6_network.prefixlen}")
+
     async def create(self):
         logger.debug("Create network interface")
 
@@ -40,7 +58,19 @@ async def create(self):
                 self.device_name,
             ]
         )
+        ipv6_gateway = self.host_ipv6
+        run(
+            [
+                ip_command,
+                "addr",
+                "add",
+                str(ipv6_gateway),
+                "dev",
+                self.device_name,
+            ]
+        )
         run([ip_command, "link", "set", self.device_name, "up"])
+        self.ndp_proxy.add_range(self.device_name, ipv6_gateway.network)
         logger.debug(f"Network interface created: {self.device_name}")
 
     async def delete(self) -> None:
@@ -48,4 +78,5 @@ async def delete(self) -> None:
         Then removes the interface from the host."""
         logger.debug(f"Removing interface {self.device_name}")
         await asyncio.sleep(0.1)  # Avoids Device/Resource busy bug
+        self.ndp_proxy.delete_range(self.device_name)
         run(["ip", "tuntap", "del", self.device_name, "mode", "tap"])
diff --git a/vm_supervisor/network/ndp_proxy.py b/vm_supervisor/network/ndp_proxy.py
new file mode 100644
index 000000000..f5ccc3751
--- /dev/null
+++ b/vm_supervisor/network/ndp_proxy.py
@@ -0,0 +1,57 @@
+"""
+Neighbourhood Discovery Proxy (NDP) functionalities.
+
+Some cloud providers do not route the whole advertised IPv6 address range to servers, but instead
+only route one address. They will issue NDP requests to the network to determine if the other
+addresses in the range are used. This means that our server (be it the hypervisor or the VMs)
+has to answer to these requests to make the VMs routable.
+
+To achieve this, we use ndppd. Each time an update is required, we overwrite /etc/ndppd.conf
+and restart the service.
+"""
+import logging
+from dataclasses import dataclass
+from ipaddress import IPv6Network
+from pathlib import Path
+from subprocess import run
+from typing import Dict
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class NdpRule:
+    address_range: IPv6Network
+
+
+class NdpProxy:
+    def __init__(self, host_network_interface: str):
+        self.host_network_interface = host_network_interface
+        self.interface_address_range_mapping: Dict[str, IPv6Network] = {}
+
+    @staticmethod
+    def _restart_ndppd():
+        logger.debug("Restarting ndppd")
+        run(["systemctl", "restart", "ndppd"])
+
+    def _update_ndppd_conf(self):
+        config = f"proxy {self.host_network_interface} {{\n"
+        for interface, address_range in self.interface_address_range_mapping.items():
+            config += f"  rule {address_range} {{\n    iface {interface}\n  }}\n"
+        config += "}\n"
+        Path("/etc/ndppd.conf").write_text(config)
+        self._restart_ndppd()
+
+    def add_range(self, interface: str, address_range: IPv6Network):
+        logger.debug("Proxying range %s -> %s", address_range, interface)
+        self.interface_address_range_mapping[interface] = address_range
+        self._update_ndppd_conf()
+
+    def delete_range(self, interface: str):
+        try:
+            address_range = self.interface_address_range_mapping.pop(interface)
+            logger.debug("Deactivated proxying for %s (%s)", interface, address_range)
+        except KeyError:
+            return
+
+        self._update_ndppd_conf()
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 02606ba0f..59677ae47 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -4,10 +4,11 @@
 
 from aleph_message.models import ExecutableMessage, ItemHash
 
-from vm_supervisor.network.hostnetwork import Network
+from vm_supervisor.network.hostnetwork import Network, make_ipv6_allocator
 
 from .conf import settings
 from .models import ExecutableContent, VmExecution
+from .vm.vm_type import VmType
 
 logger = logging.getLogger(__name__)
 
@@ -29,11 +30,17 @@ class VmPool:
     def __init__(self):
         self.counter = settings.START_ID_INDEX
         self.executions = {}
+
         self.network = (
             Network(
-                vm_address_pool_range=settings.IPV4_ADDRESS_POOL,
+                vm_ipv4_address_pool_range=settings.IPV4_ADDRESS_POOL,
                 vm_network_size=settings.IPV4_NETWORK_PREFIX_LENGTH,
                 external_interface=settings.NETWORK_INTERFACE,
+                ipv6_allocator=make_ipv6_allocator(
+                    allocation_policy=settings.IPV6_ALLOCATION_POLICY,
+                    address_pool=settings.IPV6_ADDRESS_POOL,
+                    subnet_prefix=settings.IPV6_SUBNET_PREFIX,
+                ),
             )
             if settings.ALLOW_VM_NETWORKING
             else None
@@ -49,7 +56,8 @@ async def create_a_vm(
         vm_id = self.get_unique_vm_id()
 
         if self.network:
-            tap_interface = await self.network.create_tap(vm_id)
+            vm_type = VmType.from_message_content(message)
+            tap_interface = await self.network.create_tap(vm_id, vm_hash, vm_type)
         else:
             tap_interface = None
 
diff --git a/vm_supervisor/views.py b/vm_supervisor/views.py
index db547293d..6a838b1b8 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views.py
@@ -62,7 +62,9 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
 
     message_ref_base32 = request.host.split(".")[0]
     if settings.FAKE_DATA_PROGRAM:
-        message_ref = ItemHash("fake-hash")
+        message_ref = ItemHash(
+            "cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe"
+        )
     else:
         try:
             message_ref = ItemHash(b32_to_b16(message_ref_base32).decode())
diff --git a/vm_supervisor/vm/firecracker/executable.py b/vm_supervisor/vm/firecracker/executable.py
index 3421ae248..462b6bd89 100644
--- a/vm_supervisor/vm/firecracker/executable.py
+++ b/vm_supervisor/vm/firecracker/executable.py
@@ -182,11 +182,25 @@ def __init__(
         self.guest_api_process = None
         self._firecracker_config = None
 
-    def get_vm_ip(self):
-        return self.tap_interface.guest_ip.with_prefixlen
-
-    def get_vm_route(self):
-        return str(self.tap_interface.host_ip).split("/", 1)[0]
+    def get_vm_ip(self) -> Optional[str]:
+        if self.tap_interface:
+            return self.tap_interface.guest_ip.with_prefixlen
+        return None
+
+    def get_vm_route(self) -> Optional[str]:
+        if self.tap_interface:
+            return str(self.tap_interface.host_ip).split("/", 1)[0]
+        return None
+
+    def get_vm_ipv6(self) -> Optional[str]:
+        if self.tap_interface:
+            return self.tap_interface.guest_ipv6.with_prefixlen
+        return None
+
+    def get_vm_ipv6_gateway(self) -> Optional[str]:
+        if self.tap_interface:
+            return str(self.tap_interface.host_ipv6.ip)
+        return None
 
     def to_dict(self):
         """Dict representation of the virtual machine. Used to record resource usage and for JSON serialization."""
diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index cb2cfb9db..311fa2848 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -170,6 +170,8 @@ def _create_network_file(self) -> bytes:
 
         ip = self.get_vm_ip()
         route = self.get_vm_route()
+        ipv6 = self.get_vm_ipv6()
+        ipv6_gateway = self.get_vm_ipv6_gateway()
 
         network = {
             "network": {
@@ -177,8 +179,9 @@ def _create_network_file(self) -> bytes:
                     "eth0": {
                         "dhcp4": False,
                         "dhcp6": False,
-                        "addresses": [str(ip)],
+                        "addresses": [ip, ipv6],
                         "gateway4": route,
+                        "gateway6": ipv6_gateway,
                         "nameservers": {
                             "addresses": settings.DNS_NAMESERVERS,
                         },
diff --git a/vm_supervisor/vm/firecracker/program.py b/vm_supervisor/vm/firecracker/program.py
index 257f04a29..ba6404047 100644
--- a/vm_supervisor/vm/firecracker/program.py
+++ b/vm_supervisor/vm/firecracker/program.py
@@ -85,6 +85,7 @@ class ProgramVmConfiguration:
     interface: Interface
     vm_hash: ItemHash
     ip: Optional[str] = None
+    ipv6: Optional[str] = None
     route: Optional[str] = None
     dns_servers: List[str] = field(default_factory=list)
     volumes: List[Volume] = field(default_factory=list)
@@ -105,7 +106,9 @@ class ConfigurationPayload:
     entrypoint: str
     code: Optional[bytes] = None
     ip: Optional[str] = None
+    ipv6: Optional[str] = None
     route: Optional[str] = None
+    ipv6_gateway: Optional[str] = None
     dns_servers: List[str] = field(default_factory=list)
     volumes: List[Volume] = field(default_factory=list)
     variables: Optional[Dict[str, str]] = None
@@ -316,20 +319,23 @@ async def _setup_configuration(
         machine to send this configuration. Other modes may use Cloud-init, ..."""
         reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
 
-        # The ip and route should not contain the network mask in order to maintain
-        # compatibility with the existing runtimes.
-        if self.enable_networking and self.tap_interface:
-            ip = self.get_vm_ip().split("/", 1)[0]
-            route = self.get_vm_route()
-        else:
-            ip, route = None, None
+        ip = self.get_vm_ip()
+        if ip:
+            # The ip and route should not contain the network mask in order to maintain
+            # compatibility with the existing runtimes.
+            ip = ip.split("/", 1)[0]
+        route = self.get_vm_route()
+        ipv6 = self.get_vm_ipv6()
+        ipv6_gateway = self.get_vm_ipv6_gateway()
 
         if not settings.DNS_NAMESERVERS:
             raise ValueError("Invalid configuration: DNS nameservers missing")
 
         config = ConfigurationPayload(
             ip=ip,
+            ipv6=ipv6,
             route=route,
+            ipv6_gateway=ipv6_gateway,
             dns_servers=settings.DNS_NAMESERVERS,
             code=code,
             encoding=self.resources.code_encoding,
diff --git a/vm_supervisor/vm/vm_type.py b/vm_supervisor/vm/vm_type.py
new file mode 100644
index 000000000..7e568862b
--- /dev/null
+++ b/vm_supervisor/vm/vm_type.py
@@ -0,0 +1,21 @@
+from enum import Enum
+
+from aleph_message.models import ExecutableContent, InstanceContent, ProgramContent
+
+
+class VmType(Enum):
+    microvm = 1
+    persistent_program = 2
+    instance = 3
+
+    @staticmethod
+    def from_message_content(content: ExecutableContent) -> "VmType":
+        if isinstance(content, InstanceContent):
+            return VmType.instance
+
+        elif isinstance(content, ProgramContent):
+            if content.on.persistent:
+                return VmType.persistent_program
+            return VmType.microvm
+
+        raise TypeError(f"Unexpected message content type: {type(content)}")

From b56f773a554a69acc6e71371e6f1755193328d7e Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Thu, 29 Jun 2023 15:58:51 +0200
Subject: [PATCH 424/990] Fix: Sending IPv6 configuration crashed old runtimes

Problem: the supervisor cannot deduce the version of the init process
of the microVM runtime. This leads to compatibility issues when adding
new features (like IPv6 support in this case).

Solution: define a version for the init process and send this value
to the supervisor when the VM boots. This version is then used to
gatekeep features when configuring the VM.

Co-authored-by: Hugo Herter 
---
 firecracker/microvm.py                   | 36 ++++++++--
 runtimes/aleph-debian-11-python/init1.py |  2 +
 vm_supervisor/utils.py                   | 15 ++++
 vm_supervisor/vm/firecracker/program.py  | 89 ++++++++++++++++++++----
 4 files changed, 123 insertions(+), 19 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 34227dcca..dd5b3ef8c 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -6,11 +6,14 @@
 import string
 from asyncio import Task
 from asyncio.base_events import Server
+from dataclasses import dataclass
 from os import getuid
 from pathlib import Path
 from pwd import getpwnam
 from tempfile import NamedTemporaryFile
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, Any, Dict
+
+import msgpack
 
 from .config import Drive, FirecrackerConfig
 
@@ -58,6 +61,14 @@ async def setfacl():
         logger.warning(f"[stderr]\n{stderr.decode()}")
 
 
+@dataclass
+class RuntimeConfiguration:
+    version: str
+
+    def supports_ipv6(self) -> bool:
+        return self.version != "1.0.0"
+
+
 class MicroVM:
     vm_id: int
     use_jailer: bool
@@ -110,6 +121,7 @@ def __init__(
         self.jailer_bin_path = jailer_bin_path
         self.drives = []
         self.init_timeout = init_timeout
+        self.runtime_config = None
 
     def to_dict(self):
         return {
@@ -278,8 +290,7 @@ def enable_file_rootfs(self, path_on_host: Path) -> Path:
             return path_on_host
 
     def enable_device_mapper_rootfs(self, path_on_host: Path) -> Path:
-        """Mount a rootfs to the VM.
-        """
+        """Mount a rootfs to the VM."""
         self.mounted_rootfs = path_on_host
         if not self.use_jailer:
             return path_on_host
@@ -358,15 +369,28 @@ async def wait_for_init(self):
         logger.debug("Waiting for init...")
         queue = asyncio.Queue()
 
-        async def unix_client_connected(*_):
-            await queue.put(True)
+        async def unix_client_connected(
+            reader: asyncio.StreamReader, _writer: asyncio.StreamWriter
+        ):
+            data = await reader.read(1_000_000)
+            if data:
+                config_dict: Dict[str, Any] = msgpack.loads(data)
+                runtime_config = RuntimeConfiguration(version=config_dict["version"])
+            else:
+                # Older runtimes do not send a config. Use a default.
+                runtime_config = RuntimeConfiguration(version="1.0.0")
+
+            logger.debug("Runtime version: %s", runtime_config)
+            await queue.put(runtime_config)
 
         self._unix_socket = await asyncio.start_unix_server(
             unix_client_connected, path=f"{self.vsock_path}_52"
         )
         system(f"chown jailman:jailman {self.vsock_path}_52")
         try:
-            await asyncio.wait_for(queue.get(), timeout=self.init_timeout)
+            self.runtime_config = await asyncio.wait_for(
+                queue.get(), timeout=self.init_timeout
+            )
             logger.debug("...signal from init received")
         except asyncio.TimeoutError:
             logger.warning("Never received signal from init")
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index aca160f0c..c012688a7 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -31,6 +31,7 @@
 
 logger.debug("Imports finished")
 
+__version__ = "2.0.0"
 ASGIApplication = NewType("ASGIApplication", Any)
 
 
@@ -86,6 +87,7 @@ class RunCodePayload:
 # Send the host that we are ready
 s0 = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
 s0.connect((2, 52))
+s0.sendall(msgpack.dumps({"version": __version__}))
 s0.close()
 
 # Configure aleph-client to use the guest API
diff --git a/vm_supervisor/utils.py b/vm_supervisor/utils.py
index 2f23d414c..3acd264f9 100644
--- a/vm_supervisor/utils.py
+++ b/vm_supervisor/utils.py
@@ -1,4 +1,5 @@
 import asyncio
+import dataclasses
 import hashlib
 import json
 import logging
@@ -9,10 +10,24 @@
 from typing import Any, Coroutine, Dict, List, Optional
 
 import aiodns
+import msgpack
 
 logger = logging.getLogger(__name__)
 
 
+class MsgpackSerializable:
+    def __post_init__(self, *args, **kwargs):
+        if not is_dataclass(self):
+            raise TypeError(f"Decorated class must be a dataclass: {self}")
+        super().__init_subclass__(*args, **kwargs)
+
+    def as_msgpack(self) -> bytes:
+        if is_dataclass(self):
+            return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True)  # type: ignore
+        else:
+            raise TypeError(f"Decorated class must be a dataclass: {self}")
+
+
 def b32_to_b16(hash: str) -> bytes:
     """Convert base32 encoded bytes to base16 encoded bytes."""
     # Add padding
diff --git a/vm_supervisor/vm/firecracker/program.py b/vm_supervisor/vm/firecracker/program.py
index ba6404047..0dda10009 100644
--- a/vm_supervisor/vm/firecracker/program.py
+++ b/vm_supervisor/vm/firecracker/program.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import dataclasses
 import logging
@@ -21,12 +23,13 @@
     NetworkInterface,
     Vsock,
 )
-from firecracker.microvm import setfacl
+from firecracker.microvm import RuntimeConfiguration, setfacl
 from vm_supervisor.conf import settings
 from vm_supervisor.models import ExecutableContent
 from vm_supervisor.network.interfaces import TapInterface
 from vm_supervisor.storage import get_code_path, get_data_path, get_runtime_path
 
+from ...utils import MsgpackSerializable
 from .executable import (
     AlephFirecrackerExecutable,
     AlephFirecrackerResources,
@@ -81,7 +84,7 @@ def from_entrypoint(cls, entrypoint: str):
 
 
 @dataclass
-class ProgramVmConfiguration:
+class ProgramVmConfiguration(MsgpackSerializable):
     interface: Interface
     vm_hash: ItemHash
     ip: Optional[str] = None
@@ -91,12 +94,60 @@ class ProgramVmConfiguration:
     volumes: List[Volume] = field(default_factory=list)
     variables: Optional[Dict[str, str]] = None
 
-    def as_msgpack(self) -> bytes:
-        return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True)
+
+@dataclass
+class ConfigurationPayload(MsgpackSerializable):
+    ...
+
+
+@dataclass
+class ConfigurationPayloadV1(ConfigurationPayload):
+    """
+    Configuration payload for runtime v1.
+    """
+
+    input_data: Optional[bytes]
+    interface: Interface
+    vm_hash: str
+    encoding: Encoding
+    entrypoint: str
+    code: Optional[bytes]
+    ip: Optional[str]
+    route: Optional[str]
+    dns_servers: List[str]
+    volumes: List[Volume]
+    variables: Optional[Dict[str, str]]
+
+    @classmethod
+    def from_program_config(
+        cls, program_config: ProgramConfiguration
+    ) -> ConfigurationPayload:
+        """Converts a program configuration into a configuration payload
+        to be sent to a runtime.
+        """
+        field_names = set(f.name for f in dataclasses.fields(cls))
+        return cls(
+            **{
+                k: v
+                for k, v in dataclasses.asdict(program_config).items()
+                if k in field_names
+            }
+        )
 
 
 @dataclass
-class ConfigurationPayload:
+class ConfigurationPayloadV2(ConfigurationPayloadV1):
+    """
+    Configuration payload for runtime v2.
+    Adds support for IPv6.
+    """
+
+    ipv6: Optional[str]
+    ipv6_gateway: Optional[str]
+
+
+@dataclass
+class ProgramConfiguration:
     """Configuration passed to the init of the virtual machine in order to start the program."""
 
     input_data: Optional[bytes]
@@ -113,8 +164,18 @@ class ConfigurationPayload:
     volumes: List[Volume] = field(default_factory=list)
     variables: Optional[Dict[str, str]] = None
 
-    def as_msgpack(self) -> bytes:
-        return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True)
+    def to_runtime_format(
+        self, runtime_config: RuntimeConfiguration
+    ) -> ConfigurationPayload:
+        if runtime_config.version == "1.0.0":
+            return ConfigurationPayloadV1.from_program_config(self)
+
+        if runtime_config.version != "2.0.0":
+            logger.warning(
+                "This runtime version may be unsupported: %s", runtime_config.version
+            )
+
+        return ConfigurationPayloadV2.from_program_config(self)
 
 
 @dataclass
@@ -127,14 +188,11 @@ class ConfigurationResponse:
 
 
 @dataclass
-class RunCodePayload:
+class RunCodePayload(MsgpackSerializable):
     """Information passed to the init of the virtual machine to launch a function/path of the program."""
 
     scope: Dict
 
-    def as_msgpack(self) -> bytes:
-        return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True)
-
 
 class AlephProgramResources(AlephFirecrackerResources):
     """Resources required by the virtual machine in order to launch the program.
@@ -331,7 +389,10 @@ async def _setup_configuration(
         if not settings.DNS_NAMESERVERS:
             raise ValueError("Invalid configuration: DNS nameservers missing")
 
-        config = ConfigurationPayload(
+        runtime_config = self.fvm.runtime_config
+        assert runtime_config
+
+        program_config = ProgramConfiguration(
             ip=ip,
             ipv6=ipv6,
             route=route,
@@ -346,7 +407,9 @@ async def _setup_configuration(
             volumes=volumes,
             variables=self.resources.message_content.variables,
         )
-        payload = config.as_msgpack()
+        # Convert the configuration in a format compatible with the runtime
+        versioned_config = program_config.to_runtime_format(runtime_config)
+        payload = versioned_config.as_msgpack()
         length = f"{len(payload)}\n".encode()
         writer.write(b"CONNECT 52\n" + length + payload)
         await writer.drain()

From a99a2245f9cd28fa563958dd9d0034e3bb6b9fa7 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 3 Jul 2023 13:19:30 +0200
Subject: [PATCH 425/990] Fix: NDP Proxy was sync and mandatory

Problems:
1. Operations on the NDP Proxy were synchronous.
2. Developing on the supervisor required the installation of the NDP Proxy.

Solutions:
1. Make operations on the NDP Proxy asynchronous.
2. Add setting `USE_NDP_PROXY` that permits users to disable the NDP Proxy.
---
 firecracker/__init__.py              |  2 +-
 firecracker/microvm.py               |  2 +-
 vm_supervisor/conf.py                |  4 ++++
 vm_supervisor/network/hostnetwork.py |  6 ++++--
 vm_supervisor/network/interfaces.py  | 11 +++++++----
 vm_supervisor/network/ndp_proxy.py   | 19 ++++++++++---------
 6 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/firecracker/__init__.py b/firecracker/__init__.py
index e24f31fd8..321ad3266 100644
--- a/firecracker/__init__.py
+++ b/firecracker/__init__.py
@@ -1,2 +1,2 @@
-from .microvm import MicroVM
 from .config import FirecrackerConfig
+from .microvm import MicroVM
diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index dd5b3ef8c..09d518947 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -11,7 +11,7 @@
 from pathlib import Path
 from pwd import getpwnam
 from tempfile import NamedTemporaryFile
-from typing import List, Optional, Tuple, Any, Dict
+from typing import Any, Dict, List, Optional, Tuple
 
 import msgpack
 
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 8a543ff92..e3933f123 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -108,6 +108,10 @@ class Settings(BaseSettings):
         description="IPv6 subnet prefix for VMs. Made configurable for testing.",
     )
     NFTABLES_CHAIN_PREFIX = "aleph"
+    USE_NDP_PROXY: bool = Field(
+        default=True,
+        description="Use the Neighbor Discovery Protocol Proxy to respond to Router Solicitation for instances on IPv6",
+    )
 
     DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf
     DNS_NAMESERVERS: Optional[List[str]] = None
diff --git a/vm_supervisor/network/hostnetwork.py b/vm_supervisor/network/hostnetwork.py
index d363452b1..cae4a0595 100644
--- a/vm_supervisor/network/hostnetwork.py
+++ b/vm_supervisor/network/hostnetwork.py
@@ -5,7 +5,7 @@
 
 from aleph_message.models import ItemHash
 
-from vm_supervisor.conf import IPv6AllocationPolicy
+from vm_supervisor.conf import IPv6AllocationPolicy, settings
 
 from ..vm.vm_type import VmType
 from .firewall import initialize_nftables, setup_nftables_for_vm, teardown_nftables
@@ -123,6 +123,7 @@ class Network:
     ipv6_address_pool: IPv6Network
     network_size: int
     external_interface: str
+    ndp_proxy: Optional[NdpProxy] = None
 
     IPV6_SUBNET_PREFIX: int = 124
 
@@ -149,7 +150,8 @@ def __init__(
         self.enable_ipv4_forwarding()
         self.enable_ipv6_forwarding()
 
-        self.ndp_proxy = NdpProxy(host_network_interface=external_interface)
+        if settings.USE_NDP_PROXY:
+            self.ndp_proxy = NdpProxy(host_network_interface=external_interface)
 
         initialize_nftables()
 
diff --git a/vm_supervisor/network/interfaces.py b/vm_supervisor/network/interfaces.py
index d20c65a2c..109a1d8b9 100644
--- a/vm_supervisor/network/interfaces.py
+++ b/vm_supervisor/network/interfaces.py
@@ -1,8 +1,9 @@
 import asyncio
 import logging
 import shutil
-from ipaddress import IPv4Interface, IPv6Address, IPv6Interface, IPv6Network
+from ipaddress import IPv4Interface, IPv6Interface, IPv6Network
 from subprocess import run
+from typing import Optional
 
 from .ipaddresses import IPv4NetworkWithInterfaces
 from .ndp_proxy import NdpProxy
@@ -20,7 +21,7 @@ def __init__(
         device_name: str,
         ip_network: IPv4NetworkWithInterfaces,
         ipv6_network: IPv6Network,
-        ndp_proxy: NdpProxy,
+        ndp_proxy: Optional[NdpProxy],
     ):
         self.device_name: str = device_name
         self.ip_network: IPv4NetworkWithInterfaces = ip_network
@@ -70,7 +71,8 @@ async def create(self):
             ]
         )
         run([ip_command, "link", "set", self.device_name, "up"])
-        self.ndp_proxy.add_range(self.device_name, ipv6_gateway.network)
+        if self.ndp_proxy:
+            await self.ndp_proxy.add_range(self.device_name, ipv6_gateway.network)
         logger.debug(f"Network interface created: {self.device_name}")
 
     async def delete(self) -> None:
@@ -78,5 +80,6 @@ async def delete(self) -> None:
         Then removes the interface from the host."""
         logger.debug(f"Removing interface {self.device_name}")
         await asyncio.sleep(0.1)  # Avoids Device/Resource busy bug
-        self.ndp_proxy.delete_range(self.device_name)
+        if self.ndp_proxy:
+            await self.ndp_proxy.delete_range(self.device_name)
         run(["ip", "tuntap", "del", self.device_name, "mode", "tap"])
diff --git a/vm_supervisor/network/ndp_proxy.py b/vm_supervisor/network/ndp_proxy.py
index f5ccc3751..577114059 100644
--- a/vm_supervisor/network/ndp_proxy.py
+++ b/vm_supervisor/network/ndp_proxy.py
@@ -13,9 +13,10 @@
 from dataclasses import dataclass
 from ipaddress import IPv6Network
 from pathlib import Path
-from subprocess import run
 from typing import Dict
 
+from vm_supervisor.utils import run_in_subprocess
+
 logger = logging.getLogger(__name__)
 
 
@@ -30,28 +31,28 @@ def __init__(self, host_network_interface: str):
         self.interface_address_range_mapping: Dict[str, IPv6Network] = {}
 
     @staticmethod
-    def _restart_ndppd():
+    async def _restart_ndppd():
         logger.debug("Restarting ndppd")
-        run(["systemctl", "restart", "ndppd"])
+        await run_in_subprocess(["systemctl", "restart", "ndppd"])
 
-    def _update_ndppd_conf(self):
+    async def _update_ndppd_conf(self):
         config = f"proxy {self.host_network_interface} {{\n"
         for interface, address_range in self.interface_address_range_mapping.items():
             config += f"  rule {address_range} {{\n    iface {interface}\n  }}\n"
         config += "}\n"
         Path("/etc/ndppd.conf").write_text(config)
-        self._restart_ndppd()
+        await self._restart_ndppd()
 
-    def add_range(self, interface: str, address_range: IPv6Network):
+    async def add_range(self, interface: str, address_range: IPv6Network):
         logger.debug("Proxying range %s -> %s", address_range, interface)
         self.interface_address_range_mapping[interface] = address_range
-        self._update_ndppd_conf()
+        await self._update_ndppd_conf()
 
-    def delete_range(self, interface: str):
+    async def delete_range(self, interface: str):
         try:
             address_range = self.interface_address_range_mapping.pop(interface)
             logger.debug("Deactivated proxying for %s (%s)", interface, address_range)
         except KeyError:
             return
 
-        self._update_ndppd_conf()
+        await self._update_ndppd_conf()

From 880b2b8bc5dd9d8da8cf4aa3885c3d3aae2d367b Mon Sep 17 00:00:00 2001
From: 1yam <40899431+1yam@users.noreply.github.com>
Date: Mon, 3 Jul 2023 17:24:29 +0200
Subject: [PATCH 426/990] Fix: Outdated command in tutorials (#354)

Solution: add sub-command upload
---
 tutorials/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tutorials/README.md b/tutorials/README.md
index 291bd7ed7..d96bc3413 100644
--- a/tutorials/README.md
+++ b/tutorials/README.md
@@ -187,7 +187,7 @@ aleph --help
 Upload your program:
 
 ```shell
-aleph program ./my-program main:app
+aleph program upload ./my-program main:app
 ```
 
 Press Enter at the following prompt to use the default runtime:

From 64d72f41edc8609da0a7ecda33718d82e786dceb Mon Sep 17 00:00:00 2001
From: 1yam <40899431+1yam@users.noreply.github.com>
Date: Tue, 4 Jul 2023 11:14:49 +0200
Subject: [PATCH 427/990] Update: Upgrade to Firecracker v1.3.3

Solution:
- Update link to v1.3.3
- Update cp command to exclude files with .debug extension
---
 packaging/Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 4bd076703..5c07899d0 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -30,10 +30,10 @@ debian-package-resources: firecracker-bins vmlinux
 firecracker-bins: target-dir build-dir
 	mkdir -p ./build/firecracker-release
 	# Download latest release
-	curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.1.0/firecracker-v1.1.0-x86_64.tgz | tar -xz --directory ./build/firecracker-release
+	curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.3.3/firecracker-v1.3.3-x86_64.tgz | tar -xz --directory ./build/firecracker-release
 	# Copy binaries:
-	cp ./build/firecracker-release/release-v*/firecracker-v* ./target/firecracker
-	cp ./build/firecracker-release/release-v*/jailer-v* ./target/jailer
+	cp ./build/firecracker-release/release-v*/firecracker-v*[!.debug] ./target/firecracker
+	cp ./build/firecracker-release/release-v*/jailer-v*[!.debug] ./target/jailer
 	chmod +x ./target/firecracker
 	chmod +x ./target/jailer
 

From 59ab9d8ca0f94ea04339f303e60b324a07e787e8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 4 Jul 2023 10:08:28 +0200
Subject: [PATCH 428/990] Fix: Unused variable could be removed

---
 vm_supervisor/vm/firecracker/executable.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/vm_supervisor/vm/firecracker/executable.py b/vm_supervisor/vm/firecracker/executable.py
index 462b6bd89..4e8bae86e 100644
--- a/vm_supervisor/vm/firecracker/executable.py
+++ b/vm_supervisor/vm/firecracker/executable.py
@@ -86,7 +86,6 @@ class AlephFirecrackerResources:
     kernel_image_path: Path
     rootfs_path: Path
     volumes: List[HostVolume]
-    volume_paths: Dict[str, Path]
     namespace: str
 
     def __init__(self, message_content: ExecutableContent, namespace: str):

From 6dc16407f7e60b092ec0d75969cb84ab5ba94113 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 29 Jun 2023 17:26:56 +0200
Subject: [PATCH 429/990] Fix: Could not SSH into a program VM

Problem: Users and developers could not specify SSH keys to connect inside the VM.

Solution: Add a configuration to VMs that forwards SSH public keys.

Allow developers to specify public keys in the configuration and enable these via a CLI argument.
When developer keys are used, keys from the message are ignored.
---
 runtimes/aleph-debian-11-python/init1.py |  8 ++++++++
 vm_supervisor/__main__.py                | 13 ++++++++++++-
 vm_supervisor/conf.py                    | 10 +++++++++-
 vm_supervisor/vm/firecracker/program.py  |  9 +++++++++
 4 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index c012688a7..d92306e55 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -72,6 +72,7 @@ class ConfigurationPayload:
     dns_servers: List[str] = field(default_factory=list)
     volumes: List[Volume] = field(default_factory=list)
     variables: Optional[Dict[str, str]] = None
+    authorized_keys: Optional[List[str]] = None
 
 
 @dataclass
@@ -175,6 +176,12 @@ def setup_input_data(input_data: bytes):
             os.system("unzip -q /opt/input.zip -d /data")
 
 
+def setup_authorized_keys(authorized_keys: Optional[List[str]]) -> None:
+    path = Path("/root/.ssh/authorized_keys")
+    path.parent.mkdir(exist_ok=True)
+    path.write_text("\n".join(key for key in authorized_keys))
+
+
 def setup_volumes(volumes: List[Volume]):
     for volume in volumes:
         logger.debug(f"Mounting /dev/{volume.device} on {volume.mount}")
@@ -506,6 +513,7 @@ def setup_system(config: ConfigurationPayload):
         dns_servers=config.dns_servers,
     )
     setup_input_data(config.input_data)
+    setup_authorized_keys(config.authorized_keys)
     logger.debug("Setup finished")
 
 
diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index c660d8fdb..b20d32e1a 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -21,7 +21,7 @@
 from aleph_message.models import ItemHash
 
 from . import metrics, supervisor
-from .conf import make_db_url, settings
+from .conf import ALLOW_DEVELOPER_SSH_KEYS, make_db_url, settings
 from .pubsub import PubSub
 from .run import run_code_on_event, run_code_on_request, start_persistent_vm
 
@@ -146,6 +146,13 @@ def parse_args(args):
         default=settings.FAKE_INSTANCE_BASE,
         help="Filesystem path of the base for the rootfs of fake instances. An empty value signals a download instead.",
     )
+    parser.add_argument(
+        "--developer-ssh-keys",
+        dest="use_developer_ssh_keys",
+        action="store_true",
+        default=False,
+        help="Authorize the developer's SSH keys to connect instead of those specified in the message",
+    )
     return parser.parse_args(args)
 
 
@@ -298,9 +305,13 @@ def main():
         DEBUG_ASYNCIO=args.debug_asyncio,
         FAKE_INSTANCE_BASE=args.fake_instance_base,
     )
+
     if args.run_fake_instance:
         settings.USE_FAKE_INSTANCE_BASE = True
 
+    if args.use_developer_ssh_keys:
+        settings.USE_DEVELOPER_SSH_KEYS = ALLOW_DEVELOPER_SSH_KEYS
+
     if sentry_sdk:
         if settings.SENTRY_DSN:
             sentry_sdk.init(
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index e3933f123..e51ce9a66 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -6,7 +6,7 @@
 from os.path import abspath, exists, isdir, isfile, join
 from pathlib import Path
 from subprocess import check_output
-from typing import Any, Dict, Iterable, List, NewType, Optional
+from typing import Any, Dict, Iterable, List, Literal, NewType, Optional, Union
 
 from pydantic import BaseSettings, Field
 
@@ -14,6 +14,9 @@
 
 Url = NewType("Url", str)
 
+# This variable may not be set from an environment variable
+ALLOW_DEVELOPER_SSH_KEYS = object()
+
 
 class DnsResolver(str, Enum):
     resolv_conf = "resolv.conf"  # Simply copy from /etc/resolv.conf
@@ -188,7 +191,12 @@ class Settings(BaseSettings):
         "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
     )
 
+    # Developer options
+
     SENTRY_DSN: Optional[str] = None
+    DEVELOPER_SSH_KEYS: Optional[List[str]] = []
+    # Using an object here forces the value to come from Python code and not from an environment variable.
+    USE_DEVELOPER_SSH_KEYS: Union[Literal[False], object] = False
 
     # Fields
     SENSITIVE_FIELDS: List[str] = Field(
diff --git a/vm_supervisor/vm/firecracker/program.py b/vm_supervisor/vm/firecracker/program.py
index 0dda10009..61e1b2ade 100644
--- a/vm_supervisor/vm/firecracker/program.py
+++ b/vm_supervisor/vm/firecracker/program.py
@@ -144,6 +144,7 @@ class ConfigurationPayloadV2(ConfigurationPayloadV1):
 
     ipv6: Optional[str]
     ipv6_gateway: Optional[str]
+    authorized_keys: Optional[List[str]]
 
 
 @dataclass
@@ -163,6 +164,7 @@ class ProgramConfiguration:
     dns_servers: List[str] = field(default_factory=list)
     volumes: List[Volume] = field(default_factory=list)
     variables: Optional[Dict[str, str]] = None
+    authorized_keys: Optional[List[str]] = None
 
     def to_runtime_format(
         self, runtime_config: RuntimeConfiguration
@@ -392,6 +394,12 @@ async def _setup_configuration(
         runtime_config = self.fvm.runtime_config
         assert runtime_config
 
+        authorized_keys: Optional[List[str]]
+        if settings.USE_DEVELOPER_SSH_KEYS:
+            authorized_keys = settings.DEVELOPER_SSH_KEYS
+        else:
+            authorized_keys = self.resources.message_content.authorized_keys
+
         program_config = ProgramConfiguration(
             ip=ip,
             ipv6=ipv6,
@@ -406,6 +414,7 @@ async def _setup_configuration(
             vm_hash=self.vm_hash,
             volumes=volumes,
             variables=self.resources.message_content.variables,
+            authorized_keys=authorized_keys,
         )
         # Convert the configuration in a format compatible with the runtime
         versioned_config = program_config.to_runtime_format(runtime_config)

From c866f040f045c5e04bd0c6466b8089774ebb3f22 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 4 Jul 2023 14:57:19 +0200
Subject: [PATCH 430/990] Feature: Debian 12 was not supported (#349)

---
 .../workflows/test-on-droplet-debian-11.yml   |   2 +-
 .../workflows/test-on-droplet-debian-12.yml   |  89 +++++++++
 .../test-on-droplet-ubuntu-22.04.yml          |   2 +-
 doc/INSTALL-Debian-12.md                      | 171 ++++++++++++++++++
 packaging/Makefile                            |  20 ++
 packaging/debian-12.dockerfile                |  18 ++
 6 files changed, 300 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/test-on-droplet-debian-12.yml
 create mode 100644 doc/INSTALL-Debian-12.md
 create mode 100644 packaging/debian-12.dockerfile

diff --git a/.github/workflows/test-on-droplet-debian-11.yml b/.github/workflows/test-on-droplet-debian-11.yml
index 1327752e0..28f06bc25 100644
--- a/.github/workflows/test-on-droplet-debian-11.yml
+++ b/.github/workflows/test-on-droplet-debian-11.yml
@@ -1,4 +1,4 @@
-name: "Run tests on DigitalOcean Droplet"
+name: "Test DigitalOcean Droplet Bullseye"
 on:
   push
 
diff --git a/.github/workflows/test-on-droplet-debian-12.yml b/.github/workflows/test-on-droplet-debian-12.yml
new file mode 100644
index 000000000..07405fe7d
--- /dev/null
+++ b/.github/workflows/test-on-droplet-debian-12.yml
@@ -0,0 +1,89 @@
+name: "Test on DigitalOcean Droplet Bookworm"
+on:
+  push
+
+jobs:
+  run_debian_12:
+    name: "Run in DigitalOcean Droplet with Debian 12"
+    runs-on: ubuntu-latest
+    concurrency: droplet-aleph-vm-debian-12
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          # Fetch the whole history for all tags and branches (required for aleph.__version__)
+          fetch-depth: 0
+
+      - name: Install doctl
+        uses: digitalocean/action-doctl@v2
+        with:
+          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+
+      - name: Setup SSH private key
+        run: |
+          mkdir ~/.ssh
+          echo $DIGITALOCEAN_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519
+          chmod 0700 ~/.ssh
+          chmod 0600 ~/.ssh/id_ed25519
+        env:
+          DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }}
+
+      - name: Create the Droplet
+        run: |
+          doctl compute droplet create \
+          --image debian-12-x64 \
+          --size c-2 \
+          --region fra1 \
+          --vpc-uuid 8c422d04-5dfa-4eca-add7-1e41b5f60d39 \
+          --enable-ipv6 \
+          --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
+          aleph-vm-ci-debian-12
+
+      - name: Build Debian Package
+        run: |
+          cd packaging && make all-podman-debian-12 && cd ..
+          ls packaging/target
+
+      - name: Wait for the system to setup and boot
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-12 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
+
+      - name: Install Aleph-VM on the Droplet
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-12 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
+          
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
+          ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
+          
+          scp packaging/target/aleph-vm.debian-12.deb root@${DROPLET_IPV4}:/opt
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.debian-12.deb"
+          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
+          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> /etc/aleph-vm/supervisor.env"
+          ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
+
+      - name: Test Aleph-VM on the Droplet
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-12 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          
+          sleep 3
+          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
+          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
+
+      - name: Schedule an instance on the Droplet by faking a call from the scheduler
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-12 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          curl --retry 5 --max-time 10 --fail -X POST -H "Content-Type: application/json" \
+            -H "X-Auth-Signature: test" \
+            -d '{"persistent_vms": [], "instances": ["67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"]}' \
+            "http://${DROPLET_IPV4}:4020/control/allocations"
+
+      - name: Cleanup
+        if: always()
+        run: |
+          doctl compute droplet delete -f aleph-vm-ci-debian-12
+
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index 4860be7b0..a4bed0a4e 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -1,4 +1,4 @@
-name: "Run tests on DigitalOcean Droplet"
+name: "Test on DigitalOcean Droplet Jammy"
 on:
   push
 
diff --git a/doc/INSTALL-Debian-12.md b/doc/INSTALL-Debian-12.md
new file mode 100644
index 000000000..2840920bc
--- /dev/null
+++ b/doc/INSTALL-Debian-12.md
@@ -0,0 +1,171 @@
+# Installing Aleph-VM on a server / Debian 12 Bookworm
+
+## 0. Introduction
+
+For production using official Debian packages.
+
+## 1. Requirements
+
+- A [supported Linux server](../vm_supervisor/README.md#1-supported-platforms)
+- A public domain name from a registrar and top level domain you trust. 
+
+In order to run an official Aleph.im Compute Resource Node (CRN), you will also need the following resources:
+
+- CPU (2 options):
+  - Min. 8 cores / 16 threads, 3.0 ghz+ CPU (gaming CPU for fast boot-up of microVMs)
+  - Min. 12 core / 24 threads, 2.4ghz+ CPU (datacenter CPU for multiple concurrent loads)
+- RAM: 64GB
+- STORAGE: 1TB (NVMe SSD preferred, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
+- BANDWIDTH: Minimum of 500 MB/s
+
+You will need a public domain name with access to add TXT and wildcard records.
+
+> 💡 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
+
+## 2. Installation
+
+Run the following commands as `root`:
+
+First install the [VM-Connector](../vm_connector/README.md) using Docker:
+```shell
+apt update
+apt upgrade
+apt install -y docker.io apparmor-profiles
+docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
+```
+
+Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
+The procedure is similar for updates.
+```shell
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.7/aleph-vm.debian-12.deb
+apt install /opt/aleph-vm.debian-12.deb
+```
+
+Reboot if required (new kernel, ...).
+
+### Configuration
+
+Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
+
+#### Hostname
+
+You will want to insert your domain name in the form of:
+```
+ALEPH_VM_DOMAIN_NAME=vm.example.org
+```
+
+#### Network configuration
+
+On some systems, the default network interface is not `eth0` and you will want to configure the default interface
+by adding:
+```
+ALEPH_VM_NETWORK_INTERFACE=enp0s1
+```
+(don't forget to replace `enp0s1` with the name of your default network interface).
+
+Debian 12 by default uses `/etc/resolv.conf` for DNS resolution. The VM Supervisor uses this by default.
+If your system uses [systemd-resolved](https://manpages.debian.org/bullseye/systemd/systemd-resolved.8.en.html)
+instead, uncomment and add the following setting:
+```
+#ALEPH_VM_DNS_RESOLUTION=resolvctl
+```
+
+> 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
+
+#### Volumes and partitions
+
+Two directories are used to store data from the network:
+- `/var/lib/aleph/vm` contains all the execution and persistent data.
+- `/var/cache/aleph/vm` contains data downloaded from the network.
+
+These two directories must be stored on the same partition.
+That partition must meet the minimum requirements specified for a CRN.
+
+> 💡 This is required due to the software using hard links to optimize performance and disk usage.
+
+#### Applying changes
+
+Finally, restart the service:
+```shell
+systemctl restart aleph-vm-supervisor
+```
+
+## 3. Reverse Proxy
+
+We document how to use Caddy as a reverse proxy since it manages and renews HTTPS certificates automatically.
+
+Any other reverse-proxy (Nginx, HAProxy, Apache2, ...) should do the job as well, just make sure to renew the 
+HTTPS/TLS certificates on time.
+
+First, create a domain name that points to the server on IPv4 (A) and IPv6 (AAAA).
+
+This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CONFIGURE_CADDY.md).
+
+Again, run these commands as `root`:
+```shell
+ apt install -y debian-keyring debian-archive-keyring apt-transport-https
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list
+apt update
+apt install caddy
+```
+
+Then, after replacing the domain `vm.example.org` with your own, use configure Caddy:
+```shell
+cat >/etc/caddy/Caddyfile < ![image](https://user-images.githubusercontent.com/404665/150202090-91a02536-4e04-4af2-967f-fe105d116e1f.png)
+
+If you face an issue, check the logs of the different services for errors:
+
+VM-Supervisor:
+```shell
+journalctl -f -u aleph-vm-supervisor.service 
+```
+
+Caddy:
+```shell
+journalctl -f -u caddy.service 
+```
+
+VM-Connector:
+```shell
+docker logs -f vm-connector
+```
+
+### Common errors
+
+#### "Network interface eth0 does not exist"
+
+Did you update the configuration file `/etc/aleph-vm/supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
+the default network interface of your server ?
+
+#### "Aleph Connector unavailable"
+
+Investigate the installation of the VM-Connector using Docker in step 2.
diff --git a/packaging/Makefile b/packaging/Makefile
index 5c07899d0..76e1d662b 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -68,6 +68,17 @@ all-podman-debian-11: version
 	file target/aleph-vm.deb
 	mv target/aleph-vm.deb target/aleph-vm.debian-11.deb
 
+all-podman-debian-12: version
+	cd .. && podman build -t localhost/aleph-vm-packaging-debian-12:latest -f ./packaging/debian-12.dockerfile .
+	mkdir -p ./target
+	podman run --rm -ti \
+		-w /opt/packaging \
+		-v ./target:/opt/packaging/target \
+		localhost/aleph-vm-packaging-debian-12:latest \
+		make
+	file target/aleph-vm.deb
+	mv target/aleph-vm.deb target/aleph-vm.debian-12.deb
+
 all-podman-ubuntu-2204: version
 	cd .. && podman build -t localhost/aleph-vm-packaging-ubuntu-2204:latest -f ./packaging/ubuntu-22.04.dockerfile .
 	mkdir -p ./target
@@ -88,6 +99,15 @@ requirements-debian-11: all-podman-debian-11
 		debian:bullseye \
 		bash -c "/opt/extract_requirements.sh /mnt/requirements-debian-11.txt"
 
+# extract Python requirements from Debian 12 container
+requirements-debian-12: all-podman-debian-12
+	podman run --rm -ti \
+		-v ./target/aleph-vm.debian-12.deb:/opt/packaging/target/aleph-vm.deb:ro \
+		-v ./extract_requirements.sh:/opt/extract_requirements.sh:ro \
+		-v ./requirements-debian-12.txt:/mnt/requirements-debian-12.txt \
+		debian:bullseye \
+		bash -c "/opt/extract_requirements.sh /mnt/requirements-debian-12.txt"
+
 # extract Python requirements from Ubuntu 22.04 container
 requirements-ubuntu-22.04: all-podman-ubuntu-2204
 	podman run --rm -ti \
diff --git a/packaging/debian-12.dockerfile b/packaging/debian-12.dockerfile
new file mode 100644
index 000000000..2e62644dd
--- /dev/null
+++ b/packaging/debian-12.dockerfile
@@ -0,0 +1,18 @@
+FROM debian:bookworm
+
+RUN apt-get update && apt-get -y upgrade && apt-get install -y \
+    make \
+    git \
+    curl \
+    sudo \
+    python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt
+COPY ../vm_supervisor ./vm_supervisor
+COPY ../guest_api ./guest_api
+COPY ../firecracker ./firecracker
+COPY ../packaging ./packaging
+COPY ../kernels ./kernels
+
+COPY ../examples/ ./examples

From 24742dc608c6153b7a23602ac1fbfe0ecf88552e Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Tue, 4 Jul 2023 15:02:07 +0200
Subject: [PATCH 431/990] Fix: change user when extracting Firecracker archive

---
 .github/workflows/build-deb-package.yml | 2 +-
 docker/vm_supervisor-dev.dockerfile     | 2 +-
 packaging/Makefile                      | 2 +-
 vm_supervisor/README.md                 | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 5e552dca8..a1aa1057e 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -26,7 +26,7 @@ jobs:
           path: packaging/target/aleph-vm.debian-11.deb
 
   build_deb_ubuntu_22_04:
-    name: "Build Debian Package"
+    name: "Build Ubuntu Package"
     runs-on: ubuntu-latest
 
     steps:
diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index efbe3df24..dcc586070 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -12,7 +12,7 @@ RUN useradd jailman
 
 RUN mkdir /opt/firecracker
 RUN chown $(whoami) /opt/firecracker
-RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.1.1/firecracker-v1.1.1-x86_64.tgz | tar -xz --directory /opt/firecracker
+RUN curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.3.3/firecracker-v1.3.3-x86_64.tgz | tar -xz --no-same-owner --directory /opt/firecracker
 RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin
 
 # Link binaries on version-agnostic paths:
diff --git a/packaging/Makefile b/packaging/Makefile
index 76e1d662b..3a222090d 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -30,7 +30,7 @@ debian-package-resources: firecracker-bins vmlinux
 firecracker-bins: target-dir build-dir
 	mkdir -p ./build/firecracker-release
 	# Download latest release
-	curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.3.3/firecracker-v1.3.3-x86_64.tgz | tar -xz --directory ./build/firecracker-release
+	curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.3.3/firecracker-v1.3.3-x86_64.tgz | tar -xz --no-same-owner --directory ./build/firecracker-release
 	# Copy binaries:
 	cp ./build/firecracker-release/release-v*/firecracker-v*[!.debug] ./target/firecracker
 	cp ./build/firecracker-release/release-v*/jailer-v*[!.debug] ./target/jailer
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index 6d9378805..0281c775e 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -65,7 +65,7 @@ useradd jailman
 from the [Firecracker project releases](https://github.com/firecracker-microvm/firecracker/releases):
 ```shell
 mkdir /opt/firecracker
-curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.1.1/firecracker-v1.1.1-x86_64.tgz | tar -xz --directory /opt/firecracker
+curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.3.3/firecracker-v1.3.3-x86_64.tgz | tar -xz --no-same-owner --directory /opt/firecracker
 
 # Link binaries on version-agnostic paths:
 ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker

From acb62feb54bcede23898b654e3bb24e60f38a786 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 4 Jul 2023 09:42:03 +0200
Subject: [PATCH 432/990] Fix: Crash when getting version from apt

The supervisor would not start if the `apt` library is found but the `aleph-vm` package is not installed.

Solution: Handle cases when the package is not found and the apt cache returns None.
---
 vm_supervisor/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm_supervisor/version.py b/vm_supervisor/version.py
index 280d0acc1..2fdf24d17 100644
--- a/vm_supervisor/version.py
+++ b/vm_supervisor/version.py
@@ -21,7 +21,7 @@ def get_version_from_apt() -> Optional[str]:
         import apt
 
         return apt.Cache().get("aleph-vm").installed.version
-    except ImportError:
+    except (ImportError, AttributeError):
         logger.warning("apt version not available")
         return None
 

From f12c553abeb8b3924f7d86511bc96c4405bf9cdd Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 4 Jul 2023 10:07:59 +0200
Subject: [PATCH 433/990] Refactor: Move views in subdirectory

---
 vm_supervisor/{views.py => views/__init__.py}  | 16 ++++++++--------
 vm_supervisor/{ => views}/templates/index.html |  0
 2 files changed, 8 insertions(+), 8 deletions(-)
 rename vm_supervisor/{views.py => views/__init__.py} (95%)
 rename vm_supervisor/{ => views}/templates/index.html (100%)

diff --git a/vm_supervisor/views.py b/vm_supervisor/views/__init__.py
similarity index 95%
rename from vm_supervisor/views.py
rename to vm_supervisor/views/__init__.py
index 6a838b1b8..3a1510b35 100644
--- a/vm_supervisor/views.py
+++ b/vm_supervisor/views/__init__.py
@@ -14,14 +14,14 @@
 
 from packaging.version import InvalidVersion, Version
 
-from . import status
-from .conf import settings
-from .metrics import get_execution_records
-from .pubsub import PubSub
-from .resources import Allocation
-from .run import pool, run_code_on_request, start_persistent_vm
-from .utils import b32_to_b16, dumps_for_json, get_ref_from_dns
-from .version import __version__
+from vm_supervisor import status
+from vm_supervisor.conf import settings
+from vm_supervisor.metrics import get_execution_records
+from vm_supervisor.pubsub import PubSub
+from vm_supervisor.resources import Allocation
+from vm_supervisor.run import pool, run_code_on_request, start_persistent_vm
+from vm_supervisor.utils import b32_to_b16, dumps_for_json, get_ref_from_dns
+from vm_supervisor.version import __version__
 
 logger = logging.getLogger(__name__)
 
diff --git a/vm_supervisor/templates/index.html b/vm_supervisor/views/templates/index.html
similarity index 100%
rename from vm_supervisor/templates/index.html
rename to vm_supervisor/views/templates/index.html

From d1c1b4a9c99a43b709fdc1eabbfb344c2923ba62 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 4 Jul 2023 11:28:31 +0200
Subject: [PATCH 434/990] Feature: Init version was hard to fetch from programs

Problem: The version of init1.py defines some of the capabilities of the platform. This was not exposed to programs running inside virtual machines.

Solution: Expose the version as an environment variable.
---
 runtimes/aleph-debian-11-python/init1.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index d92306e55..e894c3754 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -92,6 +92,7 @@ class RunCodePayload:
 s0.close()
 
 # Configure aleph-client to use the guest API
+os.environ["ALEPH_INIT_VERSION"] = __version__
 os.environ["ALEPH_API_HOST"] = "http://localhost"
 os.environ["ALEPH_API_UNIX_SOCKET"] = "/tmp/socat-socket"
 os.environ["ALEPH_REMOTE_CRYPTO_HOST"] = "http://localhost"

From f4ee81993e238fa24acd8b7df7648ea64ccbce1d Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 4 Jul 2023 09:43:24 +0200
Subject: [PATCH 435/990] Fix: Missing dependencies in Debian package

Package python3-yaml is required to generate the cloud-init configuration.

Package python3-dotenv is required by Pydantic to load settings from a .env file during development.
---
 packaging/aleph-vm/DEBIAN/control | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index fa4c6972a..c47f0fee5 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv
 Section: aleph-im
 Priority: Extra

From b2b4286e1ba90af09b6535852be2723a48618d73 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 4 Jul 2023 15:14:02 +0200
Subject: [PATCH 436/990] Fix: Interface must be up before route creation

---
 runtimes/aleph-debian-11-python/init1.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index e894c3754..e4f055e8b 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -147,6 +147,9 @@ def setup_network(
 
         system(f"ip addr add {ip} dev eth0")
 
+        # Interface must be up before a route can use it
+        system("ip link set eth0 up")
+
         if route:
             system(f"ip route add default via {route} dev eth0")
             logger.debug(f"IP and route set: {ip} via {route}")
@@ -155,13 +158,14 @@ def setup_network(
 
     if ipv6:
         logger.debug("Setting up IPv6")
-        system(f"ip addr add {ipv6} dev eth0")
+        system(f"ip -6 addr add {ipv6} dev eth0")
+
+        # Interface must be up before a route can use it
+        system("ip -6 link set eth0 up")
+
         system(f"ip -6 route add default via {ipv6_gateway} dev eth0")
         logger.debug(f"IPv6 setup to address {ipv6}")
 
-    if ip or ipv6:
-        system("ip link set eth0 up")
-
     with open("/etc/resolv.conf", "wb") as resolvconf_fd:
         for server in dns_servers:
             resolvconf_fd.write(f"nameserver {server}\n".encode())

From 37baa61925b21e5fc2319d1d02ba8ee9182a5809 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 4 Jul 2023 22:00:13 +0200
Subject: [PATCH 437/990] Refactor: Process IPv4 and IPv6 in a similar way

---
 runtimes/aleph-debian-11-python/init1.py | 54 ++++++++++--------------
 1 file changed, 23 insertions(+), 31 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index e4f055e8b..c02762f39 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -114,20 +114,16 @@ def setup_variables(variables: Optional[Dict[str, str]]):
 
 
 def setup_network(
-    ip: Optional[str],
+    ipv4: Optional[str],
     ipv6: Optional[str],
-    route: Optional[str],
+    ipv4_gateway: Optional[str],
     ipv6_gateway: Optional[str],
     dns_servers: Optional[List[str]] = None,
 ):
     """Setup the system with info from the host."""
     dns_servers = dns_servers or []
     if not os.path.exists("/sys/class/net/eth0"):
-        logger.info("No network interface eth0")
-        return
-
-    if not (ip or ipv6):
-        logger.info("No network IP")
+        logger.error("No network interface eth0")
         return
 
     # Configure loopback networking
@@ -135,36 +131,32 @@ def setup_network(
     system("ip addr add ::1/128 dev lo")
     system("ip link set lo up")
 
-    if ip:
-        logger.debug("Setting up IPv4")
+    # Forward compatibility with future supervisors that pass the mask with the IP.
+    if ipv4 and ("/" not in ipv4):
+        logger.warning(
+            "Not passing the mask with the IP is deprecated and will be unsupported"
+        )
+        ipv4 = f"{ipv4}/24"
 
-        # Forward compatibility with future supervisors that pass the mask with the IP.
-        if "/" not in ip:
-            logger.warning(
-                "Not passing the mask with the IP is deprecated and will be unsupported"
-            )
-            ip = f"{ip}/24"
+    addresses = [ip for ip in [ipv4, ipv6] if ip]
+    gateways = [gateway for gateway in [ipv4_gateway, ipv6_gateway] if gateway]
 
+    for address in addresses:
         system(f"ip addr add {ip} dev eth0")
 
-        # Interface must be up before a route can use it
+    # Interface must be up before a route can use it
+    if addresses:
         system("ip link set eth0 up")
+    else:
+        logger.debug("No ip address provided")
 
-        if route:
-            system(f"ip route add default via {route} dev eth0")
-            logger.debug(f"IP and route set: {ip} via {route}")
-        else:
-            logger.warning("IPv4 set with no network route")
-
-    if ipv6:
-        logger.debug("Setting up IPv6")
-        system(f"ip -6 addr add {ipv6} dev eth0")
+    for gateway in gateways:
+        system("ip route add default via {gateway} def eth0")
 
-        # Interface must be up before a route can use it
-        system("ip -6 link set eth0 up")
+    if not gateways:
+        logger.debug("No ip gateway provided")
 
-        system(f"ip -6 route add default via {ipv6_gateway} dev eth0")
-        logger.debug(f"IPv6 setup to address {ipv6}")
+       system("ip link set eth0 up")
 
     with open("/etc/resolv.conf", "wb") as resolvconf_fd:
         for server in dns_servers:
@@ -511,9 +503,9 @@ def setup_system(config: ConfigurationPayload):
     setup_variables(config.variables)
     setup_volumes(config.volumes)
     setup_network(
-        ip=config.ip,
+        ipv4=config.ip,
         ipv6=config.ipv6,
-        route=config.route,
+        ipv4_gateway=config.route,
         ipv6_gateway=config.ipv6_gateway,
         dns_servers=config.dns_servers,
     )

From 1e44eb5f3da4d964e898a79ff7fcbcefe888a6bf Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 16 May 2023 15:39:13 +0200
Subject: [PATCH 438/990] Fix: exemple_fastapi didn't use aleph-sdk-python

---
 examples/example_fastapi/main.py | 35 +++++++++++++++++---------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index 5f66f8624..810295c3a 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -16,11 +16,11 @@
 import aiohttp
 
 logger.debug("import aleph_client")
-from aleph_client.asynchronous import create_post, get_messages
-from aleph_client.chains.remote import RemoteAccount
-from aleph_client.types import StorageEnum
-from aleph_client.vm.app import AlephApp
-from aleph_client.vm.cache import VmCache
+from aleph.sdk.client import AlephClient, AuthenticatedAlephClient
+from aleph.sdk.chains.remote import RemoteAccount
+from aleph.sdk.types import StorageEnum
+from aleph.sdk.vm.app import AlephApp
+from aleph.sdk.vm.cache import VmCache
 
 logger.debug("import fastapi")
 from fastapi import FastAPI
@@ -68,9 +68,10 @@ async def environ() -> Dict[str, str]:
 @app.get("/messages")
 async def read_aleph_messages():
     """Read data from Aleph using the Aleph Client library."""
-    data = await get_messages(
-        hashes=["f246f873c3e0f637a15c566e7a465d2ecbb83eaa024d54ccb8fb566b549a929e"]
-    )
+    async with AlephClient() as client:
+        data = await client.get_messages(
+            hashes=["f246f873c3e0f637a15c566e7a465d2ecbb83eaa024d54ccb8fb566b549a929e"]
+        )
     return {"Messages": data}
 
 
@@ -153,15 +154,17 @@ async def post_a_message():
         "answer": 42,
         "something": "interesting",
     }
-    response = await create_post(
+    async with AuthenticatedAlephClient(
         account=account,
-        post_content=content,
-        post_type="test",
-        ref=None,
-        channel="TEST",
-        inline=True,
-        storage_engine=StorageEnum.storage,
-    )
+    ) as client:
+        response = await client.create_post(
+            post_content=content,
+            post_type="test",
+            ref=None,
+            channel="TEST",
+            inline=True,
+            storage_engine=StorageEnum.storage,
+        )
     return {
         "response": response,
     }

From 290ccb0cb0921578aa51d7c7786fda2b793ab0ff Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 4 Jul 2023 09:40:26 +0200
Subject: [PATCH 439/990] Fix: IPv6 forwarding caused issue on some development
 servers.

Solution: Add a setting to disable IPv6 forwarding. Forwarding is enabled by default.
---
 vm_supervisor/conf.py                |  4 ++++
 vm_supervisor/network/hostnetwork.py | 15 ++++++++-------
 vm_supervisor/pool.py                |  2 ++
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index e51ce9a66..f3dd3befd 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -110,6 +110,10 @@ class Settings(BaseSettings):
         default=124,
         description="IPv6 subnet prefix for VMs. Made configurable for testing.",
     )
+    IPV6_FORWARDING_ENABLED: bool = Field(
+        default=True,
+        description="Enable IPv6 forwarding on the host. Required for IPv6 connectivity in VMs."
+    )
     NFTABLES_CHAIN_PREFIX = "aleph"
     USE_NDP_PROXY: bool = Field(
         default=True,
diff --git a/vm_supervisor/network/hostnetwork.py b/vm_supervisor/network/hostnetwork.py
index cae4a0595..bba6ae362 100644
--- a/vm_supervisor/network/hostnetwork.py
+++ b/vm_supervisor/network/hostnetwork.py
@@ -5,7 +5,7 @@
 
 from aleph_message.models import ItemHash
 
-from vm_supervisor.conf import IPv6AllocationPolicy, settings
+from vm_supervisor.conf import IPv6AllocationPolicy
 
 from ..vm.vm_type import VmType
 from .firewall import initialize_nftables, setup_nftables_for_vm, teardown_nftables
@@ -115,8 +115,8 @@ def make_ipv6_allocator(
 
 
 class Network:
-    ipv4_forward_state_before_setup: Optional[int]
-    ipv6_forward_state_before_setup: Optional[int]
+    ipv4_forward_state_before_setup: Optional[int] = None
+    ipv6_forward_state_before_setup: Optional[int] = None
     ipv4_address_pool: IPv4NetworkWithInterfaces = IPv4NetworkWithInterfaces(
         "172.16.0.0/12"
     )
@@ -133,6 +133,8 @@ def __init__(
         vm_network_size: int,
         external_interface: str,
         ipv6_allocator: IPv6Allocator,
+        use_ndp_proxy: bool,
+        ipv6_forwarding_enabled: bool = True,
     ) -> None:
         """Sets up the Network class with some information it needs so future function calls work as expected"""
         self.ipv4_address_pool = IPv4NetworkWithInterfaces(vm_ipv4_address_pool_range)
@@ -144,13 +146,12 @@ def __init__(
 
         self.network_size = vm_network_size
         self.external_interface = external_interface
-        self.ipv4_forward_state_before_setup = None
-        self.ipv6_forward_state_before_setup = None
 
         self.enable_ipv4_forwarding()
-        self.enable_ipv6_forwarding()
+        if ipv6_forwarding_enabled:
+            self.enable_ipv6_forwarding()
 
-        if settings.USE_NDP_PROXY:
+        if use_ndp_proxy:
             self.ndp_proxy = NdpProxy(host_network_interface=external_interface)
 
         initialize_nftables()
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 59677ae47..5389ee8f1 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -41,6 +41,8 @@ def __init__(self):
                     address_pool=settings.IPV6_ADDRESS_POOL,
                     subnet_prefix=settings.IPV6_SUBNET_PREFIX,
                 ),
+                use_ndp_proxy=settings.USE_NDP_PROXY,
+                ipv6_forwarding_enabled=settings.IPV6_FORWARDING_ENABLED,
             )
             if settings.ALLOW_VM_NETWORKING
             else None

From 67551a002fdf2ffa9497858ed4ded494a26db179 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 6 Jul 2023 10:39:59 +0200
Subject: [PATCH 440/990] Fix: Runtime dependencies were outdated (#309)

---
 runtimes/aleph-debian-11-python/create_disk_image.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
index 243a0fd13..a6e1e85e5 100755
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-11-python/create_disk_image.sh
@@ -31,10 +31,10 @@ apt-get install -y --no-install-recommends --no-install-suggests \
   \
   iputils-ping curl
 
-pip3 install 'fastapi~=0.71.0'
+pip3 install 'fastapi~=0.95.1'
 
 echo "Pip installing aleph-client"
-pip3 install 'aleph-client>=0.4.6' 'coincurve==15.0.0'
+pip3 install 'aleph-sdk-python==0.7.0b1'
 
 # Compile all Python bytecode
 python3 -m compileall -f /usr/local/lib/python3.9

From 576d8d3bdf7102e8b23a59e3f92401d65dbfb8a6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 6 Jul 2023 10:55:28 +0200
Subject: [PATCH 441/990] Fix: Issues in init1.py

Fixes some typos, formatting.

The runtime moved to using the aleph-sdk-python, closing an aiohttp session is not required anymore.
---
 runtimes/aleph-debian-11-python/init1.py | 25 ++++++++----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index c02762f39..214439850 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -142,7 +142,7 @@ def setup_network(
     gateways = [gateway for gateway in [ipv4_gateway, ipv6_gateway] if gateway]
 
     for address in addresses:
-        system(f"ip addr add {ip} dev eth0")
+        system(f"ip addr add {address} dev eth0")
 
     # Interface must be up before a route can use it
     if addresses:
@@ -151,12 +151,12 @@ def setup_network(
         logger.debug("No ip address provided")
 
     for gateway in gateways:
-        system("ip route add default via {gateway} def eth0")
+        system(f"ip route add default via {gateway} def eth0")
 
     if not gateways:
         logger.debug("No ip gateway provided")
 
-       system("ip link set eth0 up")
+        system("ip link set eth0 up")
 
     with open("/etc/resolv.conf", "wb") as resolvconf_fd:
         for server in dns_servers:
@@ -354,15 +354,13 @@ async def make_request(session, scope):
 
 
 def show_loading():
-    body = {
-        "body": Path("/root/loading.html").read_text()
-    }
+    body = {"body": Path("/root/loading.html").read_text()}
     headers = {
         "headers": [
-            [b'Content-Type', b'text/html'],
-            [b'Connection', b'keep-alive'],
-            [b'Keep-Alive', b'timeout=5'],
-            [b'Transfer-Encoding', b'chunked']
+            [b"Content-Type", b"text/html"],
+            [b"Connection", b"keep-alive"],
+            [b"Keep-Alive", b"timeout=5"],
+            [b"Transfer-Encoding", b"chunked"],
         ],
         "status": 503,
     }
@@ -406,13 +404,6 @@ async def process_instruction(
             application.terminate()
             logger.debug("Application terminated")
             # application.communicate()
-        else:
-            # Close the cached session in aleph_client:
-            from aleph_client.asynchronous import get_fallback_session
-
-            session: aiohttp.ClientSession = get_fallback_session()
-            await session.close()
-            logger.debug("Aiohttp cached session closed")
         yield b"STOP\n"
         logger.debug("Supervisor informed of halt")
         raise ShutdownException

From e0d83bc3c8511968ce0fd5d21fb8f8ecf2bccd94 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Wed, 5 Jul 2023 10:54:42 +0200
Subject: [PATCH 442/990] Fix: VM does not boot if no SSH key is specified

Problem: The authorized_keys file could not be generated if
the `config.authorized_keys` settings was None because it was
used in a list comprehension.

Solution: check for None early.
---
 runtimes/aleph-debian-11-python/init1.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index 214439850..e098fa236 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -173,7 +173,7 @@ def setup_input_data(input_data: bytes):
             os.system("unzip -q /opt/input.zip -d /data")
 
 
-def setup_authorized_keys(authorized_keys: Optional[List[str]]) -> None:
+def setup_authorized_keys(authorized_keys: List[str]) -> None:
     path = Path("/root/.ssh/authorized_keys")
     path.parent.mkdir(exist_ok=True)
     path.write_text("\n".join(key for key in authorized_keys))
@@ -501,7 +501,8 @@ def setup_system(config: ConfigurationPayload):
         dns_servers=config.dns_servers,
     )
     setup_input_data(config.input_data)
-    setup_authorized_keys(config.authorized_keys)
+    if authorized_keys := config.authorized_keys:
+        setup_authorized_keys(authorized_keys)
     logger.debug("Setup finished")
 
 

From 5854ccf57ac2163f7d57220bff32d587db600f40 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 6 Jul 2023 10:30:25 +0200
Subject: [PATCH 443/990] Fix: Pytest failed due to missing argument

The method `allocator.allocate_vm_ipv6_subnet` now requires the argument `vm_type`, which was not provided in the tests.
---
 tests/supervisor/test_ipv6_allocator.py | 3 +++
 vm_supervisor/views/__init__.py         | 1 -
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/supervisor/test_ipv6_allocator.py b/tests/supervisor/test_ipv6_allocator.py
index cd85ff4d1..ff354ed86 100644
--- a/tests/supervisor/test_ipv6_allocator.py
+++ b/tests/supervisor/test_ipv6_allocator.py
@@ -1,5 +1,7 @@
 import os
 
+from vm_supervisor.vm.vm_type import VmType
+
 # Avoid failures linked to settings when initializing the global VmPool object
 os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
 
@@ -19,5 +21,6 @@ def test_static_ipv6_allocator():
         vm_hash=ItemHash(
             "8920215b2e961a4d4c59a8ceb2803af53f91530ff53d6704273ab4d380bc6446"
         ),
+        vm_type=VmType.microvm,
     )
     assert ip_subnet == IPv6Network("1111:2222:3333:4444:0001:8920:215b:2e90/124")
diff --git a/vm_supervisor/views/__init__.py b/vm_supervisor/views/__init__.py
index 3a1510b35..f787b4065 100644
--- a/vm_supervisor/views/__init__.py
+++ b/vm_supervisor/views/__init__.py
@@ -13,7 +13,6 @@
 from pydantic import ValidationError
 
 from packaging.version import InvalidVersion, Version
-
 from vm_supervisor import status
 from vm_supervisor.conf import settings
 from vm_supervisor.metrics import get_execution_records

From 0a429f5ce47204af164c46071213d6fe8d54284b Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Thu, 6 Jul 2023 15:06:45 +0200
Subject: [PATCH 444/990] Fix: typos and merge issues in init.py

---
 runtimes/aleph-debian-11-python/init1.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index e098fa236..ff326f632 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -151,13 +151,11 @@ def setup_network(
         logger.debug("No ip address provided")
 
     for gateway in gateways:
-        system(f"ip route add default via {gateway} def eth0")
+        system(f"ip route add default via {gateway} dev eth0")
 
     if not gateways:
         logger.debug("No ip gateway provided")
 
-        system("ip link set eth0 up")
-
     with open("/etc/resolv.conf", "wb") as resolvconf_fd:
         for server in dns_servers:
             resolvconf_fd.write(f"nameserver {server}\n".encode())

From 14f9991ef9c2a77085d0d0dd49335803d851fc45 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Thu, 13 Jul 2023 12:25:38 +0200
Subject: [PATCH 445/990] Fix: CI was broken because of aleph-message (#377)

Problem: we shipped aleph-message 0.4.0a2 instead of the latest
0.4.0.
---
 .github/workflows/test-new-runtime-examples.yml    | 4 ++--
 .github/workflows/test-on-droplet-debian-11.yml    | 5 ++---
 .github/workflows/test-on-droplet-debian-12.yml    | 5 ++---
 .github/workflows/test-on-droplet-ubuntu-22.04.yml | 5 ++---
 docker/vm_supervisor-dev.dockerfile                | 2 +-
 examples/volumes/Dockerfile                        | 2 +-
 packaging/Makefile                                 | 2 +-
 vm_supervisor/README.md                            | 2 +-
 vm_supervisor/conf.py                              | 2 +-
 9 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/test-new-runtime-examples.yml b/.github/workflows/test-new-runtime-examples.yml
index 8305aa984..966cf0bcb 100644
--- a/.github/workflows/test-new-runtime-examples.yml
+++ b/.github/workflows/test-new-runtime-examples.yml
@@ -38,9 +38,9 @@ jobs:
           --image debian-11-x64 \
           --size c-2 \
           --region fra1 \
-          --vpc-uuid 8c422d04-5dfa-4eca-add7-1e41b5f60d39 \
+          --vpc-uuid 992896c8-c089-4da3-9288-f81e28c095a4 \
           --enable-ipv6 \
-          --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
+          --ssh-keys b3:ff:08:7f:57:00:fd:7a:14:00:f2:35:0a:f6:e8:55 \
           aleph-vm-ci-runtime
 
       - name: "Build custom runtime"
diff --git a/.github/workflows/test-on-droplet-debian-11.yml b/.github/workflows/test-on-droplet-debian-11.yml
index 28f06bc25..b99d58ae7 100644
--- a/.github/workflows/test-on-droplet-debian-11.yml
+++ b/.github/workflows/test-on-droplet-debian-11.yml
@@ -35,9 +35,9 @@ jobs:
           --image debian-11-x64 \
           --size c-2 \
           --region fra1 \
-          --vpc-uuid 8c422d04-5dfa-4eca-add7-1e41b5f60d39 \
+          --vpc-uuid 992896c8-c089-4da3-9288-f81e28c095a4 \
           --enable-ipv6 \
-          --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
+          --ssh-keys b3:ff:08:7f:57:00:fd:7a:14:00:f2:35:0a:f6:e8:55 \
           aleph-vm-ci-debian-11
 
       - name: Build Debian Package
@@ -86,4 +86,3 @@ jobs:
         if: always()
         run: |
           doctl compute droplet delete -f aleph-vm-ci-debian-11
-
diff --git a/.github/workflows/test-on-droplet-debian-12.yml b/.github/workflows/test-on-droplet-debian-12.yml
index 07405fe7d..7eecd5825 100644
--- a/.github/workflows/test-on-droplet-debian-12.yml
+++ b/.github/workflows/test-on-droplet-debian-12.yml
@@ -35,9 +35,9 @@ jobs:
           --image debian-12-x64 \
           --size c-2 \
           --region fra1 \
-          --vpc-uuid 8c422d04-5dfa-4eca-add7-1e41b5f60d39 \
+          --vpc-uuid 992896c8-c089-4da3-9288-f81e28c095a4 \
           --enable-ipv6 \
-          --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
+          --ssh-keys b3:ff:08:7f:57:00:fd:7a:14:00:f2:35:0a:f6:e8:55 \
           aleph-vm-ci-debian-12
 
       - name: Build Debian Package
@@ -86,4 +86,3 @@ jobs:
         if: always()
         run: |
           doctl compute droplet delete -f aleph-vm-ci-debian-12
-
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index a4bed0a4e..1a2da380e 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -35,9 +35,9 @@ jobs:
           --image ubuntu-22-04-x64 \
           --size c-2 \
           --region fra1 \
-          --vpc-uuid 8c422d04-5dfa-4eca-add7-1e41b5f60d39 \
+          --vpc-uuid 992896c8-c089-4da3-9288-f81e28c095a4 \
           --enable-ipv6 \
-          --ssh-keys 18:09:36:58:79:44:bb:84:45:c8:6f:9a:f6:b8:0a:c5 \
+          --ssh-keys b3:ff:08:7f:57:00:fd:7a:14:00:f2:35:0a:f6:e8:55 \
           aleph-vm-ci-ubuntu-22-04
 
       - name: Build Ubuntu Package
@@ -91,4 +91,3 @@ jobs:
         if: always()
         run: |
           doctl compute droplet delete -f aleph-vm-ci-ubuntu-22-04
-
diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index dcc586070..ee457230d 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -19,7 +19,7 @@ RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://s3.amazonaws.com/spec.ccf
 RUN ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker
 RUN ln /opt/firecracker/release-*/jailer-v* /opt/firecracker/jailer
 
-RUN pip3 install typing-extensions 'aleph-message==0.4.0a2'
+RUN pip3 install typing-extensions 'aleph-message==0.4.0'
 
 RUN mkdir -p /var/lib/aleph/vm/jailer
 
diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile
index d21bffb23..6b85c1fff 100644
--- a/examples/volumes/Dockerfile
+++ b/examples/volumes/Dockerfile
@@ -6,6 +6,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 RUN python3 -m venv /opt/venv
-RUN /opt/venv/bin/pip install 'aleph-message==0.4.0a2'
+RUN /opt/venv/bin/pip install 'aleph-message==0.4.0'
 
 CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs
diff --git a/packaging/Makefile b/packaging/Makefile
index 3a222090d..fae419292 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -17,7 +17,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.0a2'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index 0281c775e..16a4a1414 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -87,7 +87,7 @@ is used to parse and validate Aleph messages.
 ```shell
 apt install -y --no-install-recommends --no-install-suggests python3-pip
 pip3 install pydantic[dotenv]
-pip3 install 'aleph-message==0.4.0a2'
+pip3 install 'aleph-message==0.4.0'
 ```
 
 ### 2.f. Create the jailer working directory:
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index f3dd3befd..2fc00b091 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -112,7 +112,7 @@ class Settings(BaseSettings):
     )
     IPV6_FORWARDING_ENABLED: bool = Field(
         default=True,
-        description="Enable IPv6 forwarding on the host. Required for IPv6 connectivity in VMs."
+        description="Enable IPv6 forwarding on the host. Required for IPv6 connectivity in VMs.",
     )
     NFTABLES_CHAIN_PREFIX = "aleph"
     USE_NDP_PROXY: bool = Field(

From 0c2c79560249b8ead442d6536d7918d93586e6ad Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Thu, 13 Jul 2023 12:55:23 +0200
Subject: [PATCH 446/990] Fixed incorrect IP value to check on init (#375)

---
 vm_supervisor/vm/firecracker/instance.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index 311fa2848..7d886753a 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -118,18 +118,22 @@ async def setup(self):
 
     async def wait_for_init(self) -> None:
         """Wait for the init process of the instance to be ready."""
-        if not self.vm_configuration:
-            raise ValueError("The VM has not been configured yet")
+        assert (
+            self.enable_networking and self.tap_interface
+        ), f"Network not enabled for VM {self.vm_id}"
+
+        ip = self.get_vm_ip()
+        if not ip:
+            raise ValueError("Host IP not available")
 
-        if not self.vm_configuration.ip:
-            raise ValueError("VM IP address not set")
+        ip = ip.split("/", 1)[0]
 
-        attempts = 5
+        attempts = 10
         timeout_seconds = 1.0
 
         for attempt in range(attempts):
             try:
-                await ping(self.vm_configuration.ip, packets=1, timeout=timeout_seconds)
+                await ping(ip, packets=1, timeout=timeout_seconds)
                 return
             except HostNotFoundError:
                 if attempt < (attempts - 1):

From 3aba1a7f8dad08677e66749fc6af54e1755b4c99 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Fri, 14 Jul 2023 13:56:49 +0200
Subject: [PATCH 447/990] Fix: CI for Ubuntu was failing too often because of
 apt/dpkg locks (#380)

Problem: the CI is looking for only one lock file, but apt/dpkg use several. Furthermore, `lslocks --json` seems to cut its output to a specific width, and configuration options (`--notruncate`) do not appear to have an impact.

Solution: repeat calls to apt-get update until we get the lock. This is not perfect but increases the rate of success.
---
 .github/workflows/test-on-droplet-ubuntu-22.04.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index 1a2da380e..35e80fe81 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -33,7 +33,7 @@ jobs:
         run: |
           doctl compute droplet create \
           --image ubuntu-22-04-x64 \
-          --size c-2 \
+          --size c-4 \
           --region fra1 \
           --vpc-uuid 992896c8-c089-4da3-9288-f81e28c095a4 \
           --enable-ipv6 \
@@ -55,11 +55,9 @@ jobs:
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-ubuntu-22-04 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
           
-          # Ubuntu droplets run upgrades at boot
+          # Ubuntu droplets run upgrades at boot, which locks apt-get
           sleep 30
-          until ! ssh root@${DROPLET_IPV4} "lslocks --json | grep /var/lib/dpkg/lock" > /dev/null; do sleep 1; echo "Waiting for dpkg lock..."; done
-          
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
+          until ! ssh root@${DROPLET_IPV4} "apt-get update" > /dev/null; do sleep 1; echo "Waiting for apt/dpkg lock..."; done          
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"

From 017d6dfa44857fc8d0898d5138695fa2af38e025 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Fri, 14 Jul 2023 13:57:26 +0200
Subject: [PATCH 448/990] Create Ubuntu rootfs (#370)

Updated creation script to create a BTRFS image of Ubuntu filesystem.
---
 runtimes/instance-debian-rootfs/Dockerfile    |  59 --
 .../create-debian-disk.sh                     |  33 -
 runtimes/instance-debian-rootfs/init0.sh      |  37 --
 runtimes/instance-debian-rootfs/init1.py      | 573 ------------------
 runtimes/instance-debian-rootfs/inittab       |  22 -
 .../nginx/health-check.conf                   |   9 -
 .../instance-debian-rootfs/nginx/index.html   |  20 -
 .../instance-debian-rootfs/nginx/nginx.conf   |  31 -
 .../instance-debian-rootfs/update_inits.sh    |  14 -
 .../instance-rootfs/create-debian-11-disk.sh  |  51 ++
 .../instance-rootfs/create-debian-12-disk.sh  |  51 ++
 .../create-ubuntu-22-04-disk.sh               |  38 ++
 12 files changed, 140 insertions(+), 798 deletions(-)
 delete mode 100644 runtimes/instance-debian-rootfs/Dockerfile
 delete mode 100755 runtimes/instance-debian-rootfs/create-debian-disk.sh
 delete mode 100644 runtimes/instance-debian-rootfs/init0.sh
 delete mode 100644 runtimes/instance-debian-rootfs/init1.py
 delete mode 100644 runtimes/instance-debian-rootfs/inittab
 delete mode 100644 runtimes/instance-debian-rootfs/nginx/health-check.conf
 delete mode 100644 runtimes/instance-debian-rootfs/nginx/index.html
 delete mode 100644 runtimes/instance-debian-rootfs/nginx/nginx.conf
 delete mode 100755 runtimes/instance-debian-rootfs/update_inits.sh
 create mode 100755 runtimes/instance-rootfs/create-debian-11-disk.sh
 create mode 100755 runtimes/instance-rootfs/create-debian-12-disk.sh
 create mode 100755 runtimes/instance-rootfs/create-ubuntu-22-04-disk.sh

diff --git a/runtimes/instance-debian-rootfs/Dockerfile b/runtimes/instance-debian-rootfs/Dockerfile
deleted file mode 100644
index 3505f7a61..000000000
--- a/runtimes/instance-debian-rootfs/Dockerfile
+++ /dev/null
@@ -1,59 +0,0 @@
-# Pull the minimal Debian image
-FROM debian
-
-# Install Nginx
-RUN apt-get -y update && apt-get -y install nginx
-
-COPY nginx/index.html /usr/share/nginx/html/index.html
-COPY nginx/health-check.conf /etc/nginx/conf.d/health-check.conf
-
-# Install all basic dependencies
-RUN apt-get install -y --no-install-recommends --no-install-suggests \
-  python3-minimal \
-  openssh-server \
-  socat libsecp256k1-0
-
-# Install all needed python modules
-RUN apt-get install -y --no-install-recommends --no-install-suggests \
-  python3-aiohttp python3-msgpack \
-  python3-setuptools python3-dev \
-  python3-pip python3-cytoolz python3-pydantic
-
-# Install NodeJS and some tools
-RUN apt-get install -y --no-install-recommends --no-install-suggests \
-  iproute2 unzip nodejs npm build-essential iputils-ping curl
-
-# Install Docker
-RUN apt-get install -y --no-install-recommends --no-install-suggests \
-  docker.io cgroupfs-mount nftables
-
-# Install Aleph dependencies
-RUN pip3 install 'fastapi~=0.71.0'
-RUN pip3 install 'aleph-client>=0.4.6' 'coincurve==15.0.0'
-
-# Compile all Python bytecode
-RUN python3 -m compileall -f /usr/local/lib/python3.9
-
-# Enable root login by ssh
-RUN echo "PubkeyAuthentication yes" >> ./rootfs/etc/ssh/sshd_config
-RUN echo "PasswordAuthentication no" >> ./rootfs/etc/ssh/sshd_config
-RUN echo "ChallengeResponseAuthentication no" >> ./rootfs/etc/ssh/sshd_config
-RUN echo "PermitRootLogin yes" >> ./rootfs/etc/ssh/sshd_config
-
-# Generate SSH host keys
-#RUN systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
-#RUN systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key
-#RUN systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key
-#RUN systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ed25519 -f /etc/ssh/ssh_host_ed25519_key
-
-# Set up a login terminal on the serial console (ttyS0):
-RUN ln -s agetty /etc/init.d/agetty.ttyS0
-RUN echo ttyS0 > /etc/securetty
-
-# Reduce size
-RUN rm -fr /root/.cache
-RUN rm -fr /var/cache
-RUN mkdir -p /var/cache/apt/archives/partial
-RUN rm -fr /usr/share/doc
-RUN rm -fr /usr/share/man
-RUN rm -fr /var/lib/apt/lists/
diff --git a/runtimes/instance-debian-rootfs/create-debian-disk.sh b/runtimes/instance-debian-rootfs/create-debian-disk.sh
deleted file mode 100755
index 42202047c..000000000
--- a/runtimes/instance-debian-rootfs/create-debian-disk.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/sh
-
-umount /mnt/vm
-rm ./rootfs.ext4
-mkdir -p /mnt/vm
-
-set -euf
-
-echo "Creating rootfs.ext4 file"
-# Create a 1,5 GB partition
-dd if=/dev/zero of=rootfs.ext4 bs=1MB count=1500
-mkfs.ext4 rootfs.ext4
-mount rootfs.ext4 /mnt/vm
-
-echo "Building Docker image"
-rm -rf ./docker-image
-docker buildx build -t docker-image --output type=local,dest=./docker-image .
-
-echo "Adding customizations"
-# Add custom inittab
-cp -vap ./inittab ./docker-image/etc/inittab
-# Copying init scripts
-cp ./init0.sh ./docker-image/sbin/init
-cp ./init1.py ./docker-image/root/init1.py
-chmod +x ./docker-image/sbin/init
-chmod +x ./docker-image/root/init1.py
-
-echo "Copying Docker image content to final rootfs file"
-cp -vap ./docker-image/. /mnt/vm
-umount /mnt/vm
-
-echo "Cleaning Docker generated files"
-rm -rf ./docker-image
diff --git a/runtimes/instance-debian-rootfs/init0.sh b/runtimes/instance-debian-rootfs/init0.sh
deleted file mode 100644
index 8890d0500..000000000
--- a/runtimes/instance-debian-rootfs/init0.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/sh
-
-set -euf
-
-mount -t proc proc /proc -o nosuid,noexec,nodev
-
-log() {
-    echo "$(cat /proc/uptime | awk '{printf $1}')" '|S' "$@"
-}
-log "init0.sh is launching"
-
-mkdir -p /dev/pts
-mkdir -p /dev/shm
-
-mount -t sysfs sys /sys -o nosuid,noexec,nodev
-mount -t tmpfs run /run -o mode=0755,nosuid,nodev
-#mount -t devtmpfs dev /dev -o mode=0755,nosuid
-mount -t devpts devpts /dev/pts -o mode=0620,gid=5,nosuid,noexec
-mount -t tmpfs shm /dev/shm -omode=1777,nosuid,nodev
-
-# List block devices
-lsblk
-
-#cat /proc/sys/kernel/random/entropy_avail
-
-# TODO: Move in init1
-mkdir -p /run/sshd
-/usr/sbin/sshd &
-service nginx start
-log "SSH UP"
-
-log "Setup socat"
-socat UNIX-LISTEN:/tmp/socat-socket,fork,reuseaddr VSOCK-CONNECT:2:53 &
-log "Socat ready"
-
-# Replace this script with the manager
-exec /root/init1.py
diff --git a/runtimes/instance-debian-rootfs/init1.py b/runtimes/instance-debian-rootfs/init1.py
deleted file mode 100644
index 6c2ae0b9a..000000000
--- a/runtimes/instance-debian-rootfs/init1.py
+++ /dev/null
@@ -1,573 +0,0 @@
-#!/usr/bin/python3 -OO
-import base64
-import logging
-
-logging.basicConfig(
-    level=logging.DEBUG,
-    format="%(relativeCreated)4f |V %(levelname)s | %(message)s",
-)
-logger = logging.getLogger(__name__)
-
-logger.debug("Imports starting")
-
-import ctypes
-import asyncio
-import os
-import socket
-from enum import Enum
-import subprocess
-import sys
-import traceback
-from contextlib import redirect_stdout
-from dataclasses import dataclass, field
-from io import StringIO
-from os import system
-from shutil import make_archive
-from typing import Optional, Dict, Any, Tuple, List, NewType, Union, AsyncIterable
-
-import aiohttp
-import msgpack
-
-logger.debug("Imports finished")
-
-ASGIApplication = NewType("ASGIApplication", Any)
-
-
-class Encoding(str, Enum):
-    plain = "plain"
-    zip = "zip"
-    squashfs = "squashfs"
-
-
-class Interface(str, Enum):
-    asgi = "asgi"
-    executable = "executable"
-
-
-class ShutdownException(Exception):
-    pass
-
-
-@dataclass
-class Volume:
-    mount: str
-    device: str
-    read_only: bool
-
-
-@dataclass
-class ConfigurationPayload:
-    input_data: bytes
-    interface: Interface
-    vm_hash: str
-    code: Optional[bytes] = None
-    encoding: Optional[Encoding] = None
-    entrypoint: Optional[str] = None
-    ip: Optional[str] = None
-    route: Optional[str] = None
-    dns_servers: List[str] = field(default_factory=list)
-    volumes: List[Volume] = field(default_factory=list)
-    variables: Optional[Dict[str, str]] = None
-
-
-@dataclass
-class RunCodePayload:
-    scope: Dict
-
-
-# Open a socket to receive instructions from the host
-s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
-s.bind((socket.VMADDR_CID_ANY, 52))
-s.listen()
-
-# Send the host that we are ready
-s0 = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
-s0.connect((2, 52))
-s0.close()
-
-# Configure aleph-client to use the guest API
-os.environ["ALEPH_API_HOST"] = "http://localhost"
-os.environ["ALEPH_API_UNIX_SOCKET"] = "/tmp/socat-socket"
-os.environ["ALEPH_REMOTE_CRYPTO_HOST"] = "http://localhost"
-os.environ["ALEPH_REMOTE_CRYPTO_UNIX_SOCKET"] = "/tmp/socat-socket"
-
-# Additional node modules from immutable volume
-os.environ["NODE_PATH"] = "/opt/node_modules"
-
-logger.debug("init1.py is launching")
-
-
-def setup_hostname(hostname: str):
-    os.environ["ALEPH_ADDRESS_TO_USE"] = hostname
-    system(f"hostname {hostname}")
-
-
-def setup_variables(variables: Optional[Dict[str, str]]):
-    if variables is None:
-        return
-    for key, value in variables.items():
-        os.environ[key] = value
-
-
-def setup_network(
-    ip: Optional[str], route: Optional[str], dns_servers: Optional[List[str]] = None
-):
-    """Setup the system with info from the host."""
-    dns_servers = dns_servers or []
-    if not os.path.exists("/sys/class/net/eth0"):
-        logger.info("No network interface eth0")
-        return
-
-    if not ip:
-        logger.info("No network IP")
-        return
-
-    logger.debug("Setting up networking")
-    system("ip addr add 127.0.0.1/8 dev lo brd + scope host")
-    system("ip addr add ::1/128 dev lo")
-    system("ip link set lo up")
-    if "/" in ip:
-        # Forward compatibility with future supervisors that pass the mask with the IP.
-        system(f"ip addr add {ip} dev eth0")
-    else:
-        logger.warning("Not passing the mask with the IP is deprecated and will be unsupported")
-        system(f"ip addr add {ip}/24 dev eth0")
-    system("ip link set eth0 up")
-
-    if route:
-        system(f"ip route add default via {route} dev eth0")
-        logger.debug(f"IP and route set: {ip} via {route}")
-    else:
-        logger.warning("IP set with no network route")
-
-    with open("/etc/resolv.conf", "wb") as resolvconf_fd:
-        for server in dns_servers:
-            resolvconf_fd.write(f"nameserver {server}\n".encode())
-
-
-def setup_input_data(input_data: bytes):
-    logger.debug("Extracting data")
-    if input_data:
-        # Unzip in /data
-        if not os.path.exists("/opt/input.zip"):
-            open("/opt/input.zip", "wb").write(input_data)
-            os.makedirs("/data", exist_ok=True)
-            os.system("unzip -q /opt/input.zip -d /data")
-
-
-def setup_volumes(volumes: List[Volume]):
-    for volume in volumes:
-        logger.debug(f"Mounting /dev/{volume.device} on {volume.mount}")
-        os.makedirs(volume.mount, exist_ok=True)
-        if volume.read_only:
-            system(f"mount -t squashfs -o ro /dev/{volume.device} {volume.mount}")
-        else:
-            system(f"mount -o rw /dev/{volume.device} {volume.mount}")
-
-    system("mount")
-
-
-def setup_code_asgi(
-    code: bytes, encoding: Encoding, entrypoint: str
-) -> ASGIApplication:
-    # Allow importing packages from /opt/packages
-    sys.path.append("/opt/packages")
-
-    logger.debug("Extracting code")
-    app: ASGIApplication
-    if encoding == Encoding.squashfs:
-        sys.path.append("/opt/code")
-        module_name, app_name = entrypoint.split(":", 1)
-        logger.debug("import module")
-        module = __import__(module_name)
-        for level in module_name.split(".")[1:]:
-            module = getattr(module, level)
-        app = getattr(module, app_name)
-    elif encoding == Encoding.zip:
-        # Unzip in /opt and import the entrypoint from there
-        if not os.path.exists("/opt/archive.zip"):
-            open("/opt/archive.zip", "wb").write(code)
-            logger.debug("Run unzip")
-            os.system("unzip -q /opt/archive.zip -d /opt")
-        sys.path.append("/opt")
-        module_name, app_name = entrypoint.split(":", 1)
-        logger.debug("import module")
-        module = __import__(module_name)
-        for level in module_name.split(".")[1:]:
-            module = getattr(module, level)
-        app = getattr(module, app_name)
-    elif encoding == Encoding.plain:
-        # Execute the code and extract the entrypoint
-        locals: Dict[str, Any] = {}
-        exec(code, globals(), locals)
-        app = locals[entrypoint]
-    else:
-        raise ValueError(f"Unknown encoding '{encoding}'")
-    return app
-
-
-def setup_code_executable(
-    code: Optional[bytes], encoding: Optional[Encoding], entrypoint: Optional[str]
-) -> subprocess.Popen:
-    if not code:
-        logger.debug("No code, it's an instance")
-        process = subprocess.Popen(["/bin/sleep", "infinity"])
-        return process
-    logger.debug("Extracting code")
-    if encoding == Encoding.squashfs:
-        path = f"/opt/code/{entrypoint}"
-        if not os.path.isfile(path):
-            os.system("find /opt/code/")
-            raise FileNotFoundError(f"No such file: {path}")
-        os.system(f"chmod +x {path}")
-    elif encoding == Encoding.zip:
-        open("/opt/archive.zip", "wb").write(code)
-        logger.debug("Run unzip")
-        os.makedirs("/opt/code", exist_ok=True)
-        os.system("unzip /opt/archive.zip -d /opt/code")
-        path = f"/opt/code/{entrypoint}"
-        if not os.path.isfile(path):
-            os.system("find /opt/code")
-            raise FileNotFoundError(f"No such file: {path}")
-        os.system(f"chmod +x {path}")
-    elif encoding == Encoding.plain:
-        os.makedirs("/opt/code", exist_ok=True)
-        path = f"/opt/code/executable {entrypoint}"
-        open(path, "wb").write(code)
-        os.system(f"chmod +x {path}")
-    else:
-        raise ValueError(f"Unknown encoding '{encoding}'. This should never happen.")
-
-    process = subprocess.Popen(path)
-    return process
-
-
-def setup_code(
-    code: Optional[bytes], encoding: Optional[Encoding], entrypoint: Optional[str], interface: Interface
-) -> Union[ASGIApplication, subprocess.Popen]:
-
-    if interface == Interface.asgi:
-        return setup_code_asgi(code=code, encoding=encoding, entrypoint=entrypoint)
-    elif interface == Interface.executable:
-        return setup_code_executable(
-            code=code, encoding=encoding, entrypoint=entrypoint
-        )
-    else:
-        raise ValueError("Invalid interface. This should never happen.")
-
-
-async def run_python_code_http(
-    application: ASGIApplication, scope: dict
-) -> Tuple[Dict, Dict, str, Optional[bytes]]:
-
-    logger.debug("Running code")
-    with StringIO() as buf, redirect_stdout(buf):
-        # Execute in the same process, saves ~20ms than a subprocess
-
-        # The body should not be part of the ASGI scope itself
-        body: bytes = scope.pop("body")
-
-        async def receive():
-            type_ = (
-                "http.request"
-                if scope["type"] in ("http", "websocket")
-                else "aleph.message"
-            )
-            return {"type": type_, "body": body, "more_body": False}
-
-        send_queue: asyncio.Queue = asyncio.Queue()
-
-        async def send(dico):
-            await send_queue.put(dico)
-
-        # TODO: Better error handling
-        logger.debug("Awaiting application...")
-        await application(scope, receive, send)
-
-        logger.debug("Waiting for headers")
-        headers: Dict
-        if scope["type"] == "http":
-            headers = await send_queue.get()
-        else:
-            headers = {}
-
-        logger.debug("Waiting for body")
-        body: Dict = await send_queue.get()
-
-        logger.debug("Waiting for buffer")
-        output = buf.getvalue()
-
-        logger.debug(f"Headers {headers}")
-        logger.debug(f"Body {body}")
-        logger.debug(f"Output {output}")
-
-    logger.debug("Getting output data")
-    output_data: bytes
-    if os.path.isdir("/data") and os.listdir("/data"):
-        make_archive("/opt/output", "zip", "/data")
-        with open("/opt/output.zip", "rb") as output_zipfile:
-            output_data = output_zipfile.read()
-    else:
-        output_data = b""
-
-    logger.debug("Returning result")
-    return headers, body, output, output_data
-
-
-async def make_request(session, scope):
-    async with session.request(
-        scope["method"],
-        url="http://localhost:8080{}".format(scope["path"]),
-        params=scope["query_string"],
-        headers=[(a.decode("utf-8"), b.decode("utf-8")) for a, b in scope["headers"]],
-        data=scope.get("body", None),
-    ) as resp:
-        headers = {
-            "headers": [
-                (a.encode("utf-8"), b.encode("utf-8")) for a, b in resp.headers.items()
-            ],
-            "status": resp.status,
-        }
-        body = {"body": await resp.content.read()}
-    return headers, body
-
-
-async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[bytes]]:
-    logger.debug("Calling localhost")
-
-    tries = 0
-    headers = None
-    body = None
-
-    timeout = aiohttp.ClientTimeout(total=5)
-    async with aiohttp.ClientSession(timeout=timeout) as session:
-        while not body:
-            try:
-                tries += 1
-                headers, body = await make_request(session, scope)
-            except aiohttp.ClientConnectorError:
-                if tries > 20:
-                    raise
-                await asyncio.sleep(0.05)
-
-    output = ""  # Process stdout is not captured per request
-    output_data = None
-    logger.debug("Returning result")
-    return headers, body, output, output_data
-
-
-async def process_instruction(
-    instruction: bytes,
-    interface: Interface,
-    application: Union[ASGIApplication, subprocess.Popen],
-) -> AsyncIterable[bytes]:
-
-    if instruction == b"halt":
-        logger.info("Received halt command")
-        system("sync")
-        logger.debug("Filesystems synced")
-        if isinstance(application, subprocess.Popen):
-            application.terminate()
-            logger.debug("Application terminated")
-            # application.communicate()
-        else:
-            # Close the cached session in aleph_client:
-            from aleph_client.asynchronous import get_fallback_session
-
-            session: aiohttp.ClientSession = get_fallback_session()
-            await session.close()
-            logger.debug("Aiohttp cached session closed")
-        yield b"STOP\n"
-        logger.debug("Supervisor informed of halt")
-        raise ShutdownException
-    elif instruction.startswith(b"!"):
-        # Execute shell commands in the form `!ls /`
-        msg = instruction[1:].decode()
-        try:
-            process_output = subprocess.check_output(
-                msg, stderr=subprocess.STDOUT, shell=True
-            )
-            yield process_output
-        except subprocess.CalledProcessError as error:
-            yield str(error).encode() + b"\n" + error.output
-    else:
-        # Python
-        logger.debug("msgpack.loads (")
-        msg_ = msgpack.loads(instruction, raw=False)
-        logger.debug("msgpack.loads )")
-        payload = RunCodePayload(**msg_)
-
-        output: Optional[str] = None
-        try:
-            headers: Dict
-            body: Dict
-            output_data: Optional[bytes]
-
-            if interface == Interface.asgi:
-                headers, body, output, output_data = await run_python_code_http(
-                    application=application, scope=payload.scope
-                )
-            elif interface == Interface.executable:
-                headers, body, output, output_data = await run_executable_http(
-                    scope=payload.scope
-                )
-            else:
-                raise ValueError("Unknown interface. This should never happen")
-
-            result = {
-                "headers": headers,
-                "body": body,
-                "output": output,
-                "output_data": output_data,
-            }
-            yield msgpack.dumps(result, use_bin_type=True)
-        except Exception as error:
-            yield msgpack.dumps(
-                {
-                    "error": str(error),
-                    "traceback": str(traceback.format_exc()),
-                    "output": output,
-                }
-            )
-
-
-def receive_data_length(client) -> int:
-    """Receive the length of the data to follow."""
-    buffer = b""
-    for _ in range(9):
-        byte = client.recv(1)
-        if byte == b"\n":
-            break
-        else:
-            buffer += byte
-    return int(buffer)
-
-
-def load_configuration(data: bytes) -> ConfigurationPayload:
-    msg_ = msgpack.loads(data, raw=False)
-    msg_["volumes"] = [Volume(**volume_dict) for volume_dict in msg_.get("volumes")]
-    return ConfigurationPayload(**msg_)
-
-
-def receive_config(client) -> ConfigurationPayload:
-    length = receive_data_length(client)
-    data = b""
-    while len(data) < length:
-        data += client.recv(1024 * 1024)
-    return load_configuration(data)
-
-
-def setup_system(config: ConfigurationPayload):
-    # Linux host names are limited to 63 characters. We therefore use the base32 representation
-    # of the item_hash instead of its common base16 representation.
-    item_hash_binary: bytes = base64.b16decode(config.vm_hash.encode().upper())
-    hostname = base64.b32encode(item_hash_binary).decode().strip('=').lower()
-    setup_hostname(hostname)
-
-    setup_variables(config.variables)
-    setup_volumes(config.volumes)
-    setup_network(config.ip, config.route, config.dns_servers)
-    setup_input_data(config.input_data)
-    logger.debug("Setup finished")
-
-
-def umount_volumes(volumes: List[Volume]):
-    "Umount user related filesystems"
-    system("sync")
-    for volume in volumes:
-        logger.debug(f"Umounting /dev/{volume.device} on {volume.mount}")
-        system(f"umount {volume.mount}")
-
-
-async def main():
-    client, addr = s.accept()
-
-    logger.debug("Receiving setup...")
-    config = receive_config(client)
-    setup_system(config)
-
-    try:
-        app: Union[ASGIApplication, subprocess.Popen] = setup_code(
-            config.code, config.encoding, config.entrypoint, config.interface
-        )
-        client.send(msgpack.dumps({"success": True}))
-    except Exception as error:
-        client.send(
-            msgpack.dumps(
-                {
-                    "success": False,
-                    "error": str(error),
-                    "traceback": str(traceback.format_exc()),
-                }
-            )
-        )
-        logger.exception("Program could not be started")
-        raise
-
-    class ServerReference:
-        "Reference used to close the server from within `handle_instruction"
-        server: asyncio.AbstractServer
-
-    server_reference = ServerReference()
-
-    async def handle_instruction(reader, writer):
-        data = await reader.read(1000_1000)  # Max 1 Mo
-
-        logger.debug("Init received msg")
-        if logger.level <= logging.DEBUG:
-            data_to_print = f"{data[:500]}..." if len(data) > 500 else data
-            logger.debug(f"<<<\n\n{data_to_print}\n\n>>>")
-
-        try:
-            async for result in process_instruction(
-                instruction=data, interface=config.interface, application=app
-            ):
-                writer.write(result)
-                await writer.drain()
-
-                logger.debug("Instruction processed")
-        except ShutdownException:
-            logger.info("Initiating shutdown")
-            writer.write(b"STOPZ\n")
-            await writer.drain()
-            logger.debug("Shutdown confirmed to supervisor")
-            server_reference.server.close()
-            logger.debug("Supervisor socket server closed")
-        finally:
-            writer.close()
-
-    server = await asyncio.start_server(handle_instruction, sock=s)
-    server_reference.server = server
-
-    addr = server.sockets[0].getsockname()
-    print(f"Serving on {addr}")
-
-    try:
-        async with server:
-            await server.serve_forever()
-    except asyncio.CancelledError:
-        logger.debug("Server was properly cancelled")
-    finally:
-        logger.warning("System shutdown")
-        server.close()
-        logger.debug("Server closed")
-        umount_volumes(config.volumes)
-        logger.debug("User volumes unmounted")
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.DEBUG)
-    asyncio.run(main())
-
-    logger.info("Unmounting system filesystems")
-    system("umount /dev/shm")
-    system("umount /dev/pts")
-    system("umount -a")
-
-    logger.info("Sending reboot syscall")
-    # Send reboot syscall, see man page
-    # https://man7.org/linux/man-pages/man2/reboot.2.html
-    libc = ctypes.CDLL(None)
-    libc.syscall(169, 0xFEE1DEAD, 672274793, 0x4321FEDC, None)
-    # The exit should not happen due to system halt.
-    sys.exit(0)
diff --git a/runtimes/instance-debian-rootfs/inittab b/runtimes/instance-debian-rootfs/inittab
deleted file mode 100644
index 7f79023b9..000000000
--- a/runtimes/instance-debian-rootfs/inittab
+++ /dev/null
@@ -1,22 +0,0 @@
-# /etc/inittab
-
-::sysinit:/sbin/init sysinit
-::sysinit:/sbin/init boot
-::wait:/sbin/init default
-
-# Set up a couple of getty's
-tty1::respawn:/sbin/getty 38400 tty1
-tty2::respawn:/sbin/getty 38400 tty2
-tty3::respawn:/sbin/getty 38400 tty3
-tty4::respawn:/sbin/getty 38400 tty4
-tty5::respawn:/sbin/getty 38400 tty5
-tty6::respawn:/sbin/getty 38400 tty6
-
-# Put a getty on the serial port
-ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100
-
-# Stuff to do for the 3-finger salute
-::ctrlaltdel:/sbin/reboot
-
-# Stuff to do before rebooting
-::shutdown:/sbin/init shutdown
\ No newline at end of file
diff --git a/runtimes/instance-debian-rootfs/nginx/health-check.conf b/runtimes/instance-debian-rootfs/nginx/health-check.conf
deleted file mode 100644
index d777bfe03..000000000
--- a/runtimes/instance-debian-rootfs/nginx/health-check.conf
+++ /dev/null
@@ -1,9 +0,0 @@
-server {
-    listen       8080;
-    server_name  localhost;
-
-    location / {
-        return 200 "healthy\n";
-        add_header Content-Type text/plain;
-    }
-}
\ No newline at end of file
diff --git a/runtimes/instance-debian-rootfs/nginx/index.html b/runtimes/instance-debian-rootfs/nginx/index.html
deleted file mode 100644
index 82e6ef7aa..000000000
--- a/runtimes/instance-debian-rootfs/nginx/index.html
+++ /dev/null
@@ -1,20 +0,0 @@
-
-
-  
-    
-    
-    
-    Hello World - Nginx Docker
-    
-  
-  
-    
    
-      Hello World
-    
    
-  
-
diff --git a/runtimes/instance-debian-rootfs/nginx/nginx.conf b/runtimes/instance-debian-rootfs/nginx/nginx.conf
deleted file mode 100644
index ee8c7fd6b..000000000
--- a/runtimes/instance-debian-rootfs/nginx/nginx.conf
+++ /dev/null
@@ -1,31 +0,0 @@
-user  nginx;
-worker_processes  auto;
-
-error_log  /var/log/nginx/error.log notice;
-pid        /var/run/nginx.pid;
-
-
-events {
-    worker_connections  1024;
-}
-
-
-http {
-    include       /etc/nginx/mime.types;
-    default_type  application/octet-stream;
-
-    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
-                      '$status $body_bytes_sent "$http_referer" '
-                      '"$http_user_agent" "$http_x_forwarded_for"';
-
-    access_log  /var/log/nginx/access.log  main;
-
-    sendfile        on;
-    #tcp_nopush     on;
-
-    keepalive_timeout  65;
-
-    #gzip  on;
-
-    include /etc/nginx/conf.d/*.conf;
-}
\ No newline at end of file
diff --git a/runtimes/instance-debian-rootfs/update_inits.sh b/runtimes/instance-debian-rootfs/update_inits.sh
deleted file mode 100755
index 55a1c99b1..000000000
--- a/runtimes/instance-debian-rootfs/update_inits.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-rm ./rootfs.squashfs
-
-set -euf
-
-cp ./init0.sh ./rootfs/sbin/init
-cp ./init1.py ./rootfs/root/init1.py
-chmod +x ./rootfs/sbin/init
-chmod +x ./rootfs/root/init1.py
-
-mksquashfs ./rootfs/ ./rootfs.squashfs
-
-echo "OK"
diff --git a/runtimes/instance-rootfs/create-debian-11-disk.sh b/runtimes/instance-rootfs/create-debian-11-disk.sh
new file mode 100755
index 000000000..271e101f6
--- /dev/null
+++ b/runtimes/instance-rootfs/create-debian-11-disk.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -euf
+
+# Variables
+ROOTFS_FILE="./debian-11.btrfs"
+MOUNT_ORIGIN_DIR="/mnt/debian"
+MOUNT_DIR="/mnt/vm"
+IMAGE_URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-genericcloud-amd64.tar.xz"
+IMAGE_NAME="debian-11-genericcloud.tar.xz"
+IMAGE_RAW_NAME="disk.raw"
+
+# Cleanup previous run
+umount "$MOUNT_ORIGIN_DIR" || true
+umount "$MOUNT_DIR" || true
+rm -f "$ROOTFS_FILE"
+
+# Prepare directories
+mkdir -p "$MOUNT_ORIGIN_DIR"
+mkdir -p "$MOUNT_DIR"
+
+# Download Debian image
+echo "Downloading Debian 11 image"
+curl -L "$IMAGE_URL" -o "$IMAGE_NAME"
+
+# Allocate 1GB rootfs.btrfs file
+echo "Allocate 1GB $ROOTFS_FILE file"
+fallocate -l 1G "$ROOTFS_FILE"
+mkfs.btrfs "$ROOTFS_FILE"
+mount "$ROOTFS_FILE" "$MOUNT_DIR"
+
+# Extract Debian image
+echo "Extracting Debian 11 image"
+tar xvf "$IMAGE_NAME"
+
+# Mount first partition of Debian Image
+LOOPDISK=$(losetup --find --show $IMAGE_RAW_NAME)
+partx -u $LOOPDISK
+mount "$LOOPDISK"p1 "$MOUNT_ORIGIN_DIR"
+
+# Copy Debian image to rootfs
+echo "Copying Debian 11 image to $ROOTFS_FILE file"
+cp -vap "$MOUNT_ORIGIN_DIR/." "$MOUNT_DIR"
+
+# Cleanup and unmount
+umount "$MOUNT_ORIGIN_DIR"
+partx -d "$LOOPDISK"
+losetup -d "$LOOPDISK"
+umount "$MOUNT_DIR"
+rm "$IMAGE_RAW_NAME"
+rm "$IMAGE_NAME"
diff --git a/runtimes/instance-rootfs/create-debian-12-disk.sh b/runtimes/instance-rootfs/create-debian-12-disk.sh
new file mode 100755
index 000000000..e236c78af
--- /dev/null
+++ b/runtimes/instance-rootfs/create-debian-12-disk.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -euf
+
+# Variables
+ROOTFS_FILE="./debian-12.btrfs"
+MOUNT_ORIGIN_DIR="/mnt/debian"
+MOUNT_DIR="/mnt/vm"
+IMAGE_URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.tar.xz"
+IMAGE_NAME="debian-12-genericcloud.tar.xz"
+IMAGE_RAW_NAME="disk.raw"
+
+# Cleanup previous run
+umount "$MOUNT_ORIGIN_DIR" || true
+umount "$MOUNT_DIR" || true
+rm -f "$ROOTFS_FILE"
+
+# Prepare directories
+mkdir -p "$MOUNT_ORIGIN_DIR"
+mkdir -p "$MOUNT_DIR"
+
+# Download Debian image
+echo "Downloading Debian 12 image"
+curl -L "$IMAGE_URL" -o "$IMAGE_NAME"
+
+# Allocate 1GB rootfs.btrfs file
+echo "Allocate 1GB $ROOTFS_FILE file"
+fallocate -l 1G "$ROOTFS_FILE"
+mkfs.btrfs "$ROOTFS_FILE"
+mount "$ROOTFS_FILE" "$MOUNT_DIR"
+
+# Extract Debian image
+echo "Extracting Debian 12 image"
+tar xvf "$IMAGE_NAME"
+
+# Mount first partition of Debian Image
+LOOPDISK=$(losetup --find --show $IMAGE_RAW_NAME)
+partx -u $LOOPDISK
+mount "$LOOPDISK"p1 "$MOUNT_ORIGIN_DIR"
+
+# Copy Debian image to rootfs
+echo "Copying Debian 12 image to $ROOTFS_FILE file"
+cp -vap "$MOUNT_ORIGIN_DIR/." "$MOUNT_DIR"
+
+# Cleanup and unmount
+umount "$MOUNT_ORIGIN_DIR"
+partx -d "$LOOPDISK"
+losetup -d "$LOOPDISK"
+umount "$MOUNT_DIR"
+rm "$IMAGE_RAW_NAME"
+rm "$IMAGE_NAME"
diff --git a/runtimes/instance-rootfs/create-ubuntu-22-04-disk.sh b/runtimes/instance-rootfs/create-ubuntu-22-04-disk.sh
new file mode 100755
index 000000000..e6a4589e9
--- /dev/null
+++ b/runtimes/instance-rootfs/create-ubuntu-22-04-disk.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -euf
+
+# Variables
+ROOTFS_FILE="./ubuntu-22-04.btrfs"
+ROOTFS_DIR="./rootfs"
+MOUNT_DIR="/mnt/vm"
+IMAGE_URL="https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64-root.tar.xz"
+IMAGE_NAME="jammy-server-cloudimg-root.tar.xz"
+
+# Cleanup previous run
+umount "$MOUNT_DIR" || true
+rm -f "$ROOTFS_FILE"
+rm -rf "$ROOTFS_DIR"
+
+# Prepare directories
+mkdir -p "$MOUNT_DIR"
+mkdir -p "$ROOTFS_DIR"
+
+# Download Ubuntu image
+echo "Downloading Ubuntu 22.04 image"
+curl -L "$IMAGE_URL" -o "$IMAGE_NAME"
+
+# Allocate 1,4 GB rootfs.btrfs file
+echo "Allocate 1,4 GB $ROOTFS_FILE file"
+fallocate -l 1400M "$ROOTFS_FILE"
+mkfs.btrfs "$ROOTFS_FILE"
+mount "$ROOTFS_FILE" "$MOUNT_DIR"
+
+# Extract Ubuntu image to rootfs
+echo "Extracting Ubuntu 22.04 image"
+tar xvf "$IMAGE_NAME" -C "$MOUNT_DIR"
+
+# Cleanup and unmount
+umount "$MOUNT_DIR"
+rm -rf "$ROOTFS_DIR"
+rm "$IMAGE_NAME"

From d4c2beb3774731f57f3f60a67fae2e186d333a5f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 14 Jul 2023 14:55:04 +0200
Subject: [PATCH 449/990] Fix: Use SDK version 0.7.0 (#371)

---
 runtimes/aleph-debian-11-python/create_disk_image.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
index a6e1e85e5..65f33d7c0 100755
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-11-python/create_disk_image.sh
@@ -34,7 +34,7 @@ apt-get install -y --no-install-recommends --no-install-suggests \
 pip3 install 'fastapi~=0.95.1'
 
 echo "Pip installing aleph-client"
-pip3 install 'aleph-sdk-python==0.7.0b1'
+pip3 install 'aleph-sdk-python==0.7.0'
 
 # Compile all Python bytecode
 python3 -m compileall -f /usr/local/lib/python3.9

From 84ed7d61f401cbe2ebe84b8cee883cd9747667ea Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Fri, 14 Jul 2023 15:43:18 +0200
Subject: [PATCH 450/990] Support BTRFS filesystem for instances (#373)

Added support for BTRFS to execute a VM instance.
---
 kernels/microvm-kernel-x86_64-5.10.config |  2 +-
 vm_supervisor/storage.py                  | 14 +++++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/kernels/microvm-kernel-x86_64-5.10.config b/kernels/microvm-kernel-x86_64-5.10.config
index 1f4df1c62..14606258f 100644
--- a/kernels/microvm-kernel-x86_64-5.10.config
+++ b/kernels/microvm-kernel-x86_64-5.10.config
@@ -2113,7 +2113,7 @@ CONFIG_FS_MBCACHE=y
 # CONFIG_JFS_FS is not set
 # CONFIG_XFS_FS is not set
 # CONFIG_GFS2_FS is not set
-# CONFIG_BTRFS_FS is not set
+CONFIG_BTRFS_FS=y
 # CONFIG_NILFS2_FS is not set
 # CONFIG_F2FS_FS is not set
 # CONFIG_FS_DAX is not set
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 439359372..5f1a45708 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -201,7 +201,8 @@ async def create_volume_file(
     volume: Union[PersistentVolume, RootfsVolume], namespace: str
 ) -> Path:
     volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
-    path = Path(settings.PERSISTENT_VOLUMES_DIR) / namespace / f"{volume_name}.ext4"
+    # Assume that the main filesystem format is BTRFS
+    path = settings.PERSISTENT_VOLUMES_DIR / namespace / f"{volume_name}.btrfs"
     if not path.is_file():
         logger.debug(f"Creating {volume.size_mib}MB volume")
         # Ensure that the parent directory exists
@@ -236,9 +237,10 @@ async def create_mapped_device(device_name: str, table_command: str) -> None:
     await run_in_subprocess(command, stdin_input=table_command.encode())
 
 
-async def e2fs_check_and_resize(device_path: Path) -> None:
-    await run_in_subprocess(["e2fsck", "-fy", str(device_path)])
-    await run_in_subprocess(["resize2fs", str(device_path)])
+async def resize_file_system(device_path: Path, mount_path: Path) -> None:
+    await run_in_subprocess(["mount", str(device_path), str(mount_path)])
+    await run_in_subprocess(["btrfs", "filesystem", "resize", "max", str(mount_path)])
+    await run_in_subprocess(["umount", str(mount_path)])
 
 
 async def create_devmapper(
@@ -271,7 +273,9 @@ async def create_devmapper(
     snapshot_table_command = f"0 {extended_block_size} snapshot {path_base_device_name} {extended_loop_device} P 8"
     await create_mapped_device(mapped_volume_name, snapshot_table_command)
 
-    await e2fs_check_and_resize(path_mapped_volume_name)
+    mount_path = Path(f"/mnt/{mapped_volume_name}")
+    mount_path.mkdir(parents=True, exist_ok=True)
+    await resize_file_system(path_mapped_volume_name, mount_path)
     await chown_to_jailman(path_base_device_name)
     await chown_to_jailman(path_mapped_volume_name)
     return path_mapped_volume_name

From 0c85e9af69dffe26244d0cd449ce2abece108ce9 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Fri, 14 Jul 2023 21:34:40 +0200
Subject: [PATCH 451/990] Implement VM snapshots (#379)

* Problem: Instances loose data between nodes, so we need a way to handle and share this data.

Feature: Implement VM snapshots that will be managed independently.

* Fix: Added some refactor for snapshots and handled some errors.

* Fix: Split snapshot class in two.

* Fix: Make it work well with threads and asyncio.

* Fix: Added code improvements with threading and fixed PR comments.

* Fix: Fixed check_disk method error.

* Fix: Fixed more snapshot errors with multithreading

* Fix: CI for Ubuntu was failing too often because of apt/dpkg locks (#380)

Problem: the CI is looking for only one lock file, but apt/dpkg use several. Furthermore, `lslocks --json` seems to cut its output to a specific width, and configuration options (`--notruncate`) do not appear to have an impact.

Solution: repeat calls to apt-get update until we get the lock. This is not perfect but increases the rate of success.

* Create Ubuntu rootfs (#370)

Updated creation script to create a BTRFS image of Ubuntu filesystem.

* Fix: Use SDK version 0.7.0 (#371)

* Support BTRFS filesystem for instances (#373)

Added support for BTRFS to execute a VM instance.

* Feature: Delete old snapshots after do a new one.

* add dependency to python3-schedule

* Fix latest_snapshot attribute error

* default snapshot frequency 1 hour

---------

Co-authored-by: Olivier Desenfans 
Co-authored-by: Hugo Herter 
Co-authored-by: Andres D. Molins 
---
 packaging/aleph-vm/DEBIAN/control          |   2 +-
 vm_supervisor/conf.py                      |  14 +++
 vm_supervisor/pool.py                      |  11 ++
 vm_supervisor/snapshot_manager.py          | 130 +++++++++++++++++++++
 vm_supervisor/snapshots.py                 |  60 ++++++++++
 vm_supervisor/storage.py                   |  40 ++++++-
 vm_supervisor/vm/firecracker/executable.py |   4 +
 vm_supervisor/vm/firecracker/instance.py   |  35 +++++-
 8 files changed, 292 insertions(+), 4 deletions(-)
 create mode 100644 vm_supervisor/snapshot_manager.py
 create mode 100644 vm_supervisor/snapshots.py

diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index c47f0fee5..e9947992c 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule
 Section: aleph-im
 Priority: Extra
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 2fc00b091..3604107ff 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -28,6 +28,10 @@ class IPv6AllocationPolicy(str, Enum):
     dynamic = "dynamic"  # Assign an available IP address.
 
 
+class SnapshotCompressionAlgorithm(str, Enum):
+    gz = "gzip"
+
+
 def etc_resolv_conf_dns_servers():
     with open("/etc/resolv.conf", "r") as resolv_file:
         for line in resolv_file.readlines():
@@ -146,6 +150,16 @@ class Settings(BaseSettings):
     MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000  # 10 MB
     MAX_DATA_ARCHIVE_SIZE = 10_000_000  # 10 MB
 
+    SNAPSHOT_FREQUENCY: int = Field(
+        default=60,
+        description="Snapshot frequency interval in minutes. It will create a VM snapshot every X minutes.",
+    )
+
+    SNAPSHOT_COMPRESSION_ALGORITHM: SnapshotCompressionAlgorithm = Field(
+        default=SnapshotCompressionAlgorithm.gz,
+        description="Snapshot compression algorithm.",
+    )
+
     # hashlib.sha256(b"secret-token").hexdigest()
     ALLOCATION_TOKEN_HASH = (
         "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda"
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 5389ee8f1..20bee71b0 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -8,6 +8,7 @@
 
 from .conf import settings
 from .models import ExecutableContent, VmExecution
+from .snapshot_manager import SnapshotManager
 from .vm.vm_type import VmType
 
 logger = logging.getLogger(__name__)
@@ -26,6 +27,7 @@ class VmPool:
     executions: Dict[ItemHash, VmExecution]
     message_cache: Dict[str, ExecutableMessage] = {}
     network: Optional[Network]
+    snapshot_manager: SnapshotManager
 
     def __init__(self):
         self.counter = settings.START_ID_INDEX
@@ -47,6 +49,9 @@ def __init__(self):
             if settings.ALLOW_VM_NETWORKING
             else None
         )
+        self.snapshot_manager = SnapshotManager()
+        logger.debug("Initializing SnapshotManager ...")
+        self.snapshot_manager.run_snapshots()
 
     async def create_a_vm(
         self, vm_hash: ItemHash, message: ExecutableContent, original: ExecutableContent
@@ -64,6 +69,10 @@ async def create_a_vm(
             tap_interface = None
 
         await execution.create(vm_id=vm_id, tap_interface=tap_interface)
+
+        # Start VM snapshots automatically
+        await self.snapshot_manager.start_for(execution=execution)
+
         return execution
 
     def get_unique_vm_id(self) -> int:
@@ -125,6 +134,8 @@ async def stop(self):
             *(execution.stop() for vm_hash, execution in self.executions.items())
         )
 
+        await self.snapshot_manager.stop_all()
+
     def get_persistent_executions(self) -> Iterable[VmExecution]:
         for vm_hash, execution in self.executions.items():
             if execution.persistent and execution.is_running:
diff --git a/vm_supervisor/snapshot_manager.py b/vm_supervisor/snapshot_manager.py
new file mode 100644
index 000000000..2e0a67668
--- /dev/null
+++ b/vm_supervisor/snapshot_manager.py
@@ -0,0 +1,130 @@
+import asyncio
+import logging
+import threading
+from time import sleep
+from typing import Dict, Optional
+
+from aleph_message.models import ItemHash
+from schedule import Job, Scheduler
+
+from .conf import settings
+from .models import VmExecution
+from .snapshots import CompressedDiskVolumeSnapshot
+
+logger = logging.getLogger(__name__)
+
+
+def wrap_async_snapshot(execution):
+    asyncio.run(do_execution_snapshot(execution))
+
+
+def run_threaded_snapshot(execution):
+    job_thread = threading.Thread(target=wrap_async_snapshot, args=(execution,))
+    job_thread.start()
+
+
+async def do_execution_snapshot(execution: VmExecution) -> CompressedDiskVolumeSnapshot:
+    try:
+        logger.debug(f"Starting new snapshot for VM {execution.vm_hash}")
+        assert execution.vm, "VM execution not set"
+
+        snapshot = await execution.vm.create_snapshot()
+        await snapshot.upload()
+
+        logger.debug(
+            f"New snapshots for VM {execution.vm_hash} created in {snapshot.path}"
+        )
+        return snapshot
+    except ValueError:
+        raise ValueError("Something failed taking an snapshot")
+
+
+def infinite_run_scheduler_jobs(scheduler: Scheduler) -> None:
+    while True:
+        scheduler.run_pending()
+        sleep(1)
+
+
+class SnapshotExecution:
+    vm_hash: ItemHash
+    execution: VmExecution
+    frequency: int
+    _scheduler: Scheduler
+    _job: Job
+
+    def __init__(
+        self,
+        scheduler: Scheduler,
+        vm_hash: ItemHash,
+        execution: VmExecution,
+        frequency: int,
+    ):
+        self.vm_hash = vm_hash
+        self.execution = execution
+        self.frequency = frequency
+        self._scheduler = scheduler
+
+    async def start(self) -> None:
+        logger.debug(
+            f"Starting snapshots for VM {self.vm_hash} every {self.frequency} minutes"
+        )
+        job = self._scheduler.every(self.frequency).minutes.do(
+            run_threaded_snapshot, self.execution
+        )
+        self._job = job
+
+    async def stop(self) -> None:
+        logger.debug(f"Stopping snapshots for VM {self.vm_hash}")
+        self._scheduler.cancel_job(self._job)
+
+
+class SnapshotManager:
+    """
+    Manage VM snapshots.
+    """
+
+    executions: Dict[ItemHash, SnapshotExecution]
+    _scheduler: Scheduler
+
+    def __init__(self):
+        self.executions = {}
+        self._scheduler = Scheduler()
+
+    def run_snapshots(self) -> None:
+        job_thread = threading.Thread(
+            target=infinite_run_scheduler_jobs,
+            args=[self._scheduler],
+            daemon=True,
+            name="SnapshotManager",
+        )
+        job_thread.start()
+
+    async def start_for(
+        self, execution: VmExecution, frequency: Optional[int] = None
+    ) -> None:
+        if not execution.is_instance:
+            raise TypeError("VM execution should be an Instance only")
+
+        if not frequency:
+            frequency = settings.SNAPSHOT_FREQUENCY
+
+        vm_hash = execution.vm_hash
+        snapshot_execution = SnapshotExecution(
+            scheduler=self._scheduler,
+            vm_hash=vm_hash,
+            execution=execution,
+            frequency=frequency,
+        )
+        self.executions[vm_hash] = snapshot_execution
+        await snapshot_execution.start()
+
+    async def stop_for(self, vm_hash: ItemHash) -> None:
+        if not self.executions[vm_hash]:
+            raise ValueError(f"Snapshot execution not running for VM {vm_hash}")
+
+        await self.executions[vm_hash].stop()
+
+    async def stop_all(self) -> None:
+        await asyncio.gather(
+            *(self.stop_for(vm_hash) for vm_hash, execution in self.executions)
+        )
diff --git a/vm_supervisor/snapshots.py b/vm_supervisor/snapshots.py
new file mode 100644
index 000000000..43d7f3b1a
--- /dev/null
+++ b/vm_supervisor/snapshots.py
@@ -0,0 +1,60 @@
+import logging
+from pathlib import Path
+from typing import Optional
+
+from aleph_message.models import ItemHash
+
+from .conf import SnapshotCompressionAlgorithm
+from .storage import compress_volume_snapshot, create_volume_snapshot
+
+logger = logging.getLogger(__name__)
+
+
+class DiskVolumeFile:
+    path: Path
+    size: int
+
+    def __init__(self, path: Path):
+        self.path = path
+        self.size = path.stat().st_size
+
+
+class CompressedDiskVolumeSnapshot(DiskVolumeFile):
+    algorithm: SnapshotCompressionAlgorithm
+
+    def __init__(self, path: Path, algorithm: SnapshotCompressionAlgorithm):
+        super().__init__(path=path)
+        self.algorithm = algorithm
+
+    def delete(self) -> None:
+        self.path.unlink(missing_ok=True)
+
+    async def upload(self) -> ItemHash:
+        # TODO: Upload snapshots to Aleph Network
+        pass
+
+
+class DiskVolumeSnapshot(DiskVolumeFile):
+    compressed: Optional[CompressedDiskVolumeSnapshot]
+
+    def delete(self) -> None:
+        if self.compressed:
+            self.compressed.delete()
+
+        self.path.unlink(missing_ok=True)
+
+    async def compress(
+        self, algorithm: SnapshotCompressionAlgorithm
+    ) -> CompressedDiskVolumeSnapshot:
+        compressed_snapshot = await compress_volume_snapshot(self.path, algorithm)
+        compressed = CompressedDiskVolumeSnapshot(
+            path=compressed_snapshot, algorithm=algorithm
+        )
+        self.compressed = compressed
+        return compressed
+
+
+class DiskVolume(DiskVolumeFile):
+    async def take_snapshot(self) -> DiskVolumeSnapshot:
+        snapshot = await create_volume_snapshot(self.path)
+        return DiskVolumeSnapshot(snapshot)
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 5f1a45708..a778b794e 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -8,8 +8,10 @@
 import logging
 import re
 import sys
+from datetime import datetime
+from enum import Enum
 from pathlib import Path
-from shutil import make_archive
+from shutil import copy2, disk_usage, make_archive
 from typing import Union
 
 import aiohttp
@@ -28,7 +30,7 @@
     VolumePersistence,
 )
 
-from .conf import settings
+from .conf import SnapshotCompressionAlgorithm, settings
 from .utils import fix_message_validation, run_in_subprocess
 
 logger = logging.getLogger(__name__)
@@ -36,6 +38,10 @@
 DEVICE_MAPPER_DIRECTORY = "/dev/mapper"
 
 
+class NotEnoughDiskSpace(Exception):
+    pass
+
+
 async def chown_to_jailman(path: Path) -> None:
     """Changes ownership of the target when running firecracker inside jailer isolation."""
     if not path.exists():
@@ -315,3 +321,33 @@ async def get_volume_path(volume: MachineVolume, namespace: str) -> Path:
             return volume_path
     else:
         raise NotImplementedError("Only immutable volumes are supported")
+
+
+async def create_volume_snapshot(path: Path) -> Path:
+    new_path = Path(f"{path}.{datetime.today().strftime('%d%m%Y-%H%M%S')}.bak")
+    copy2(path, new_path)
+    return new_path
+
+
+async def compress_volume_snapshot(
+    path: Path,
+    algorithm: SnapshotCompressionAlgorithm = SnapshotCompressionAlgorithm.gz,
+) -> Path:
+    if algorithm != SnapshotCompressionAlgorithm.gz:
+        raise NotImplementedError
+
+    new_path = Path(f"{path}.gz")
+
+    await run_in_subprocess(
+        [
+            "gzip",
+            str(path),
+        ]
+    )
+
+    return new_path
+
+
+def check_disk_space(bytes_to_use: int) -> bool:
+    host_disk_usage = disk_usage("/")
+    return host_disk_usage.free >= bytes_to_use
diff --git a/vm_supervisor/vm/firecracker/executable.py b/vm_supervisor/vm/firecracker/executable.py
index 4e8bae86e..b179624f2 100644
--- a/vm_supervisor/vm/firecracker/executable.py
+++ b/vm_supervisor/vm/firecracker/executable.py
@@ -28,6 +28,7 @@
 from vm_supervisor.models import ExecutableContent
 from vm_supervisor.network.firewall import teardown_nftables_for_vm
 from vm_supervisor.network.interfaces import TapInterface
+from vm_supervisor.snapshots import CompressedDiskVolumeSnapshot
 from vm_supervisor.storage import get_volume_path
 
 logger = logging.getLogger(__name__)
@@ -287,3 +288,6 @@ async def teardown(self):
             teardown_nftables_for_vm(self.vm_id)
             await self.tap_interface.delete()
         await self.stop_guest_api()
+
+    async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
+        raise NotImplementedError()
diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index 7d886753a..0835f71f2 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -19,7 +19,17 @@
 from firecracker.microvm import setfacl
 from vm_supervisor.conf import settings
 from vm_supervisor.network.interfaces import TapInterface
-from vm_supervisor.storage import create_devmapper
+from vm_supervisor.snapshots import (
+    CompressedDiskVolumeSnapshot,
+    DiskVolume,
+    DiskVolumeSnapshot,
+)
+from vm_supervisor.storage import (
+    NotEnoughDiskSpace,
+    check_disk_space,
+    create_devmapper,
+    create_volume_file,
+)
 from vm_supervisor.utils import HostNotFoundError, ping
 
 from ...utils import run_in_subprocess
@@ -52,6 +62,7 @@ async def download_all(self):
 class AlephFirecrackerInstance(AlephFirecrackerExecutable):
     vm_configuration: BaseConfiguration
     resources: AlephInstanceResources
+    latest_snapshot: Optional[DiskVolumeSnapshot]
     is_instance = True
 
     def __init__(
@@ -64,6 +75,7 @@ def __init__(
         hardware_resources: MachineResources = MachineResources(),
         tap_interface: Optional[TapInterface] = None,
     ):
+        self.latest_snapshot = None
         super().__init__(
             vm_id,
             vm_hash,
@@ -146,6 +158,27 @@ async def configure(self):
         # Configuration of instances is sent during `self.setup()` by passing it via a volume.
         pass
 
+    async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
+        """Create a VM snapshot"""
+        volume_path = await create_volume_file(
+            self.resources.message_content.rootfs, self.resources.namespace
+        )
+        volume = DiskVolume(path=volume_path)
+
+        if not check_disk_space(volume.size):
+            raise NotEnoughDiskSpace
+
+        snapshot = await volume.take_snapshot()
+        compressed_snapshot = await snapshot.compress(
+            settings.SNAPSHOT_COMPRESSION_ALGORITHM
+        )
+
+        if self.latest_snapshot:
+            self.latest_snapshot.delete()
+
+        self.latest_snapshot = snapshot
+        return compressed_snapshot
+
     def _encode_user_data(self) -> bytes:
         """Creates user data configuration file for cloud-init tool"""
 

From 5aaa8b44d16c7d6f6c5cb82e49df649afde5dc16 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Sun, 16 Jul 2023 00:20:05 +0200
Subject: [PATCH 452/990] Fix: invalid condition for apt locks on Ubuntu CI
 (#386)

---
 .github/workflows/test-on-droplet-ubuntu-22.04.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index 35e80fe81..b1e9b6f28 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -57,9 +57,9 @@ jobs:
           
           # Ubuntu droplets run upgrades at boot, which locks apt-get
           sleep 30
-          until ! ssh root@${DROPLET_IPV4} "apt-get update" > /dev/null; do sleep 1; echo "Waiting for apt/dpkg lock..."; done          
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
+          until ssh root@${DROPLET_IPV4} "apt-get update" > /dev/null; do sleep 1; echo "Waiting for apt/dpkg lock..."; done          
+          until ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y" > /dev/null; do sleep 1; echo "Waiting for apt/dpkg lock..."; done
+          until ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles" > /dev/null; do sleep 1; echo "Waiting for apt/dpkg lock..."; done
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
           
           scp packaging/target/aleph-vm.ubuntu-22.04.deb root@${DROPLET_IPV4}:/opt

From 905f942d4487f3bacc82996ad2ea116b0a513238 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Sun, 16 Jul 2023 16:46:37 +0200
Subject: [PATCH 453/990] Feature: instance operator API (#382)

Problem: instance owners wish to perform operations on their VMs like rebooting, streaming logs, etc.
Solution: provide a set of authenticated endpoints.
---
 firecracker/microvm.py          |  13 +-
 packaging/Makefile              |   2 +-
 tests/supervisor/test_jwk.py    |  66 +++++++++
 vm_supervisor/supervisor.py     |   5 +
 vm_supervisor/views/operator.py | 237 ++++++++++++++++++++++++++++++++
 5 files changed, 318 insertions(+), 5 deletions(-)
 create mode 100644 tests/supervisor/test_jwk.py
 create mode 100644 vm_supervisor/views/operator.py

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 09d518947..9c3a1e6e2 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -77,11 +77,11 @@ class MicroVM:
     proc: Optional[asyncio.subprocess.Process] = None
     stdout_task: Optional[Task] = None
     stderr_task: Optional[Task] = None
+    log_queues: List
     config_file_path: Optional[Path] = None
     drives: List[Drive]
     init_timeout: float
     mounted_rootfs: Optional[Path] = None
-
     _unix_socket: Server
 
     @property
@@ -122,6 +122,7 @@ def __init__(
         self.drives = []
         self.init_timeout = init_timeout
         self.runtime_config = None
+        self.log_queues: List[asyncio.Queue] = []
 
     def to_dict(self):
         return {
@@ -343,6 +344,8 @@ async def print_logs(self):
             await asyncio.sleep(0.01)  # Todo: Use signal here
         while True:
             stdout = await self.proc.stdout.readline()
+            for queue in self.log_queues:
+                await queue.put(('stdout', stdout))
             if stdout:
                 print(stdout.decode().strip())
             else:
@@ -352,9 +355,11 @@ async def print_logs_stderr(self):
         while not self.proc:
             await asyncio.sleep(0.01)  # Todo: Use signal here
         while True:
-            stdout = await self.proc.stderr.readline()
-            if stdout:
-                print(stdout.decode().strip())
+            stderr = await self.proc.stderr.readline()
+            for queue in self.log_queues:
+                await queue.put(('stderr', stderr))
+            if stderr:
+                print(stderr.decode().strip())
             else:
                 await asyncio.sleep(0.001)
 
diff --git a/packaging/Makefile b/packaging/Makefile
index fae419292..d5140c8ad 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -17,7 +17,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.0'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.0' 'jwskate==0.8.0' 'eth-account==0.9.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux
diff --git a/tests/supervisor/test_jwk.py b/tests/supervisor/test_jwk.py
new file mode 100644
index 000000000..e1db545bb
--- /dev/null
+++ b/tests/supervisor/test_jwk.py
@@ -0,0 +1,66 @@
+import os
+
+from aiohttp import web
+
+# Avoid failures linked to settings when initializing the global VmPool object
+os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
+
+from typing import Any, Dict
+
+import pytest
+
+from vm_supervisor.views.operator import authenticate_jwk
+
+
+@pytest.fixture
+def valid_jwk_headers(mocker):
+    mocker.patch(
+        "vm_supervisor.views.operator.is_token_still_valid", lambda timestamp: True
+    )
+    return {
+        "X-SignedPubKey": '{"payload":"7b227075626b6579223a7b22616c67223a224553323536222c22637276223a22502d323536222c22657874223a747275652c226b65795f6f7073223a5b22766572696679225d2c226b7479223a224543222c2278223a224b65763844614d7356454673365a6b4679525a4272796344564138566a334f656e49756f34743561374634222c2279223a2279597343556d715978654767673643743736794f47525873545867446444795234644f5639514c6f6b6477227d2c22616c67223a224543445341222c22646f6d61696e223a226c6f63616c686f7374222c2261646472657373223a22307833343932346566393435623933316431653932393337353535366636396365326537666535646363222c2265787069726573223a313638393337353132342e3532317d","signature":"0x58e1498a6c4f88ac1982e7147ff49405ffe1b9633e048bb74cf741abb05ce0b63bb406f3079f641ae89f597654ecd2a704d37ffbf86a28e462140033cc0eedcb1c"}',
+        "X-SignedOperation": '{"time":"2023-07-14T22:14:14.132Z","signature":"96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0add"}',
+    }
+
+
+@pytest.mark.asyncio
+async def test_valid_signature(valid_jwk_headers: Dict[str, Any], mocker):
+    request = mocker.AsyncMock()
+    request.headers = valid_jwk_headers
+    await authenticate_jwk(request)
+
+
+@pytest.mark.asyncio
+async def test_invalid_signature(valid_jwk_headers: Dict[str, Any], mocker):
+    valid_jwk_headers[
+        "X-SignedOperation"
+    ] = '{"time":"2023-07-14T22:14:14.132Z","signature":"96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0ade"}'
+    request = mocker.AsyncMock()
+    request.headers = valid_jwk_headers
+
+    with pytest.raises(web.HTTPUnauthorized):
+        await authenticate_jwk(request)
+
+
+@pytest.mark.asyncio
+async def test_expired_token(valid_jwk_headers: Dict[str, Any], mocker):
+    mocker.patch(
+        "vm_supervisor.views.operator.is_token_still_valid", lambda timestamp: False
+    )
+    request = mocker.AsyncMock()
+    request.headers = valid_jwk_headers
+
+    with pytest.raises(web.HTTPUnauthorized):
+        await authenticate_jwk(request)
+
+
+@pytest.mark.parametrize("missing_header", ["X-SignedPubKey", "X-SignedOperation"])
+@pytest.mark.asyncio
+async def test_missing_headers(valid_jwk_headers: Dict[str, Any], mocker, missing_header: str):
+    del valid_jwk_headers[missing_header]
+
+    request = mocker.AsyncMock()
+    request.headers = valid_jwk_headers
+
+    with pytest.raises(web.HTTPBadRequest):
+        await authenticate_jwk(request)
diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 3d9095bdd..9a95dec9d 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -28,6 +28,7 @@
     status_check_version,
     update_allocations,
 )
+from .views.operator import operate_erase, operate_expire, operate_stop, stream_logs
 
 logger = logging.getLogger(__name__)
 
@@ -55,6 +56,10 @@ async def server_version_middleware(
         web.get("/about/usage/system", about_system_usage),
         web.get("/about/config", about_config),
         web.post("/control/allocations", update_allocations),
+        web.get("/control/machine/{ref}/logs", stream_logs),
+        web.post("/control/machine/{ref}/expire", operate_expire),
+        web.post("/control/machine/{ref}/stop", operate_stop),
+        web.post("/control/machine/{ref}/erase", operate_erase),
         web.get("/status/check/fastapi", status_check_fastapi),
         web.get("/status/check/version", status_check_version),
         web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
diff --git a/vm_supervisor/views/operator.py b/vm_supervisor/views/operator.py
new file mode 100644
index 000000000..0ea1a2a97
--- /dev/null
+++ b/vm_supervisor/views/operator.py
@@ -0,0 +1,237 @@
+import asyncio
+import json
+import logging
+from datetime import datetime, timedelta
+from typing import Awaitable, Callable
+
+import aiohttp.web_exceptions
+from aiohttp import web
+from aiohttp.web_urldispatcher import UrlMappingMatchInfo
+from aleph_message.exceptions import UnknownHashError
+from aleph_message.models import ItemHash
+from eth_account import Account
+from eth_account.messages import encode_defunct
+from jwskate import Jwk
+
+from ..models import VmExecution
+from ..run import pool
+
+logger = logging.getLogger(__name__)
+
+
+import functools
+
+
+def is_token_still_valid(timestamp):
+    """
+    Checks if a token has exprired based on its timestamp
+    """
+    timestamp = int(timestamp)
+    current_datetime = datetime.now()
+    target_datetime = datetime.fromtimestamp(timestamp)
+
+    return target_datetime > current_datetime
+
+
+def verify_wallet_signature(signature, message, address):
+    """
+    Verifies a signature issued by a wallet
+    """
+    enc_msg = encode_defunct(hexstr=message)
+    computed_address = Account.recover_message(enc_msg, signature=signature)
+
+    return computed_address.lower() == address.lower()
+
+
+def get_json_from_hex(str: str):
+    """
+    Converts a hex string to a json object
+    """
+    return json.loads(bytes.fromhex(str).decode("utf-8"))
+
+
+async def authenticate_jwk(request: web.Request):
+    signed_keypair = request.headers.get("X-SignedPubKey")
+    if not signed_keypair:
+        raise web.HTTPBadRequest(reason="Missing X-SignedPubKey header")
+
+    try:
+        keypair_dict = json.loads(signed_keypair)
+        payload = keypair_dict.get("payload")
+        signature = keypair_dict.get("signature")
+    except (json.JSONDecodeError, KeyError):
+        raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey format")
+
+    try:
+        json_payload = get_json_from_hex(payload)
+    except json.JSONDecodeError:
+        raise web.HTTPBadRequest(reason="")
+
+    if not verify_wallet_signature(signature, payload, json_payload.get("address")):
+        raise web.HTTPUnauthorized(reason="Invalid signature")
+
+    expires = json_payload.get("expires")
+    if not expires or not is_token_still_valid(expires):
+        raise web.HTTPUnauthorized(reason="Token expired")
+
+    signed_operation = request.headers.get("X-SignedOperation")
+    if not signed_operation:
+        raise web.HTTPBadRequest(reason="Missing X-SignedOperation header")
+
+    json_web_key = Jwk(json_payload.get("pubkey"))
+    try:
+        payload = json.loads(signed_operation)
+    except json.JSONDecodeError:
+        raise web.HTTPBadRequest(reason="Could not decode X-SignedOperation")
+
+    # The signature is not part of the signed payload, remove it
+    payload_signature = payload.pop("signature")
+    signed_payload = json.dumps(payload, separators=(",", ":")).encode("utf-8")
+
+    if json_web_key.verify(
+        data=signed_payload,
+        signature=bytes.fromhex(payload_signature),
+        alg="ES256",
+    ):
+        logger.debug("Signature verified")
+    else:
+        raise web.HTTPUnauthorized(reason="Signature could not verified")
+
+
+def require_jwk_authentication(
+    handler: Callable[[web.Request], Awaitable[web.StreamResponse]]
+):
+    @functools.wraps(handler)
+    async def wrapper(request):
+        try:
+            await authenticate_jwk(request)
+        except web.HTTPException as e:
+            return web.json_response(data={"error": e.reason}, status=e.status)
+
+        return await handler(request)
+
+    return wrapper
+
+
+def get_itemhash_or_400(match_info: UrlMappingMatchInfo) -> ItemHash:
+    try:
+        ref = match_info["ref"]
+    except KeyError:
+        raise aiohttp.web_exceptions.HTTPBadRequest(body="Missing field: 'ref'")
+    try:
+        return ItemHash(ref)
+    except UnknownHashError:
+        raise aiohttp.web_exceptions.HTTPBadRequest(body=f"Invalid ref: '{ref}'")
+
+
+def get_execution_or_404(ref: ItemHash) -> VmExecution:
+    """Return the execution corresponding to the ref or raise an HTTP 404 error."""
+    execution = pool.executions.get(ref)
+    if execution:
+        return execution
+    else:
+        raise web.HTTPNotFound(body=f"No virtual machine with ref {ref}")
+
+
+@require_jwk_authentication
+async def stream_logs(request: web.Request):
+    # TODO: Add user authentication
+    vm_hash = get_itemhash_or_400(request.match_info)
+    execution = get_execution_or_404(vm_hash)
+
+    if execution.vm is None:
+        raise web.HTTPBadRequest(body=f"VM {vm_hash} is not running")
+
+    queue: asyncio.Queue = asyncio.Queue()
+    try:
+        ws = web.WebSocketResponse()
+        try:
+            await ws.prepare(request)
+
+            execution.vm.fvm.log_queues.append(queue)
+
+            while True:
+                log_type, message = await queue.get()
+                assert log_type in ("stdout", "stderr")
+
+                await ws.send_json({"type": log_type, "message": message.decode()})
+        finally:
+            await ws.close()
+    finally:
+        execution.vm.fvm.log_queues.remove(queue)
+        queue.empty()
+
+
+@require_jwk_authentication
+async def operate_expire(request: web.Request):
+    """Stop the virtual machine, smoothly if possible."""
+    # TODO: Add user authentication
+    vm_hash = get_itemhash_or_400(request.match_info)
+    timeout = float(ItemHash(request.match_info["timeout"]))
+    if not 0 < timeout < timedelta(days=10).total_seconds():
+        return web.HTTPBadRequest(body="Invalid timeout duration")
+
+    execution = get_execution_or_404(vm_hash)
+
+    logger.info(f"Expiring in {timeout} seconds: {execution.vm_hash}")
+    await execution.expire(timeout=timeout)
+    execution.persistent = False
+
+    return web.Response(
+        status=200, body=f"Expiring VM with ref {vm_hash} in {timeout} seconds"
+    )
+
+
+@require_jwk_authentication
+async def operate_stop(request: web.Request):
+    """Stop the virtual machine, smoothly if possible."""
+    # TODO: Add user authentication
+    vm_hash = get_itemhash_or_400(request.match_info)
+
+    logger.debug(f"Iterating through running executions... {pool.executions}")
+    execution = get_execution_or_404(vm_hash)
+
+    if execution.is_running:
+        logger.info(f"Stopping {execution.vm_hash}")
+        await execution.stop()
+        execution.persistent = False
+        return web.Response(status=200, body=f"Stopped VM with ref {vm_hash}")
+    else:
+        return web.Response(status=200, body=f"Already stopped, nothing to do")
+
+
+@require_jwk_authentication
+async def operate_reboot(request: web.Request):
+    """
+    Reboots the virtual machine, smoothly if possible.
+    """
+    vm_hash = get_itemhash_or_400(request.match_info)
+    execution = get_execution_or_404(vm_hash)
+
+    # TODO: implement this endpoint
+    logger.info(f"Rebooting {execution.vm_hash}")
+    return web.Response(status=200, body=f"Rebooted {execution.vm_hash}")
+
+
+@require_jwk_authentication
+async def operate_erase(request: web.Request):
+    """Delete all data stored by a virtual machine.
+    Stop the virtual machine first if needed.
+    """
+    vm_hash = get_itemhash_or_400(request.match_info)
+    execution = get_execution_or_404(vm_hash)
+
+    logger.info(f"Erasing {execution.vm_hash}")
+
+    # Stop the VM
+    await execution.stop()
+    execution.persistent = False
+
+    # Delete all data
+    if execution.resources is not None:
+        for volume in execution.resources.volumes:
+            if not volume.read_only:
+                logger.info(f"Deleting volume {volume.path_on_host}")
+                volume.path_on_host.unlink()
+
+    return web.Response(status=200, body=f"Erased VM with ref {vm_hash}")

From 55d735b74101ca9969b75a82700f4a61de72d3eb Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Sun, 16 Jul 2023 18:01:11 +0200
Subject: [PATCH 454/990] Fix: only start snapshots for instances (#387)

Problems:
* the supervisor attempts to start snapshot jobs for programs,
  which we do not want to support.
* the supervisor does not stop the snapshot job for an instance
  when the instance is stopped.

Solutions:
* add a check to avoid the issue.
* pass the snapshot manager object to each VM execution to stop
  the snapshot task when the execution is stopped.
---
 vm_supervisor/models.py           | 16 ++++++++++++++--
 vm_supervisor/pool.py             | 13 +++++++++----
 vm_supervisor/snapshot_manager.py | 11 +++++++----
 3 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 3e2d9541a..912fa0796 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -5,7 +5,7 @@
 from asyncio import Task
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Dict, Optional, Union
+from typing import TYPE_CHECKING, Dict, Optional, Union
 
 from aleph_message.models import (
     ExecutableContent,
@@ -17,6 +17,10 @@
 from .conf import settings
 from .metrics import ExecutionRecord, save_execution_data, save_record
 from .network.interfaces import TapInterface
+
+if TYPE_CHECKING:
+    from .snapshot_manager import SnapshotManager
+
 from .pubsub import PubSub
 from .utils import create_task_log_exceptions, dumps_for_json
 from .vm import AlephFirecrackerInstance
@@ -90,7 +94,11 @@ def vm_id(self) -> Optional[int]:
         return self.vm.vm_id if self.vm else None
 
     def __init__(
-        self, vm_hash: ItemHash, message: ExecutableContent, original: ExecutableContent
+        self,
+        vm_hash: ItemHash,
+        message: ExecutableContent,
+        original: ExecutableContent,
+        snapshot_manager: "SnapshotManager",
     ):
         self.uuid = uuid.uuid1()  # uuid1() includes the hardware address and timestamp
         self.vm_hash = vm_hash
@@ -100,6 +108,7 @@ def __init__(
         self.ready_event = asyncio.Event()
         self.concurrent_runs = 0
         self.runs_done_event = asyncio.Event()
+        self.snapshot_manager = snapshot_manager
 
     def to_dict(self) -> Dict:
         return {
@@ -217,6 +226,9 @@ async def stop(self):
         self.cancel_expiration()
         self.cancel_update()
 
+        if isinstance(self.message, InstanceContent):
+            await self.snapshot_manager.stop_for(self.vm_hash)
+
     def start_watching_for_updates(self, pubsub: PubSub):
         if not self.update_task:
             self.update_task = create_task_log_exceptions(
diff --git a/vm_supervisor/pool.py b/vm_supervisor/pool.py
index 20bee71b0..fe232d105 100644
--- a/vm_supervisor/pool.py
+++ b/vm_supervisor/pool.py
@@ -3,6 +3,7 @@
 from typing import Dict, Iterable, Optional
 
 from aleph_message.models import ExecutableMessage, ItemHash
+from aleph_message.models.execution.instance import InstanceContent
 
 from vm_supervisor.network.hostnetwork import Network, make_ipv6_allocator
 
@@ -57,7 +58,12 @@ async def create_a_vm(
         self, vm_hash: ItemHash, message: ExecutableContent, original: ExecutableContent
     ) -> VmExecution:
         """Create a new Aleph Firecracker VM from an Aleph function message."""
-        execution = VmExecution(vm_hash=vm_hash, message=message, original=original)
+        execution = VmExecution(
+            vm_hash=vm_hash,
+            message=message,
+            original=original,
+            snapshot_manager=self.snapshot_manager,
+        )
         self.executions[vm_hash] = execution
         await execution.prepare()
         vm_id = self.get_unique_vm_id()
@@ -71,7 +77,8 @@ async def create_a_vm(
         await execution.create(vm_id=vm_id, tap_interface=tap_interface)
 
         # Start VM snapshots automatically
-        await self.snapshot_manager.start_for(execution=execution)
+        if isinstance(message, InstanceContent):
+            await self.snapshot_manager.start_for(execution=execution)
 
         return execution
 
@@ -134,8 +141,6 @@ async def stop(self):
             *(execution.stop() for vm_hash, execution in self.executions.items())
         )
 
-        await self.snapshot_manager.stop_all()
-
     def get_persistent_executions(self) -> Iterable[VmExecution]:
         for vm_hash, execution in self.executions.items():
             if execution.persistent and execution.is_running:
diff --git a/vm_supervisor/snapshot_manager.py b/vm_supervisor/snapshot_manager.py
index 2e0a67668..75a865552 100644
--- a/vm_supervisor/snapshot_manager.py
+++ b/vm_supervisor/snapshot_manager.py
@@ -103,7 +103,7 @@ async def start_for(
         self, execution: VmExecution, frequency: Optional[int] = None
     ) -> None:
         if not execution.is_instance:
-            raise TypeError("VM execution should be an Instance only")
+            raise NotImplementedError("Snapshots are not implemented for programs.")
 
         if not frequency:
             frequency = settings.SNAPSHOT_FREQUENCY
@@ -119,10 +119,13 @@ async def start_for(
         await snapshot_execution.start()
 
     async def stop_for(self, vm_hash: ItemHash) -> None:
-        if not self.executions[vm_hash]:
-            raise ValueError(f"Snapshot execution not running for VM {vm_hash}")
+        try:
+            snapshot_execution = self.executions.pop(vm_hash)
+        except KeyError:
+            logger.warning("Could not find snapshot task for instance %s", vm_hash)
+            return
 
-        await self.executions[vm_hash].stop()
+        await snapshot_execution.stop()
 
     async def stop_all(self) -> None:
         await asyncio.gather(

From 8a830daaab7032fd08740c87de14b98821d622e2 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Tue, 18 Jul 2023 15:25:18 +0200
Subject: [PATCH 455/990] Fix: allow allocations without persistent VMs (#388)

Problem: the Allocation model requires a value to be specified for the
`persistent_vms` field.

Solution: default to an empty set.
---
 vm_supervisor/resources.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/vm_supervisor/resources.py b/vm_supervisor/resources.py
index 07425e0b3..d3931cf67 100644
--- a/vm_supervisor/resources.py
+++ b/vm_supervisor/resources.py
@@ -7,7 +7,7 @@
 import psutil
 from aiohttp import web
 from aleph_message.models.execution.environment import CpuProperties
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from .conf import settings
 
@@ -122,7 +122,7 @@ async def about_system_usage(request: web.Request):
 
 
 class Allocation(BaseModel):
-    persistent_vms: Set[str]
-    instances: Set[str] = set()
-    on_demand_vms: Set[str] = set()
+    persistent_vms: Set[str] = Field(default_factory=set)
+    instances: Set[str] = Field(default_factory=set)
+    on_demand_vms: Set[str] = Field(default_factory=set)
     jobs: Optional[Set] = None

From b7993f6b41111c5ee69bcd28e6b184856e43715d Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Tue, 18 Jul 2023 15:46:18 +0200
Subject: [PATCH 456/990] Fix: allow allocations without persistent VMs (#389)

Problem: the Allocation model requires a value to be specified for the
`persistent_vms` field.

Solution: default to an empty set.

From 4ebb8b8b2fb17e18030786fa838a1afef5a14861 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Tue, 18 Jul 2023 16:06:37 +0200
Subject: [PATCH 457/990] Fix: allow None as value for Allocation.on_demand_vms
 (#390)

---
 vm_supervisor/resources.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vm_supervisor/resources.py b/vm_supervisor/resources.py
index d3931cf67..fb2740891 100644
--- a/vm_supervisor/resources.py
+++ b/vm_supervisor/resources.py
@@ -124,5 +124,5 @@ async def about_system_usage(request: web.Request):
 class Allocation(BaseModel):
     persistent_vms: Set[str] = Field(default_factory=set)
     instances: Set[str] = Field(default_factory=set)
-    on_demand_vms: Set[str] = Field(default_factory=set)
-    jobs: Optional[Set] = None
+    on_demand_vms: Optional[Set[str]] = None
+    jobs: Optional[Set[str]] = None

From c000cf1408c1a7d5c71aaf7fe4cb01e629281eb5 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Thu, 20 Jul 2023 14:17:07 +0200
Subject: [PATCH 458/990] Fix: remove the `network` key from the cloud-init
 network file (#392)

Problem: the current network config file is incompatible with Debian 11.
This appears to be because the `network` key is not required.

Solution: remove the `network` key and put its content directly in the
dictionary.
---
 vm_supervisor/vm/firecracker/instance.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index 0835f71f2..733345e7b 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -211,21 +211,19 @@ def _create_network_file(self) -> bytes:
         ipv6_gateway = self.get_vm_ipv6_gateway()
 
         network = {
-            "network": {
-                "ethernets": {
-                    "eth0": {
-                        "dhcp4": False,
-                        "dhcp6": False,
-                        "addresses": [ip, ipv6],
-                        "gateway4": route,
-                        "gateway6": ipv6_gateway,
-                        "nameservers": {
-                            "addresses": settings.DNS_NAMESERVERS,
-                        },
+            "ethernets": {
+                "eth0": {
+                    "dhcp4": False,
+                    "dhcp6": False,
+                    "addresses": [ip, ipv6],
+                    "gateway4": route,
+                    "gateway6": ipv6_gateway,
+                    "nameservers": {
+                        "addresses": settings.DNS_NAMESERVERS,
                     },
                 },
-                "version": 2,
             },
+            "version": 2,
         }
 
         return yaml.safe_dump(

From 070a2131ebcae1a91f1da5ee3739849dfea67c89 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Thu, 20 Jul 2023 15:06:12 +0200
Subject: [PATCH 459/990] Fixed multiple block device size issues (#393)

* Fix: Added a new block device with total volume size as base from the snapshot one.

* Fix: make black code quality fixes.

---------

Co-authored-by: Andres D. Molins 
---
 vm_supervisor/storage.py | 42 +++++++++++++++++++++++++++-------------
 1 file changed, 29 insertions(+), 13 deletions(-)

diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index a778b794e..9250e4c66 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -243,7 +243,8 @@ async def create_mapped_device(device_name: str, table_command: str) -> None:
     await run_in_subprocess(command, stdin_input=table_command.encode())
 
 
-async def resize_file_system(device_path: Path, mount_path: Path) -> None:
+async def resize_and_tune_file_system(device_path: Path, mount_path: Path) -> None:
+    await run_in_subprocess(["btrfstune", "-m", str(device_path)])
     await run_in_subprocess(["mount", str(device_path), str(mount_path)])
     await run_in_subprocess(["btrfs", "filesystem", "resize", "max", str(mount_path)])
     await run_in_subprocess(["umount", str(mount_path)])
@@ -262,27 +263,42 @@ async def create_devmapper(
     if path_mapped_volume_name.is_block_device():
         return path_mapped_volume_name
 
-    volume_path = await create_volume_file(volume, namespace)
     parent_path = await get_rootfs_base_path(volume.parent.ref)
 
-    base_loop_device = await create_loopback_device(parent_path, read_only=True)
-    base_block_size: int = await get_block_size(parent_path)
-    extended_loop_device = await create_loopback_device(volume_path)
+    image_volume_name = volume.parent.ref
+    image_block_size: int = await get_block_size(parent_path)
+    path_image_device_name = Path(DEVICE_MAPPER_DIRECTORY) / image_volume_name
+    if not path_image_device_name.is_block_device():
+        image_loop_device = await create_loopback_device(parent_path, read_only=True)
+
+        base_table_command = f"0 {image_block_size} linear {image_loop_device} 0"
+        await create_mapped_device(image_volume_name, base_table_command)
+
+    volume_path = await create_volume_file(volume, namespace)
     extended_block_size: int = await get_block_size(volume_path)
 
-    base_table_command = f"0 {base_block_size} linear {base_loop_device} 0\n{base_block_size} {extended_block_size} zero"
-    base_volume_name = volume.parent.ref
-    path_base_device_name = Path(DEVICE_MAPPER_DIRECTORY) / base_volume_name
-    if not path_base_device_name.is_block_device():
-        await create_mapped_device(base_volume_name, base_table_command)
+    mapped_volume_name_base = f"{namespace}_base"
+    path_mapped_volume_name_base = (
+        Path(DEVICE_MAPPER_DIRECTORY) / mapped_volume_name_base
+    )
+    if not path_mapped_volume_name_base.is_block_device():
+        base_table_command = (
+            f"0 {image_block_size} linear {path_image_device_name} 0\n"
+            f"{image_block_size} {extended_block_size} zero "
+        )
+
+        await create_mapped_device(mapped_volume_name_base, base_table_command)
+
+    extended_loop_device = await create_loopback_device(volume_path)
 
-    snapshot_table_command = f"0 {extended_block_size} snapshot {path_base_device_name} {extended_loop_device} P 8"
+    snapshot_table_command = f"0 {extended_block_size} snapshot {path_mapped_volume_name_base} {extended_loop_device} P 8"
     await create_mapped_device(mapped_volume_name, snapshot_table_command)
 
     mount_path = Path(f"/mnt/{mapped_volume_name}")
     mount_path.mkdir(parents=True, exist_ok=True)
-    await resize_file_system(path_mapped_volume_name, mount_path)
-    await chown_to_jailman(path_base_device_name)
+    await resize_and_tune_file_system(path_mapped_volume_name, mount_path)
+    await chown_to_jailman(path_image_device_name)
+    await chown_to_jailman(path_mapped_volume_name_base)
     await chown_to_jailman(path_mapped_volume_name)
     return path_mapped_volume_name
 

From 23ab0ac7715f654b162450cd5a6602916f4b8abd Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Thu, 20 Jul 2023 15:26:16 +0200
Subject: [PATCH 460/990] Fix: supervisor stops instances for no reason (#396)

Problem: the check to stop persistent programs is too broad and includes
instances in the list when checking which persistent programs were
unscheduled.

Solution: check both instances and programs at the same time to reduce
redundancy.
---
 vm_supervisor/views/__init__.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/vm_supervisor/views/__init__.py b/vm_supervisor/views/__init__.py
index f787b4065..bc034f21d 100644
--- a/vm_supervisor/views/__init__.py
+++ b/vm_supervisor/views/__init__.py
@@ -207,23 +207,19 @@ async def update_allocations(request: web.Request):
         logger.info(f"Starting long running VM {vm_hash}")
         await start_persistent_vm(vm_hash, pubsub)
 
-    # Stop VMs
-    for execution in pool.get_persistent_executions():
-        if execution.vm_hash not in allocation.persistent_vms:
-            logger.info(f"Stopping long running VM {execution.vm_hash}")
-            await execution.stop()
-            execution.persistent = False
-
     # Start Instances
     for instance_hash in allocation.instances:
         instance_hash = ItemHash(instance_hash)
         logger.info(f"Starting instance {instance_hash}")
         await start_persistent_vm(instance_hash, pubsub)
 
-    # Stop Instances
-    for execution in pool.get_instance_executions():
-        if execution.vm_hash not in allocation.instances:
-            logger.info(f"Stopping instance {execution.vm_hash}")
+    # Stop unscheduled persistent programs and instances.
+    # Instances are also marked with persistent = True.
+    allocations = allocation.persistent_vms | allocation.instances
+    for execution in pool.get_persistent_executions():
+        if execution.vm_hash not in allocations:
+            vm_type = "instance" if execution.is_instance else "persistent program"
+            logger.info(f"Stopping %s %s", vm_type, execution.vm_hash)
             await execution.stop()
             execution.persistent = False
 

From 0662402f55e86ccca9c488df94f12e64ce1b33ac Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Thu, 20 Jul 2023 15:33:43 +0200
Subject: [PATCH 461/990] Added Devmapper flow comments (#395)

* Fix: Added comments about BTRFS specific things.

* Fix: Added some comments about devmapper flow.

---------

Co-authored-by: Andres D. Molins 
---
 vm_supervisor/storage.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 9250e4c66..dc4951ef7 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -244,6 +244,7 @@ async def create_mapped_device(device_name: str, table_command: str) -> None:
 
 
 async def resize_and_tune_file_system(device_path: Path, mount_path: Path) -> None:
+    # This tune is needed to assign a random fsid to BTRFS device to be able to mount it
     await run_in_subprocess(["btrfstune", "-m", str(device_path)])
     await run_in_subprocess(["mount", str(device_path), str(mount_path)])
     await run_in_subprocess(["btrfs", "filesystem", "resize", "max", str(mount_path)])
@@ -260,6 +261,7 @@ async def create_devmapper(
     mapped_volume_name = f"{namespace}_{volume_name}"
     path_mapped_volume_name = Path(DEVICE_MAPPER_DIRECTORY) / mapped_volume_name
 
+    # Check if rootfs volume is created
     if path_mapped_volume_name.is_block_device():
         return path_mapped_volume_name
 
@@ -268,9 +270,11 @@ async def create_devmapper(
     image_volume_name = volume.parent.ref
     image_block_size: int = await get_block_size(parent_path)
     path_image_device_name = Path(DEVICE_MAPPER_DIRECTORY) / image_volume_name
+    # Checks if parent rootfs image block device is created
     if not path_image_device_name.is_block_device():
         image_loop_device = await create_loopback_device(parent_path, read_only=True)
 
+        # Creates the parent rootfs image block device with the entire image size
         base_table_command = f"0 {image_block_size} linear {image_loop_device} 0"
         await create_mapped_device(image_volume_name, base_table_command)
 
@@ -282,15 +286,16 @@ async def create_devmapper(
         Path(DEVICE_MAPPER_DIRECTORY) / mapped_volume_name_base
     )
     if not path_mapped_volume_name_base.is_block_device():
+        # Creates the base rootfs block device with the entire rootfs size using the image block device as source
         base_table_command = (
             f"0 {image_block_size} linear {path_image_device_name} 0\n"
             f"{image_block_size} {extended_block_size} zero "
         )
-
         await create_mapped_device(mapped_volume_name_base, base_table_command)
 
     extended_loop_device = await create_loopback_device(volume_path)
 
+    # Creates the final rootfs block device that is a snapshot of the base block device
     snapshot_table_command = f"0 {extended_block_size} snapshot {path_mapped_volume_name_base} {extended_loop_device} P 8"
     await create_mapped_device(mapped_volume_name, snapshot_table_command)
 

From dca79dd758de9a3d07a1f29fd5029007168181f7 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Thu, 20 Jul 2023 15:53:01 +0200
Subject: [PATCH 462/990] Fix: Debian rootfs issues (#394)

* Fix: Apply fix on Debian scripts to generate the rootfs filesystem.

* Fix: Added some mkfs options on Debian 12 script.

---------

Co-authored-by: Andres D. Molins 
---
 runtimes/instance-rootfs/create-debian-11-disk.sh | 6 +++++-
 runtimes/instance-rootfs/create-debian-12-disk.sh | 5 ++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/runtimes/instance-rootfs/create-debian-11-disk.sh b/runtimes/instance-rootfs/create-debian-11-disk.sh
index 271e101f6..5d7b51fd4 100755
--- a/runtimes/instance-rootfs/create-debian-11-disk.sh
+++ b/runtimes/instance-rootfs/create-debian-11-disk.sh
@@ -26,7 +26,7 @@ curl -L "$IMAGE_URL" -o "$IMAGE_NAME"
 # Allocate 1GB rootfs.btrfs file
 echo "Allocate 1GB $ROOTFS_FILE file"
 fallocate -l 1G "$ROOTFS_FILE"
-mkfs.btrfs "$ROOTFS_FILE"
+mkfs.btrfs -m single --label root "$ROOTFS_FILE"
 mount "$ROOTFS_FILE" "$MOUNT_DIR"
 
 # Extract Debian image
@@ -38,6 +38,9 @@ LOOPDISK=$(losetup --find --show $IMAGE_RAW_NAME)
 partx -u $LOOPDISK
 mount "$LOOPDISK"p1 "$MOUNT_ORIGIN_DIR"
 
+# Fix boot partition missing
+sed -i '$d' "$MOUNT_ORIGIN_DIR"/etc/fstab
+
 # Copy Debian image to rootfs
 echo "Copying Debian 11 image to $ROOTFS_FILE file"
 cp -vap "$MOUNT_ORIGIN_DIR/." "$MOUNT_DIR"
@@ -47,5 +50,6 @@ umount "$MOUNT_ORIGIN_DIR"
 partx -d "$LOOPDISK"
 losetup -d "$LOOPDISK"
 umount "$MOUNT_DIR"
+
 rm "$IMAGE_RAW_NAME"
 rm "$IMAGE_NAME"
diff --git a/runtimes/instance-rootfs/create-debian-12-disk.sh b/runtimes/instance-rootfs/create-debian-12-disk.sh
index e236c78af..3078c5c26 100755
--- a/runtimes/instance-rootfs/create-debian-12-disk.sh
+++ b/runtimes/instance-rootfs/create-debian-12-disk.sh
@@ -26,7 +26,7 @@ curl -L "$IMAGE_URL" -o "$IMAGE_NAME"
 # Allocate 1GB rootfs.btrfs file
 echo "Allocate 1GB $ROOTFS_FILE file"
 fallocate -l 1G "$ROOTFS_FILE"
-mkfs.btrfs "$ROOTFS_FILE"
+mkfs.btrfs -m single --label root "$ROOTFS_FILE"
 mount "$ROOTFS_FILE" "$MOUNT_DIR"
 
 # Extract Debian image
@@ -38,6 +38,9 @@ LOOPDISK=$(losetup --find --show $IMAGE_RAW_NAME)
 partx -u $LOOPDISK
 mount "$LOOPDISK"p1 "$MOUNT_ORIGIN_DIR"
 
+# Fix boot partition missing
+sed -i '$d' "$MOUNT_ORIGIN_DIR"/etc/fstab
+
 # Copy Debian image to rootfs
 echo "Copying Debian 12 image to $ROOTFS_FILE file"
 cp -vap "$MOUNT_ORIGIN_DIR/." "$MOUNT_DIR"

From f1c0427fdcc320d619e5e11f388c8bc2816b1ccd Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Tue, 25 Jul 2023 13:43:23 +0200
Subject: [PATCH 463/990] Feature: install Kubo on CRNs (#385)

* Feature: install Kubo on CRNs

Problem: we wish to upload large snapshot files to the aleph.im network.
The aleph.im API requires large files to be available on IPFS.

Solution: install Kubo (IPFS daemon) on CRNs.

* fix init cmds

* systemd

* more fixes

* fix

* dep mgmt
---
 packaging/Makefile                            |  7 ++-
 packaging/aleph-vm/DEBIAN/postinst            |  1 +
 .../system/aleph-vm-supervisor.service        |  3 +-
 .../aleph-vm/etc/systemd/system/ipfs.service  | 47 +++++++++++++++++++
 4 files changed, 56 insertions(+), 2 deletions(-)
 create mode 100644 packaging/aleph-vm/etc/systemd/system/ipfs.service

diff --git a/packaging/Makefile b/packaging/Makefile
index d5140c8ad..e5bc51eca 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -20,12 +20,13 @@ debian-package-code:
 	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.0' 'jwskate==0.8.0' 'eth-account==0.9.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
-debian-package-resources: firecracker-bins vmlinux
+debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo
 	rm -fr ./aleph-vm/opt/firecracker
 	mkdir -p ./aleph-vm/opt/firecracker
 	cp -pr ./target/vmlinux.bin ./aleph-vm/opt/firecracker/
 	cp -pr ./target/firecracker ./aleph-vm/opt/firecracker/
 	cp -pr ./target/jailer ./aleph-vm/opt/firecracker/
+	cp -pr ./target/kubo/kubo ./aleph-vm/opt/kubo
 
 firecracker-bins: target-dir build-dir
 	mkdir -p ./build/firecracker-release
@@ -42,6 +43,10 @@ vmlinux:
 	curl -fsSL -o ./target/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.2.2/vmlinux.bin
 	#cp ../kernels/vmlinux.bin ./target/vmlinux.bin
 
+download-ipfs-kubo: target-dir build-dir
+	mkdir -p ./target/kubo
+	curl -fsSL https://dist.ipfs.tech/kubo/v0.21.0/kubo_v0.21.0_linux-amd64.tar.gz | tar -xz --directory ./target/kubo
+
 version:
 	python3 ./version_from_git.py --inplace deb aleph-vm/DEBIAN/control
 	python3 ./version_from_git.py --inplace __version__ ../vm_supervisor/version.py
diff --git a/packaging/aleph-vm/DEBIAN/postinst b/packaging/aleph-vm/DEBIAN/postinst
index f7d9f642a..99af91c2d 100755
--- a/packaging/aleph-vm/DEBIAN/postinst
+++ b/packaging/aleph-vm/DEBIAN/postinst
@@ -12,6 +12,7 @@ mkdir -p /var/lib/aleph/vm/jailer
 # Systemd is absent from containers
 if ! [[ -v container ]]; then
   systemctl daemon-reload
+  systemctl enable ipfs.service
   systemctl enable aleph-vm-supervisor.service
   systemctl restart aleph-vm-supervisor.service
 fi
diff --git a/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service b/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
index 4d5c41929..e5a904d28 100644
--- a/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
+++ b/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
@@ -1,6 +1,7 @@
 [Unit]
 Description=Aleph.im VM execution engine
-After=network.target
+After=network.target ipfs.service
+Requires=ipfs.service
 
 [Service]
 User=0
diff --git a/packaging/aleph-vm/etc/systemd/system/ipfs.service b/packaging/aleph-vm/etc/systemd/system/ipfs.service
new file mode 100644
index 000000000..f3506a275
--- /dev/null
+++ b/packaging/aleph-vm/etc/systemd/system/ipfs.service
@@ -0,0 +1,47 @@
+# This file will be overwritten on package upgrades, avoid customizations here.
+#
+# To make persistent changes, create file in
+# "/etc/systemd/system/ipfs.service.d/overwrite.conf" with
+# `systemctl edit ipfs.service`. This file will be parsed after this
+# file has been parsed.
+#
+# To overwrite a variable, like ExecStart you have to specify it once
+# blank and a second time with a new value, like:
+# ExecStart=
+# ExecStart=/usr/bin/ipfs daemon --flag1 --flag2
+#
+# For more info about custom unit files see systemd.unit(5).
+
+[Unit]
+Description=InterPlanetary File System (IPFS) daemon
+Documentation=https://docs.ipfs.tech/
+After=network.target
+
+[Service]
+
+# enable for 1-1024 port listening
+#AmbientCapabilities=CAP_NET_BIND_SERVICE
+# enable to specify a custom path see docs/environment-variables.md for further documentations
+#Environment=IPFS_PATH=/custom/ipfs/path
+# enable to specify a higher limit for open files/connections
+#LimitNOFILE=1000000
+
+#don't use swap
+MemorySwapMax=0
+
+# Don't timeout on startup. Opening the IPFS repo can take a long time in some cases (e.g., when
+# badger is recovering) and migrations can delay startup.
+#
+# Ideally, we'd be a bit smarter about this but there's no good way to do that without hooking
+# systemd dependencies deeper into go-ipfs.
+TimeoutStartSec=infinity
+
+Type=notify
+StateDirectory=ipfs
+Environment=IPFS_PATH="${HOME}"
+ExecStart=/opt/kubo/ipfs daemon --init --migrate
+Restart=on-failure
+KillSignal=SIGINT
+
+[Install]
+WantedBy=default.target

From b9522f4d3abf8b3716756b1233e1189a887c4d77 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Sun, 30 Jul 2023 14:19:12 +0200
Subject: [PATCH 464/990] Fix: start IPFS daemon with the server profile

Problem: in normal mode, the IPFS daemon advertises itself with private
addresses. Some cloud providers do not like this at all and will send
abuse notices related to port scanning.

Solution: start the daemon with the server profile, which only
advertises the node with public addresses and avoids port scanning.
---
 packaging/aleph-vm/etc/systemd/system/ipfs.service | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/aleph-vm/etc/systemd/system/ipfs.service b/packaging/aleph-vm/etc/systemd/system/ipfs.service
index f3506a275..914593bcf 100644
--- a/packaging/aleph-vm/etc/systemd/system/ipfs.service
+++ b/packaging/aleph-vm/etc/systemd/system/ipfs.service
@@ -39,7 +39,7 @@ TimeoutStartSec=infinity
 Type=notify
 StateDirectory=ipfs
 Environment=IPFS_PATH="${HOME}"
-ExecStart=/opt/kubo/ipfs daemon --init --migrate
+ExecStart=/opt/kubo/ipfs daemon --init --init-profile=server --migrate
 Restart=on-failure
 KillSignal=SIGINT
 

From 4df135017a7d1db68e95645411a568387c273ad2 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 25 Aug 2023 17:25:52 +0200
Subject: [PATCH 465/990] Fix: Minor code fixes and cleanups

---
 firecracker/microvm.py          | 4 +---
 vm_supervisor/storage.py        | 2 +-
 vm_supervisor/views/operator.py | 4 +++-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 9c3a1e6e2..5f656df2d 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -30,7 +30,6 @@ class MicroVMFailedInit(Exception):
 
 # extend the json.JSONEncoder class to support bytes
 class JSONBytesEncoder(json.JSONEncoder):
-
     # overload method default
     def default(self, obj):
         # Match all the types you want to handle in your converter
@@ -77,7 +76,7 @@ class MicroVM:
     proc: Optional[asyncio.subprocess.Process] = None
     stdout_task: Optional[Task] = None
     stderr_task: Optional[Task] = None
-    log_queues: List
+    log_queues: List[asyncio.Queue]
     config_file_path: Optional[Path] = None
     drives: List[Drive]
     init_timeout: float
@@ -160,7 +159,6 @@ async def start(self, config: FirecrackerConfig) -> asyncio.subprocess.Process:
     async def start_firecracker(
         self, config: FirecrackerConfig
     ) -> asyncio.subprocess.Process:
-
         if os.path.exists(VSOCK_PATH):
             os.remove(VSOCK_PATH)
         if os.path.exists(self.socket_path):
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index dc4951ef7..4d9db0d6b 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -38,7 +38,7 @@
 DEVICE_MAPPER_DIRECTORY = "/dev/mapper"
 
 
-class NotEnoughDiskSpace(Exception):
+class NotEnoughDiskSpace(OSError):
     pass
 
 
diff --git a/vm_supervisor/views/operator.py b/vm_supervisor/views/operator.py
index 0ea1a2a97..e7d0cf616 100644
--- a/vm_supervisor/views/operator.py
+++ b/vm_supervisor/views/operator.py
@@ -164,7 +164,9 @@ async def stream_logs(request: web.Request):
 
 @require_jwk_authentication
 async def operate_expire(request: web.Request):
-    """Stop the virtual machine, smoothly if possible."""
+    """Stop the virtual machine, smoothly if possible.
+
+    A timeout may be specified to delay the action."""
     # TODO: Add user authentication
     vm_hash = get_itemhash_or_400(request.match_info)
     timeout = float(ItemHash(request.match_info["timeout"]))

From e1f4e0d85dbd646b64df4b3d1215ab908d0bfa9c Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Mon, 11 Sep 2023 23:37:30 +0200
Subject: [PATCH 466/990] Fix: no HTTPs in IPv4 diagnostic check

Problem: a node operator reports a timeout when attempting to connect to
`https://9.9.9.9`. However, he can ping `9.9.9.9` just fine.

Solution: as the diagnostic check is targeted at IPv4 connectivity only,
use a socket to attempt to connect to 9.9.9.9:53 directly.
---
 examples/example_fastapi/main.py | 17 +++++------------
 vm_supervisor/status.py          |  1 -
 2 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index 810295c3a..d278b70aa 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -98,18 +98,11 @@ async def ip_address():
 
 @app.get("/ip/4")
 async def connect_ipv4():
-    """Connect to the Quad9 VPN provider using their IPv4 address.
-    The webserver on that address returns a 404 error, so we accept that response code.
-    """
-    timeout = aiohttp.ClientTimeout(total=5)
-    async with aiohttp.ClientSession(
-        connector=aiohttp.TCPConnector(), timeout=timeout
-    ) as session:
-        async with session.get("https://9.9.9.9") as resp:
-            # We expect this endpoint to return a 404 error
-            if resp.status != 404:
-                resp.raise_for_status()
-            return {"result": True, "headers": resp.headers}
+    """Connect to the Quad9 VPN provider using their IPv4 address."""
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.settimeout(5)
+    sock.connect(("9.9.9.9", 53))
+    return {"result": True}
 
 
 @app.get("/ip/6")
diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py
index e5482bddf..4f4cf5e79 100644
--- a/vm_supervisor/status.py
+++ b/vm_supervisor/status.py
@@ -68,7 +68,6 @@ async def check_ipv4(session: ClientSession) -> bool:
     try:
         result: Dict = await get_json_from_vm(session, "/ip/4")
         assert result["result"] is True
-        assert "headers" in result
         return True
     except ClientResponseError:
         return False

From 2b39b16019ba2122d5de35a73e37da531ced939b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 19 Sep 2023 10:23:57 +0200
Subject: [PATCH 467/990] Fix: Caddyfile syntax broken in 2.7.4 (#405)

The syntax of the Caddyfile documented in the quick install guides is broken in Caddy 2.7.4.

This removes the unused `on_demand` section from the configuration, restoring compatibility with newer versions of Caddy.

We maintain the `on_demand_tls` section in case we may re-enable wildcard domains on the default config.
---
 doc/INSTALL-Debian-11.md    | 3 ---
 doc/INSTALL-Debian-12.md    | 3 ---
 doc/INSTALL-Ubuntu-22.04.md | 3 ---
 3 files changed, 9 deletions(-)

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index 5d2d3f00c..03c940f96 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -121,9 +121,6 @@ cat >/etc/caddy/Caddyfile </etc/caddy/Caddyfile </etc/caddy/Caddyfile <
Date: Tue, 19 Sep 2023 18:51:20 +0200
Subject: [PATCH 468/990] CI: Update to actions/checkout@v4 (#407)

---
 .github/workflows/build-deb-package.yml       | 24 ++++++++-----------
 .github/workflows/code-quality.yml            |  2 +-
 .github/workflows/codeql-analysis.yml         |  2 +-
 .github/workflows/test-build-examples.yml     |  2 +-
 .../workflows/test-new-runtime-examples.yml   |  2 +-
 .../workflows/test-on-droplet-debian-11.yml   |  2 +-
 .../workflows/test-on-droplet-debian-12.yml   |  2 +-
 .../test-on-droplet-ubuntu-22.04.yml          |  2 +-
 8 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index a1aa1057e..ef9d04b67 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -9,12 +9,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Unshallow
-        run: |
-          git fetch --prune --unshallow
-          git describe --tags
+        uses: actions/checkout@v4
+        with:
+          # Fetch the whole history for all tags and branches (required for aleph.__version__)
+          fetch-depth: 0
 
       - run: |
           cd packaging && make all-podman-debian-11 && cd ..
@@ -31,12 +29,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Unshallow
-        run: |
-          git fetch --prune --unshallow
-          git describe --tags
+        uses: actions/checkout@v4
+        with:
+          # Fetch the whole history for all tags and branches (required for aleph.__version__)
+          fetch-depth: 0
 
       - run: |
           cd packaging && make all-podman-ubuntu-2204 && cd ..
@@ -53,7 +49,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
         run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
@@ -75,7 +71,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - run: |
           docker build -t aleph-vm-build-squashfs -f examples/volumes/Dockerfile examples/volumes
diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
index 2d9bcd3fd..3e8b32e01 100644
--- a/.github/workflows/code-quality.yml
+++ b/.github/workflows/code-quality.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
         run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index deeb02846..d5a5fff40 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -39,7 +39,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
diff --git a/.github/workflows/test-build-examples.yml b/.github/workflows/test-build-examples.yml
index 1884bdc0e..220b579f4 100644
--- a/.github/workflows/test-build-examples.yml
+++ b/.github/workflows/test-build-examples.yml
@@ -11,7 +11,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
         run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
diff --git a/.github/workflows/test-new-runtime-examples.yml b/.github/workflows/test-new-runtime-examples.yml
index 966cf0bcb..43f77f8c2 100644
--- a/.github/workflows/test-new-runtime-examples.yml
+++ b/.github/workflows/test-new-runtime-examples.yml
@@ -10,7 +10,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # Fetch the whole history for all tags and branches (required for aleph.__version__)
           fetch-depth: 0
diff --git a/.github/workflows/test-on-droplet-debian-11.yml b/.github/workflows/test-on-droplet-debian-11.yml
index b99d58ae7..3dc052c84 100644
--- a/.github/workflows/test-on-droplet-debian-11.yml
+++ b/.github/workflows/test-on-droplet-debian-11.yml
@@ -10,7 +10,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # Fetch the whole history for all tags and branches (required for aleph.__version__)
           fetch-depth: 0
diff --git a/.github/workflows/test-on-droplet-debian-12.yml b/.github/workflows/test-on-droplet-debian-12.yml
index 7eecd5825..0d59003b9 100644
--- a/.github/workflows/test-on-droplet-debian-12.yml
+++ b/.github/workflows/test-on-droplet-debian-12.yml
@@ -10,7 +10,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # Fetch the whole history for all tags and branches (required for aleph.__version__)
           fetch-depth: 0
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index b1e9b6f28..ea70f8392 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -10,7 +10,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # Fetch the whole history for all tags and branches (required for aleph.__version__)
           fetch-depth: 0

From 295b4396452524ea6180c7a0c9f61463d29efc38 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 19 Sep 2023 11:40:30 +0200
Subject: [PATCH 469/990] Fix: Operators had to specify network interface

The network interfaces defaulted to `eth0`, which not the default on all supported systems.

This uses the default network interface instead, and should require less configuration from node operators and developers.

Fixes #253
---
 doc/INSTALL-Debian-12.md    |  3 ++-
 doc/INSTALL-Ubuntu-22.04.md |  3 ++-
 vm_supervisor/conf.py       | 16 +++++++++++++++-
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/doc/INSTALL-Debian-12.md b/doc/INSTALL-Debian-12.md
index 598dd1dbb..fdb342224 100644
--- a/doc/INSTALL-Debian-12.md
+++ b/doc/INSTALL-Debian-12.md
@@ -56,7 +56,8 @@ ALEPH_VM_DOMAIN_NAME=vm.example.org
 
 #### Network configuration
 
-On some systems, the default network interface is not `eth0` and you will want to configure the default interface
+The default network interface is detected automatically from the IP routes. 
+On some systems, this is not the desired configuration and you will want to configure the default interface
 by adding:
 ```
 ALEPH_VM_NETWORK_INTERFACE=enp0s1
diff --git a/doc/INSTALL-Ubuntu-22.04.md b/doc/INSTALL-Ubuntu-22.04.md
index da12dadf3..31898dcfa 100644
--- a/doc/INSTALL-Ubuntu-22.04.md
+++ b/doc/INSTALL-Ubuntu-22.04.md
@@ -64,7 +64,8 @@ ALEPH_VM_DNS_RESOLUTION=resolvectl
 
 > 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
 
-On some systems, the default network interface is not `eth0` and you will want to configure the default interface
+The default network interface is detected automatically from the IP routes. 
+On some systems, this is not the desired configuration and you will want to configure the default interface
 by adding:
 ```
 ALEPH_VM_NETWORK_INTERFACE=enp0s1
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index 3604107ff..c9e304921 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -69,6 +69,16 @@ def resolvectl_dns_servers_ipv4(interface: str) -> Iterable[str]:
             yield server
 
 
+def get_default_interface() -> Optional[str]:
+    """Returns the default network interface"""
+    with open("/proc/net/route", "r") as f:
+        for line in f.readlines():
+            parts = line.strip().split()
+            if parts[1] == "00000000":  # Indicates default route
+                return parts[0]
+    return None
+
+
 class Settings(BaseSettings):
     SUPERVISOR_HOST = "127.0.0.1"
     SUPERVISOR_PORT: int = 4020
@@ -93,7 +103,7 @@ class Settings(BaseSettings):
 
     # Networking does not work inside Docker/Podman
     ALLOW_VM_NETWORKING = True
-    NETWORK_INTERFACE = "eth0"
+    NETWORK_INTERFACE: Optional[str] = None
     IPV4_ADDRESS_POOL = Field(
         default="172.16.0.0/12",
         description="IPv4 address range used to provide networks to VMs.",
@@ -236,6 +246,7 @@ def check(self):
         assert isfile(self.FIRECRACKER_PATH), f"File not found {self.FIRECRACKER_PATH}"
         assert isfile(self.JAILER_PATH), f"File not found {self.JAILER_PATH}"
         assert isfile(self.LINUX_PATH), f"File not found {self.LINUX_PATH}"
+        assert self.NETWORK_INTERFACE, f"Network interface is not specified"
         assert self.CONNECTOR_URL.startswith(
             "http://"
         ) or self.CONNECTOR_URL.startswith("https://")
@@ -271,6 +282,9 @@ def setup(self):
         os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True)
         os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True)
 
+        if not self.NETWORK_INTERFACE:
+            self.NETWORK_INTERFACE = get_default_interface()
+
         if self.DNS_NAMESERVERS is None and self.DNS_RESOLUTION:
             if self.DNS_RESOLUTION == DnsResolver.resolv_conf:
                 self.DNS_NAMESERVERS = list(etc_resolv_conf_dns_servers())

From 1d0ddad2bc9178005fa8a83a826ea30cb8d3e26d Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 19 Sep 2023 11:56:28 +0200
Subject: [PATCH 470/990] Fix: Operators had to specify DNS resolution

The DNS resolver defaulted to `resolv.conf`, which is not the default on all supported systems.

This adds an option to detect if systemd-resolved is available, and else defaults to `/etc/resolv.conf`.
---
 .../test-on-droplet-ubuntu-22.04.yml          |  1 -
 doc/INSTALL-Debian-12.md                      | 12 +++---
 doc/INSTALL-Ubuntu-22.04.md                   | 20 +++++----
 vm_supervisor/conf.py                         | 42 ++++++++++++++-----
 4 files changed, 48 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
index ea70f8392..622b1db10 100644
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
@@ -65,7 +65,6 @@ jobs:
           scp packaging/target/aleph-vm.ubuntu-22.04.deb root@${DROPLET_IPV4}:/opt
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.ubuntu-22.04.deb"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
-          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_DNS_RESOLUTION=resolvectl >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
 
diff --git a/doc/INSTALL-Debian-12.md b/doc/INSTALL-Debian-12.md
index fdb342224..6bb612e8f 100644
--- a/doc/INSTALL-Debian-12.md
+++ b/doc/INSTALL-Debian-12.md
@@ -56,19 +56,19 @@ ALEPH_VM_DOMAIN_NAME=vm.example.org
 
 #### Network configuration
 
+The network configuration is detected automatically.
+
 The default network interface is detected automatically from the IP routes. 
-On some systems, this is not the desired configuration and you will want to configure the default interface
-by adding:
+You can configure the default interface manually instead by adding:
 ```
 ALEPH_VM_NETWORK_INTERFACE=enp0s1
 ```
 (don't forget to replace `enp0s1` with the name of your default network interface).
 
-Debian 12 by default uses `/etc/resolv.conf` for DNS resolution. The VM Supervisor uses this by default.
-If your system uses [systemd-resolved](https://manpages.debian.org/bullseye/systemd/systemd-resolved.8.en.html)
-instead, uncomment and add the following setting:
+You can configure the DNS resolver manually by using one of the following options:
 ```
-#ALEPH_VM_DNS_RESOLUTION=resolvctl
+ALEPH_VM_DNS_RESOLUTION=resolvectl
+ALEPH_VM_DNS_RESOLUTION=resolv.conf
 ```
 
 > 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
diff --git a/doc/INSTALL-Ubuntu-22.04.md b/doc/INSTALL-Ubuntu-22.04.md
index 31898dcfa..bb62fc3e1 100644
--- a/doc/INSTALL-Ubuntu-22.04.md
+++ b/doc/INSTALL-Ubuntu-22.04.md
@@ -56,22 +56,24 @@ ALEPH_VM_DOMAIN_NAME=vm.example.org
 
 #### Network configuration
 
-Ubuntu 22.04 by default uses [systemd-resolved](https://manpages.ubuntu.com/manpages/jammy/man8/systemd-resolved.service.8.html)
-for DNS resolution. The following setting configures the VM Supervisor to use it instead of reading the default `/etc/resolv.conf`.
-```
-ALEPH_VM_DNS_RESOLUTION=resolvectl
-```
-
-> 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
+The network configuration is detected automatically.
 
 The default network interface is detected automatically from the IP routes. 
-On some systems, this is not the desired configuration and you will want to configure the default interface
-by adding:
+You can configure the default interface manually instead by adding:
 ```
 ALEPH_VM_NETWORK_INTERFACE=enp0s1
 ```
 (don't forget to replace `enp0s1` with the name of your default network interface).
 
+You can configure the DNS resolver manually by using one of the following options:
+```
+ALEPH_VM_DNS_RESOLUTION=resolvectl
+ALEPH_VM_DNS_RESOLUTION=resolv.conf
+```
+
+> 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
+
+
 #### Volumes and partitions
 
 Two directories are used to store data from the network:
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index c9e304921..e4a46f0da 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -5,7 +5,7 @@
 from enum import Enum
 from os.path import abspath, exists, isdir, isfile, join
 from pathlib import Path
-from subprocess import check_output
+from subprocess import CalledProcessError, check_output
 from typing import Any, Dict, Iterable, List, Literal, NewType, Optional, Union
 
 from pydantic import BaseSettings, Field
@@ -19,6 +19,7 @@
 
 
 class DnsResolver(str, Enum):
+    detect = "detect"  # Detect the resolver used by the system
     resolv_conf = "resolv.conf"  # Simply copy from /etc/resolv.conf
     resolvectl = "resolvectl"  # Systemd-resolved, common on Ubuntu
 
@@ -79,6 +80,30 @@ def get_default_interface() -> Optional[str]:
     return None
 
 
+def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> List[str]:
+    # The match syntax is not yet available as of Python 3.9
+    # match dns_resolver:
+    if dns_resolver == DnsResolver.detect:
+        # Use a try-except approach since resolvectl can be present but disabled and raise the following
+        # "Failed to get global data: Unit dbus-org.freedesktop.resolve1.service not found."
+        try:
+            return list(resolvectl_dns_servers_ipv4(interface=network_interface))
+        except (FileNotFoundError, CalledProcessError):
+            if Path("/etc/resolv.conf").exists():
+                return list(etc_resolv_conf_dns_servers())
+            else:
+                raise FileNotFoundError("No DNS resolver found")
+
+    elif dns_resolver == DnsResolver.resolv_conf:
+        return list(etc_resolv_conf_dns_servers())
+
+    elif dns_resolver == DnsResolver.resolvectl:
+        return list(resolvectl_dns_servers_ipv4(interface=network_interface))
+
+    else:
+        assert "No DNS resolve defined, this should never happen."
+
+
 class Settings(BaseSettings):
     SUPERVISOR_HOST = "127.0.0.1"
     SUPERVISOR_PORT: int = 4020
@@ -134,7 +159,7 @@ class Settings(BaseSettings):
         description="Use the Neighbor Discovery Protocol Proxy to respond to Router Solicitation for instances on IPv6",
     )
 
-    DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.resolv_conf
+    DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.detect
     DNS_NAMESERVERS: Optional[List[str]] = None
 
     FIRECRACKER_PATH = Path("/opt/firecracker/firecracker")
@@ -286,15 +311,10 @@ def setup(self):
             self.NETWORK_INTERFACE = get_default_interface()
 
         if self.DNS_NAMESERVERS is None and self.DNS_RESOLUTION:
-            if self.DNS_RESOLUTION == DnsResolver.resolv_conf:
-                self.DNS_NAMESERVERS = list(etc_resolv_conf_dns_servers())
-
-            elif self.DNS_RESOLUTION == DnsResolver.resolvectl:
-                self.DNS_NAMESERVERS = list(
-                    resolvectl_dns_servers_ipv4(interface=self.NETWORK_INTERFACE)
-                )
-            else:
-                assert "This should never happen"
+            self.DNS_NAMESERVERS = obtain_dns_ips(
+                dns_resolver=self.DNS_RESOLUTION,
+                network_interface=self.NETWORK_INTERFACE,
+            )
 
     def display(self) -> str:
         attributes: Dict[str, Any] = {}

From 3bfada47703df02904c86e8a0c69df4dfc543630 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 20 Sep 2023 18:21:15 +0200
Subject: [PATCH 471/990] Cleanup: Fix Mypy errors in init1.py

---
 runtimes/aleph-debian-11-python/init1.py | 27 ++++++++++++------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index ff326f632..006109aaa 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -24,7 +24,7 @@
 from io import StringIO
 from os import system
 from shutil import make_archive
-from typing import Any, AsyncIterable, Dict, List, NewType, Optional, Tuple, Union
+from typing import Any, AsyncIterable, Dict, List, NewType, Optional, Tuple, Union, cast
 
 import aiohttp
 import msgpack
@@ -62,9 +62,9 @@ class ConfigurationPayload:
     input_data: bytes
     interface: Interface
     vm_hash: str
-    code: bytes = None
-    encoding: Encoding = None
-    entrypoint: str = None
+    code: bytes
+    encoding: Encoding
+    entrypoint: str
     ip: Optional[str] = None
     ipv6: Optional[str] = None
     route: Optional[str] = None
@@ -225,7 +225,7 @@ def setup_code_asgi(
         app = locals[entrypoint]
     else:
         raise ValueError(f"Unknown encoding '{encoding}'")
-    return app
+    return ASGIApplication(app)
 
 
 def setup_code_executable(
@@ -261,9 +261,9 @@ def setup_code_executable(
 
 
 def setup_code(
-    code: Optional[bytes],
-    encoding: Optional[Encoding],
-    entrypoint: Optional[str],
+    code: bytes,
+    encoding: Encoding,
+    entrypoint: str,
     interface: Interface,
 ) -> Union[ASGIApplication, subprocess.Popen]:
     if interface == Interface.asgi:
@@ -284,7 +284,7 @@ async def run_python_code_http(
         # Execute in the same process, saves ~20ms than a subprocess
 
         # The body should not be part of the ASGI scope itself
-        body: bytes = scope.pop("body")
+        request_body: bytes = scope.pop("body")
 
         async def receive():
             type_ = (
@@ -292,7 +292,7 @@ async def receive():
                 if scope["type"] in ("http", "websocket")
                 else "aleph.message"
             )
-            return {"type": type_, "body": body, "more_body": False}
+            return {"type": type_, "body": request_body, "more_body": False}
 
         send_queue: asyncio.Queue = asyncio.Queue()
 
@@ -311,13 +311,13 @@ async def send(dico):
             headers = {}
 
         logger.debug("Waiting for body")
-        body: Dict = await send_queue.get()
+        response_body: Dict = await send_queue.get()
 
         logger.debug("Waiting for buffer")
         output = buf.getvalue()
 
         logger.debug(f"Headers {headers}")
-        logger.debug(f"Body {body}")
+        logger.debug(f"Body {response_body}")
         logger.debug(f"Output {output}")
 
     logger.debug("Getting output data")
@@ -330,7 +330,7 @@ async def send(dico):
         output_data = b""
 
     logger.debug("Returning result")
-    return headers, body, output, output_data
+    return headers, response_body, output, output_data
 
 
 async def make_request(session, scope):
@@ -429,6 +429,7 @@ async def process_instruction(
             output_data: Optional[bytes]
 
             if interface == Interface.asgi:
+                application = cast(ASGIApplication, application)
                 headers, body, output, output_data = await run_python_code_http(
                     application=application, scope=payload.scope
                 )

From 2b210c53c3ed5907fe34887cdd6f9856068cc35b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 25 Sep 2023 10:43:47 +0200
Subject: [PATCH 472/990] Fix: Noisy tracebacks filled logs

The Diagnostics VM checks for the proper handling of exceptions.

This fills the logs with noisy stack traces.

Solution: Add a setting that defaults to ignoring that specific error and stacktrace.
---
 vm_supervisor/conf.py |  1 +
 vm_supervisor/run.py  | 16 +++++++++++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index e4a46f0da..a5d54cea4 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -124,6 +124,7 @@ class Settings(BaseSettings):
     USE_JAILER = True
     # System logs make boot ~2x slower
     PRINT_SYSTEM_LOGS = False
+    IGNORE_TRACEBACK_FROM_DIAGNOSTICS = True
     DEBUG_ASYNCIO = False
 
     # Networking does not work inside Docker/Podman
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 67aa2d21b..934013cc3 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -131,7 +131,21 @@ async def run_code_on_request(
         logger.debug(f"Result from VM: <<<\n\n{str(result)[:1000]}\n\n>>>")
 
         if "traceback" in result:
-            logger.warning(result["traceback"])
+            # An error took place, the stacktrace of the error will be returned.
+            # TODO: Add an option for VM developers to prevent stacktraces from being exposed.
+
+            # The Diagnostics VM checks for the proper handling of exceptions.
+            # This fills the logs with noisy stack traces, so we ignore this specific error.
+            ignored_error = 'raise CustomError("Whoops")'
+
+            if (
+                settings.IGNORE_TRACEBACK_FROM_DIAGNOSTICS
+                and ignored_error in result["traceback"]
+            ):
+                logger.debug('Ignored traceback from CustomError("Whoops")')
+            else:
+                logger.warning(result["traceback"])
+
             return web.Response(
                 status=500,
                 reason="Error in VM execution",

From d5061f53af55c5692696339ff85503b08f43e3b1 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 25 Sep 2023 10:54:25 +0200
Subject: [PATCH 473/990] Cleanup: Missing types and variables shadowing

Minor code cleanup.

Sharing variable names shadowed the variables from outer scope, and types were missing.
---
 vm_supervisor/vm/firecracker/program.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/vm_supervisor/vm/firecracker/program.py b/vm_supervisor/vm/firecracker/program.py
index 61e1b2ade..b4a0153f0 100644
--- a/vm_supervisor/vm/firecracker/program.py
+++ b/vm_supervisor/vm/firecracker/program.py
@@ -4,6 +4,7 @@
 import dataclasses
 import logging
 import os.path
+from asyncio import StreamReader, StreamWriter
 from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
@@ -439,17 +440,19 @@ async def run_code(
         logger.debug("running code")
         scope = scope or {}
 
-        async def communicate(reader, writer, scope) -> bytes:
-            payload = RunCodePayload(scope=scope)
+        async def communicate(
+            reader_: StreamReader, writer_: StreamWriter, scope_: dict
+        ) -> bytes:
+            payload = RunCodePayload(scope=scope_)
 
-            writer.write(b"CONNECT 52\n" + payload.as_msgpack())
-            await writer.drain()
+            writer_.write(b"CONNECT 52\n" + payload.as_msgpack())
+            await writer_.drain()
 
-            ack: bytes = await reader.readline()
+            ack: bytes = await reader_.readline()
             logger.debug(f"ack={ack.decode()}")
 
             logger.debug("waiting for VM response")
-            response: bytes = await reader.read()
+            response: bytes = await reader_.read()
 
             return response
 

From bf9ba51ff8c378c229b6c09d9178cae9de688bbe Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 25 Sep 2023 12:14:11 +0200
Subject: [PATCH 474/990] Fix: Code quality was not enforced

Problem: The code quality of the source code and the scripts was not verified automatically and could result into bugs or noisy commits from other contributors when fixing them.

Solution:

1. Add the tool `flake8` to the CI tests and adds code quality on the runtime Python code.

2. Enable Python tests on the runtime init as well.

3. Add `shellcheck` to check the quality of shell scripts.
---
 .github/workflows/code-quality.yml | 31 +++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
index 3e8b32e01..97c209941 100644
--- a/.github/workflows/code-quality.yml
+++ b/.github/workflows/code-quality.yml
@@ -3,7 +3,7 @@ name: Test code quality
 on: push
 
 jobs:
-  code-quality:
+  code-quality-python:
     runs-on: ubuntu-22.04
 
     steps:
@@ -26,16 +26,37 @@ jobs:
       - name: Test with Black
         run: |
           black --check ./vm_supervisor
+          black --check ./runtimes/aleph-debian-11-python/init1.py
 
       - name: Test with isort
         run: |
           isort --check-only --profile=black ./vm_supervisor
+          isort --check-only --profile=black ./runtimes/aleph-debian-11-python/init1.py
 
       - name: Test with MyPy
         run: |
           mypy --ignore-missing-imports ./vm_supervisor
-#          mypy --config-file ./mypy.ini ./vm_supervisor
+          mypy --ignore-missing-imports ./runtimes/aleph-debian-11-python/init1.py
 
-#      - name: Test with flake8
-#        run: |
-#          flake8 ./vm_supervisor
+      - name: Test with flake8
+        run: |
+          flake8 --extend-ignore E501 ./vm_supervisor
+          flake8 --extend-ignore E501,E402 ./runtimes/aleph-debian-11-python/init1.py
+
+  code-quality-shell:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
+      - name: Install required system packages only for Ubuntu Linux
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Run Shellcheck on all shell scripts
+        run: |
+          find ./ -type f -name "*.sh" -exec shellcheck {} \;

From 6fc4023cb8c0ca0c469fc5496c4f56d33c749aba Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 25 Sep 2023 12:15:20 +0200
Subject: [PATCH 475/990] Cleanup: Code quality issues reported by tooling.

This fixes a series of small code quality issues raised by our code quality tools.
---
 .github/workflows/code-quality.yml                |  2 +-
 docker/run_vm_supervisor.sh                       |  4 +++-
 examples/example_http_js/src/run.sh               |  3 +++
 packaging/extract_requirements.sh                 |  2 +-
 runtimes/aleph-debian-11-python/init0.sh          |  2 +-
 runtimes/aleph-debian-11-python/init1.py          |  2 +-
 runtimes/instance-rootfs/create-debian-11-disk.sh |  2 +-
 runtimes/instance-rootfs/create-debian-12-disk.sh |  2 +-
 vm_supervisor/conf.py                             |  2 +-
 vm_supervisor/metrics.py                          |  4 ++--
 vm_supervisor/storage.py                          |  1 -
 vm_supervisor/views/__init__.py                   |  2 +-
 vm_supervisor/views/operator.py                   |  6 ++----
 vm_supervisor/vm/firecracker/executable.py        | 15 +++++++--------
 vm_supervisor/vm/firecracker/program.py           |  2 +-
 15 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
index 97c209941..117ee7ef6 100644
--- a/.github/workflows/code-quality.yml
+++ b/.github/workflows/code-quality.yml
@@ -21,7 +21,7 @@ jobs:
 
       - name: Install required Python packages
         run: |
-          python3 -m pip install mypy pytest black isort
+          python3 -m pip install mypy pytest black isort flake8
 
       - name: Test with Black
         run: |
diff --git a/docker/run_vm_supervisor.sh b/docker/run_vm_supervisor.sh
index 7b076783c..5499dc324 100755
--- a/docker/run_vm_supervisor.sh
+++ b/docker/run_vm_supervisor.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -euf
+
 # Use Podman if installed, else use Docker
 if hash podman 2> /dev/null
 then
@@ -17,4 +19,4 @@ $DOCKER_COMMAND run -ti --rm \
   -v "$(pwd)/firecracker:/opt/aleph-vm/firecracker:ro" \
   --device /dev/kvm \
   -p 4020:4020 \
-  alephim/vm-supervisor-dev $@
+  alephim/vm-supervisor-dev "$@"
diff --git a/examples/example_http_js/src/run.sh b/examples/example_http_js/src/run.sh
index d56a2caf4..18430a26a 100755
--- a/examples/example_http_js/src/run.sh
+++ b/examples/example_http_js/src/run.sh
@@ -1,3 +1,6 @@
 #!/bin/sh
+
+set -euf
+
 cd /opt/code
 node /opt/code/server.js
diff --git a/packaging/extract_requirements.sh b/packaging/extract_requirements.sh
index aad72b7de..d443a0099 100755
--- a/packaging/extract_requirements.sh
+++ b/packaging/extract_requirements.sh
@@ -5,4 +5,4 @@ export DEBIAN_FRONTEND=noninteractive
 
 apt update
 apt --yes install /opt/packaging/target/aleph-vm.deb
-pip freeze > $1
+pip freeze > "$1"
diff --git a/runtimes/aleph-debian-11-python/init0.sh b/runtimes/aleph-debian-11-python/init0.sh
index 8eb1b62bf..75659b0b9 100644
--- a/runtimes/aleph-debian-11-python/init0.sh
+++ b/runtimes/aleph-debian-11-python/init0.sh
@@ -5,7 +5,7 @@ set -euf
 mount -t proc proc /proc -o nosuid,noexec,nodev
 
 log() {
-    echo "$(cat /proc/uptime | awk '{printf $1}')" '|S' "$@"
+    echo "$(awk '{print $1}' /proc/uptime)" '|S' "$@"
 }
 log "init0.sh is launching"
 
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index 006109aaa..26747a7b1 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -32,7 +32,7 @@
 logger.debug("Imports finished")
 
 __version__ = "2.0.0"
-ASGIApplication = NewType("ASGIApplication", Any)
+ASGIApplication = NewType("ASGIApplication", Any)  # type: ignore
 
 
 class Encoding(str, Enum):
diff --git a/runtimes/instance-rootfs/create-debian-11-disk.sh b/runtimes/instance-rootfs/create-debian-11-disk.sh
index 5d7b51fd4..1ee49c8b0 100755
--- a/runtimes/instance-rootfs/create-debian-11-disk.sh
+++ b/runtimes/instance-rootfs/create-debian-11-disk.sh
@@ -35,7 +35,7 @@ tar xvf "$IMAGE_NAME"
 
 # Mount first partition of Debian Image
 LOOPDISK=$(losetup --find --show $IMAGE_RAW_NAME)
-partx -u $LOOPDISK
+partx -u "$LOOPDISK"
 mount "$LOOPDISK"p1 "$MOUNT_ORIGIN_DIR"
 
 # Fix boot partition missing
diff --git a/runtimes/instance-rootfs/create-debian-12-disk.sh b/runtimes/instance-rootfs/create-debian-12-disk.sh
index 3078c5c26..cfa0130a5 100755
--- a/runtimes/instance-rootfs/create-debian-12-disk.sh
+++ b/runtimes/instance-rootfs/create-debian-12-disk.sh
@@ -35,7 +35,7 @@ tar xvf "$IMAGE_NAME"
 
 # Mount first partition of Debian Image
 LOOPDISK=$(losetup --find --show $IMAGE_RAW_NAME)
-partx -u $LOOPDISK
+partx -u "$LOOPDISK"
 mount "$LOOPDISK"p1 "$MOUNT_ORIGIN_DIR"
 
 # Fix boot partition missing
diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index a5d54cea4..bfa76e230 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -272,7 +272,7 @@ def check(self):
         assert isfile(self.FIRECRACKER_PATH), f"File not found {self.FIRECRACKER_PATH}"
         assert isfile(self.JAILER_PATH), f"File not found {self.JAILER_PATH}"
         assert isfile(self.LINUX_PATH), f"File not found {self.LINUX_PATH}"
-        assert self.NETWORK_INTERFACE, f"Network interface is not specified"
+        assert self.NETWORK_INTERFACE, "Network interface is not specified"
         assert self.CONNECTOR_URL.startswith(
             "http://"
         ) or self.CONNECTOR_URL.startswith("https://")
diff --git a/vm_supervisor/metrics.py b/vm_supervisor/metrics.py
index a5687bbb7..8f54aeba3 100644
--- a/vm_supervisor/metrics.py
+++ b/vm_supervisor/metrics.py
@@ -67,7 +67,7 @@ async def save_execution_data(execution_uuid: UUID, execution_data: str):
 
 async def save_record(record: ExecutionRecord):
     """Record the resource usage in database"""
-    session = Session()
+    session = Session()  # noqa: F821 undefined name 'Session'
     try:
         session.add(record)
         session.commit()
@@ -77,7 +77,7 @@ async def save_record(record: ExecutionRecord):
 
 async def get_execution_records() -> Iterable[ExecutionRecord]:
     """Get the execution records from the database."""
-    session = Session()
+    session = Session()  # noqa: F821 undefined name 'Session'
     try:
         return session.query(ExecutionRecord).all()
     finally:
diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 4d9db0d6b..51e71ee20 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -9,7 +9,6 @@
 import re
 import sys
 from datetime import datetime
-from enum import Enum
 from pathlib import Path
 from shutil import copy2, disk_usage, make_archive
 from typing import Union
diff --git a/vm_supervisor/views/__init__.py b/vm_supervisor/views/__init__.py
index bc034f21d..5165a5b92 100644
--- a/vm_supervisor/views/__init__.py
+++ b/vm_supervisor/views/__init__.py
@@ -219,7 +219,7 @@ async def update_allocations(request: web.Request):
     for execution in pool.get_persistent_executions():
         if execution.vm_hash not in allocations:
             vm_type = "instance" if execution.is_instance else "persistent program"
-            logger.info(f"Stopping %s %s", vm_type, execution.vm_hash)
+            logger.info("Stopping %s %s", vm_type, execution.vm_hash)
             await execution.stop()
             execution.persistent = False
 
diff --git a/vm_supervisor/views/operator.py b/vm_supervisor/views/operator.py
index e7d0cf616..6bbd4ec9d 100644
--- a/vm_supervisor/views/operator.py
+++ b/vm_supervisor/views/operator.py
@@ -1,4 +1,5 @@
 import asyncio
+import functools
 import json
 import logging
 from datetime import datetime, timedelta
@@ -19,9 +20,6 @@
 logger = logging.getLogger(__name__)
 
 
-import functools
-
-
 def is_token_still_valid(timestamp):
     """
     Checks if a token has exprired based on its timestamp
@@ -199,7 +197,7 @@ async def operate_stop(request: web.Request):
         execution.persistent = False
         return web.Response(status=200, body=f"Stopped VM with ref {vm_hash}")
     else:
-        return web.Response(status=200, body=f"Already stopped, nothing to do")
+        return web.Response(status=200, body="Already stopped, nothing to do")
 
 
 @require_jwk_authentication
diff --git a/vm_supervisor/vm/firecracker/executable.py b/vm_supervisor/vm/firecracker/executable.py
index b179624f2..0d3985d53 100644
--- a/vm_supervisor/vm/firecracker/executable.py
+++ b/vm_supervisor/vm/firecracker/executable.py
@@ -9,16 +9,10 @@
 from multiprocessing import Process, set_start_method
 from os.path import exists, isfile
 from pathlib import Path
-from typing import Any, Dict, Generic, List, Optional, TypeVar
+from typing import Dict, Generic, List, Optional, TypeVar
 
-from aleph_message.models import ItemHash
-
-psutil: Optional[Any]
-try:
-    import psutil  # type: ignore [no-redef]
-except ImportError:
-    psutil = None
 from aiohttp import ClientResponseError
+from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
 
 from firecracker.config import FirecrackerConfig
@@ -31,6 +25,11 @@
 from vm_supervisor.snapshots import CompressedDiskVolumeSnapshot
 from vm_supervisor.storage import get_volume_path
 
+try:
+    import psutil  # type: ignore [no-redef]
+except ImportError:
+    psutil = None
+
 logger = logging.getLogger(__name__)
 set_start_method("spawn")
 
diff --git a/vm_supervisor/vm/firecracker/program.py b/vm_supervisor/vm/firecracker/program.py
index b4a0153f0..9171bad8f 100644
--- a/vm_supervisor/vm/firecracker/program.py
+++ b/vm_supervisor/vm/firecracker/program.py
@@ -64,7 +64,7 @@ def read_input_data(path_to_data: Optional[Path]) -> Optional[bytes]:
         return None
 
     if os.path.getsize(path_to_data) > settings.MAX_DATA_ARCHIVE_SIZE:
-        raise FileTooLargeError(f"Data file too large to pass as an inline zip")
+        raise FileTooLargeError("Data file too large to pass as an inline zip")
 
     return path_to_data.read_bytes()
 

From f7c28d30af0b425b3bd818c77e41b9811c59c487 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 27 Jul 2023 22:10:20 +0200
Subject: [PATCH 476/990] Fix: Solved VM shutdown errors

---
 firecracker/microvm.py | 2 +-
 vm_supervisor/run.py   | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 5f656df2d..20325e868 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -81,7 +81,7 @@ class MicroVM:
     drives: List[Drive]
     init_timeout: float
     mounted_rootfs: Optional[Path] = None
-    _unix_socket: Server
+    _unix_socket: Optional[Server] = None
 
     @property
     def namespace_path(self):
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 934013cc3..714326213 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -16,6 +16,7 @@
 from .models import VmExecution
 from .pool import VmPool
 from .pubsub import PubSub
+from .utils import HostNotFoundError
 from .vm.firecracker.program import (
     FileTooLargeError,
     ResourceDownloadError,
@@ -75,6 +76,12 @@ async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
         raise HTTPInternalServerError(reason="Error during runtime initialisation")
+    except HostNotFoundError as error:
+        logger.exception(error)
+        pool.forget_vm(vm_hash=vm_hash)
+        raise HTTPInternalServerError(
+            reason="Error during vm initialisation, vm ping without response"
+        )
 
     if not execution.vm:
         raise ValueError("The VM has not been created")

From 1ff127f1e96dc2c2ed72eadef3c7441a3252b74d Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Fri, 28 Jul 2023 15:52:58 +0200
Subject: [PATCH 477/990] Fix: Fixed cloud-init refresh problems and improved
 way to create volume files.

---
 vm_supervisor/storage.py                 |  8 ++---
 vm_supervisor/vm/firecracker/instance.py | 42 +++++++++++++++++-------
 2 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/vm_supervisor/storage.py b/vm_supervisor/storage.py
index 51e71ee20..edc7ab5a7 100644
--- a/vm_supervisor/storage.py
+++ b/vm_supervisor/storage.py
@@ -193,9 +193,7 @@ async def create_ext4(path: Path, size_mib: int) -> bool:
         logger.debug(f"File already exists, skipping ext4 creation on {path}")
         return False
     tmp_path = f"{path}.tmp"
-    await run_in_subprocess(
-        ["dd", "if=/dev/zero", f"of={tmp_path}", "bs=1M", f"count={size_mib}"]
-    )
+    await run_in_subprocess(["fallocate", "-l", f"{size_mib}M", str(tmp_path)])
     await run_in_subprocess(["mkfs.ext4", tmp_path])
     await chown_to_jailman(Path(tmp_path))
     Path(tmp_path).rename(path)
@@ -213,9 +211,7 @@ async def create_volume_file(
         # Ensure that the parent directory exists
         path.parent.mkdir(exist_ok=True)
         # Create an empty file the right size
-        await run_in_subprocess(
-            ["dd", "if=/dev/zero", f"of={path}", "bs=1M", f"count={volume.size_mib}"]
-        )
+        await run_in_subprocess(["fallocate", "-l", f"{volume.size_mib}M", str(path)])
         await chown_to_jailman(path)
     return path
 
diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index 733345e7b..dbe9fd32a 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -1,4 +1,5 @@
 import asyncio
+import json
 import logging
 from pathlib import Path
 from tempfile import NamedTemporaryFile
@@ -189,6 +190,8 @@ def _encode_user_data(self) -> bytes:
             "disable_root": False,
             "ssh_pwauth": False,
             "ssh_authorized_keys": ssh_authorized_keys,
+            # Avoid the resize error because we already do it on the VM disk creation stage
+            "resize_rootfs": False,
         }
 
         cloud_config_header = "#cloud-config\n"
@@ -230,27 +233,42 @@ def _create_network_file(self) -> bytes:
             network, default_flow_style=False, sort_keys=False
         ).encode()
 
+    def _create_metadata_file(self) -> bytes:
+        """Creates metadata configuration file for cloud-init tool"""
+
+        metadata = {
+            "instance-id": f"iid-instance-{self.vm_id}",
+            "local-hostname": str(self.vm_hash),
+        }
+
+        return json.dumps(metadata).encode()
+
     async def _create_cloud_init_drive(self) -> Drive:
         """Creates the cloud-init volume to configure and setup the VM"""
 
         disk_image_path = settings.EXECUTION_ROOT / f"cloud-init-{self.vm_hash}.img"
 
-        with NamedTemporaryFile() as main_config_file:
+        with NamedTemporaryFile() as user_data_config_file:
             user_data = self._encode_user_data()
-            main_config_file.write(user_data)
-            main_config_file.flush()
+            user_data_config_file.write(user_data)
+            user_data_config_file.flush()
             with NamedTemporaryFile() as network_config_file:
                 network_config = self._create_network_file()
                 network_config_file.write(network_config)
                 network_config_file.flush()
-
-                await run_in_subprocess(
-                    [
-                        "cloud-localds",
-                        f"--network-config={network_config_file.name}",
-                        str(disk_image_path),
-                        main_config_file.name,
-                    ]
-                )
+                with NamedTemporaryFile() as metadata_config_file:
+                    metadata_config = self._create_metadata_file()
+                    metadata_config_file.write(metadata_config)
+                    metadata_config_file.flush()
+
+                    await run_in_subprocess(
+                        [
+                            "cloud-localds",
+                            f"--network-config={network_config_file.name}",
+                            str(disk_image_path),
+                            user_data_config_file.name,
+                            metadata_config_file.name,
+                        ]
+                    )
 
         return self.fvm.enable_drive(disk_image_path, read_only=True)

From f259a3cd515f4e2524c9946a07bcfde15d6e3278 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Mon, 31 Jul 2023 16:03:09 +0200
Subject: [PATCH 478/990] Fix: Ensure that one VM failed execution don't stop
 the allocation process.

---
 firecracker/microvm.py |  2 +-
 vm_supervisor/run.py   | 45 ++++++++++++++++++++++++++----------------
 2 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/firecracker/microvm.py b/firecracker/microvm.py
index 20325e868..4e827d1e7 100644
--- a/firecracker/microvm.py
+++ b/firecracker/microvm.py
@@ -467,7 +467,7 @@ async def teardown(self):
             await asyncio.sleep(1)
             root_fs = self.mounted_rootfs.name
             system(f"dmsetup remove {root_fs}")
-            if self.use_jailer:
+            if self.use_jailer and Path(self.jailer_path).is_dir():
                 shutil.rmtree(self.jailer_path)
 
         if self._unix_socket:
diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 714326213..9f6faada3 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -49,6 +49,7 @@ async def build_event_scope(event) -> Dict[str, Any]:
 
 
 async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
+    execution: Optional[VmExecution] = None
     message, original_message = await load_updated_message(vm_hash)
     pool.message_cache[vm_hash] = message
 
@@ -79,11 +80,8 @@ async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
     except HostNotFoundError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(
-            reason="Error during vm initialisation, vm ping without response"
-        )
 
-    if not execution.vm:
+    if not execution or execution.vm:
         raise ValueError("The VM has not been created")
 
     return execution
@@ -248,21 +246,28 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub):
             await execution.stop()
 
 
-async def start_persistent_vm(vm_hash: ItemHash, pubsub: PubSub) -> VmExecution:
+async def start_persistent_vm(
+    vm_hash: ItemHash, pubsub: PubSub
+) -> Optional[VmExecution]:
     execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
-    if not execution:
-        logger.info(f"Starting persistent virtual machine with id: {vm_hash}")
-        execution = await create_vm_execution(vm_hash=vm_hash)
-    # If the VM was already running in lambda mode, it should not expire
-    # as long as it is also scheduled as long-running
-    execution.persistent = True
-    execution.cancel_expiration()
+    try:
+        if not execution:
+            logger.info(f"Starting persistent virtual machine with id: {vm_hash}")
+            execution = await create_vm_execution(vm_hash=vm_hash)
+        # If the VM was already running in lambda mode, it should not expire
+        # as long as it is also scheduled as long-running
+        execution.persistent = True
+        execution.cancel_expiration()
+
+        await execution.becomes_ready()
 
-    await execution.becomes_ready()
+        if settings.WATCH_FOR_UPDATES:
+            execution.start_watching_for_updates(pubsub=pubsub)
 
-    if settings.WATCH_FOR_UPDATES:
-        execution.start_watching_for_updates(pubsub=pubsub)
+    # TODO: Handle all the exceptions, for now Always return a 200 code for now
+    except:
+        pass
 
     return execution
 
@@ -270,6 +275,12 @@ async def start_persistent_vm(vm_hash: ItemHash, pubsub: PubSub) -> VmExecution:
 async def stop_persistent_vm(vm_hash: ItemHash) -> Optional[VmExecution]:
     logger.info(f"Stopping persistent VM {vm_hash}")
     execution = await pool.get_running_vm(vm_hash)
-    if execution:
-        await execution.stop()
+
+    try:
+        if execution:
+            await execution.stop()
+    # TODO: Handle all the exceptions, for now Always return a 200 code for now
+    except:
+        pass
+
     return execution

From 8626a58b8f34a18ac65176328a79f9dea9993aeb Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Mon, 31 Jul 2023 17:32:19 +0200
Subject: [PATCH 479/990] Fix: Allow abort the process if it fails.

---
 vm_supervisor/run.py | 33 +++++++++++----------------------
 1 file changed, 11 insertions(+), 22 deletions(-)

diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index 9f6faada3..b27a30223 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -246,28 +246,21 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub):
             await execution.stop()
 
 
-async def start_persistent_vm(
-    vm_hash: ItemHash, pubsub: PubSub
-) -> Optional[VmExecution]:
+async def start_persistent_vm(vm_hash: ItemHash, pubsub: PubSub) -> VmExecution:
     execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
-    try:
-        if not execution:
-            logger.info(f"Starting persistent virtual machine with id: {vm_hash}")
-            execution = await create_vm_execution(vm_hash=vm_hash)
+    if not execution:
+        logger.info(f"Starting persistent virtual machine with id: {vm_hash}")
+        execution = await create_vm_execution(vm_hash=vm_hash)
         # If the VM was already running in lambda mode, it should not expire
         # as long as it is also scheduled as long-running
-        execution.persistent = True
-        execution.cancel_expiration()
+    execution.persistent = True
+    execution.cancel_expiration()
 
-        await execution.becomes_ready()
+    await execution.becomes_ready()
 
-        if settings.WATCH_FOR_UPDATES:
-            execution.start_watching_for_updates(pubsub=pubsub)
-
-    # TODO: Handle all the exceptions, for now Always return a 200 code for now
-    except:
-        pass
+    if settings.WATCH_FOR_UPDATES:
+        execution.start_watching_for_updates(pubsub=pubsub)
 
     return execution
 
@@ -276,11 +269,7 @@ async def stop_persistent_vm(vm_hash: ItemHash) -> Optional[VmExecution]:
     logger.info(f"Stopping persistent VM {vm_hash}")
     execution = await pool.get_running_vm(vm_hash)
 
-    try:
-        if execution:
-            await execution.stop()
-    # TODO: Handle all the exceptions, for now Always return a 200 code for now
-    except:
-        pass
+    if execution:
+        await execution.stop()
 
     return execution

From 7ca664023c78261f82d6a1ee5f7354f8243315d9 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Mon, 31 Jul 2023 19:11:10 +0200
Subject: [PATCH 480/990] Fix: Fixed instances start issue.

---
 vm_supervisor/run.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index b27a30223..cc7ed7430 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -49,7 +49,6 @@ async def build_event_scope(event) -> Dict[str, Any]:
 
 
 async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
-    execution: Optional[VmExecution] = None
     message, original_message = await load_updated_message(vm_hash)
     pool.message_cache[vm_hash] = message
 
@@ -81,7 +80,7 @@ async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
 
-    if not execution or execution.vm:
+    if not execution.vm:
         raise ValueError("The VM has not been created")
 
     return execution
@@ -252,8 +251,9 @@ async def start_persistent_vm(vm_hash: ItemHash, pubsub: PubSub) -> VmExecution:
     if not execution:
         logger.info(f"Starting persistent virtual machine with id: {vm_hash}")
         execution = await create_vm_execution(vm_hash=vm_hash)
-        # If the VM was already running in lambda mode, it should not expire
-        # as long as it is also scheduled as long-running
+
+    # If the VM was already running in lambda mode, it should not expire
+    # as long as it is also scheduled as long-running
     execution.persistent = True
     execution.cancel_expiration()
 

From 7b65528de277ad92a48a206243be2408e537b849 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 21 Sep 2023 11:14:45 +0200
Subject: [PATCH 481/990] Fix: Large hostname issue.

---
 vm_supervisor/vm/firecracker/instance.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index dbe9fd32a..be4f886ba 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -1,4 +1,5 @@
 import asyncio
+import base64
 import json
 import logging
 from pathlib import Path
@@ -236,9 +237,11 @@ def _create_network_file(self) -> bytes:
     def _create_metadata_file(self) -> bytes:
         """Creates metadata configuration file for cloud-init tool"""
 
+        hostname = base64.b32encode(self.vm_hash).decode().strip("=").lower()
+
         metadata = {
             "instance-id": f"iid-instance-{self.vm_id}",
-            "local-hostname": str(self.vm_hash),
+            "local-hostname": hostname,
         }
 
         return json.dumps(metadata).encode()

From f253ac7328faeec9fa8b9d974d4e910500cf5a61 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 25 Sep 2023 14:32:31 +0200
Subject: [PATCH 482/990] Fix: Errors in allocation exited entire scheduling

Problem: An error in the allocation of one persistent VM or instance crashed the allocation of all remaining VMs.

Solution: Handle errors in the creation of VMs as not to stop the entire process. Return the allocations that were successful and those who were not.

Split the exception handling between direct HTTP calls and allocations for distinct handling.
---
 vm_supervisor/run.py                    | 30 ++++++++-
 vm_supervisor/views/__init__.py         | 85 ++++++++++++++++++++-----
 vm_supervisor/vm/firecracker/program.py | 14 +---
 3 files changed, 96 insertions(+), 33 deletions(-)

diff --git a/vm_supervisor/run.py b/vm_supervisor/run.py
index cc7ed7430..4d51fc238 100644
--- a/vm_supervisor/run.py
+++ b/vm_supervisor/run.py
@@ -79,6 +79,7 @@ async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
     except HostNotFoundError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
+        raise HTTPInternalServerError(reason="Host did not respond to ping")
 
     if not execution.vm:
         raise ValueError("The VM has not been created")
@@ -86,6 +87,29 @@ async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
     return execution
 
 
+async def create_vm_execution_or_raise_http_error(vm_hash: ItemHash) -> VmExecution:
+    try:
+        return await create_vm_execution(vm_hash=vm_hash)
+    except ResourceDownloadError as error:
+        logger.exception(error)
+        pool.forget_vm(vm_hash=vm_hash)
+        raise HTTPBadRequest(reason="Code, runtime or data not available")
+    except FileTooLargeError as error:
+        raise HTTPInternalServerError(reason=error.args[0])
+    except VmSetupError as error:
+        logger.exception(error)
+        pool.forget_vm(vm_hash=vm_hash)
+        raise HTTPInternalServerError(reason="Error during vm initialisation")
+    except MicroVMFailedInit as error:
+        logger.exception(error)
+        pool.forget_vm(vm_hash=vm_hash)
+        raise HTTPInternalServerError(reason="Error during runtime initialisation")
+    except HostNotFoundError as error:
+        logger.exception(error)
+        pool.forget_vm(vm_hash=vm_hash)
+        raise HTTPInternalServerError(reason="Host did not respond to ping")
+
+
 async def run_code_on_request(
     vm_hash: ItemHash, path: str, request: web.Request
 ) -> web.Response:
@@ -96,7 +120,7 @@ async def run_code_on_request(
     execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
-        execution = await create_vm_execution(vm_hash=vm_hash)
+        execution = await create_vm_execution_or_raise_http_error(vm_hash=vm_hash)
 
     logger.debug(f"Using vm={execution.vm_id}")
 
@@ -190,7 +214,7 @@ async def run_code_on_request(
         if settings.REUSE_TIMEOUT > 0:
             if settings.WATCH_FOR_UPDATES:
                 execution.start_watching_for_updates(pubsub=request.app["pubsub"])
-            execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT)
+            _ = execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT)
         else:
             await execution.stop()
 
@@ -203,7 +227,7 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub):
     execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
-        execution = await create_vm_execution(vm_hash=vm_hash)
+        execution = await create_vm_execution_or_raise_http_error(vm_hash=vm_hash)
 
     logger.debug(f"Using vm={execution.vm_id}")
 
diff --git a/vm_supervisor/views/__init__.py b/vm_supervisor/views/__init__.py
index 5165a5b92..8d0cd0c38 100644
--- a/vm_supervisor/views/__init__.py
+++ b/vm_supervisor/views/__init__.py
@@ -3,15 +3,17 @@
 from hashlib import sha256
 from pathlib import Path
 from string import Template
-from typing import Awaitable, Optional
+from typing import Awaitable, Dict, Optional
 
 import aiodns
 import aiohttp
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPNotFound
+from aleph_message.exceptions import UnknownHashError
 from aleph_message.models import ItemHash
 from pydantic import ValidationError
 
+from firecracker.microvm import MicroVMFailedInit
 from packaging.version import InvalidVersion, Version
 from vm_supervisor import status
 from vm_supervisor.conf import settings
@@ -19,8 +21,15 @@
 from vm_supervisor.pubsub import PubSub
 from vm_supervisor.resources import Allocation
 from vm_supervisor.run import pool, run_code_on_request, start_persistent_vm
-from vm_supervisor.utils import b32_to_b16, dumps_for_json, get_ref_from_dns
+from vm_supervisor.utils import (
+    HostNotFoundError,
+    b32_to_b16,
+    dumps_for_json,
+    get_ref_from_dns,
+)
 from vm_supervisor.version import __version__
+from vm_supervisor.vm.firecracker.executable import ResourceDownloadError, VmSetupError
+from vm_supervisor.vm.firecracker.program import FileTooLargeError
 
 logger = logging.getLogger(__name__)
 
@@ -201,20 +210,7 @@ async def update_allocations(request: web.Request):
 
     pubsub: PubSub = request.app["pubsub"]
 
-    # Start VMs
-    for vm_hash in allocation.persistent_vms:
-        vm_hash = ItemHash(vm_hash)
-        logger.info(f"Starting long running VM {vm_hash}")
-        await start_persistent_vm(vm_hash, pubsub)
-
-    # Start Instances
-    for instance_hash in allocation.instances:
-        instance_hash = ItemHash(instance_hash)
-        logger.info(f"Starting instance {instance_hash}")
-        await start_persistent_vm(instance_hash, pubsub)
-
-    # Stop unscheduled persistent programs and instances.
-    # Instances are also marked with persistent = True.
+    # First free resources from persistent programs and instances that are not scheduled anymore.
     allocations = allocation.persistent_vms | allocation.instances
     for execution in pool.get_persistent_executions():
         if execution.vm_hash not in allocations:
@@ -223,10 +219,65 @@ async def update_allocations(request: web.Request):
             await execution.stop()
             execution.persistent = False
 
+    # Second start persistent VMs and instances sequentially to limit resource usage.
+
+    # Exceptions that can be raised when starting a VM:
+    vm_creation_exceptions = (
+        UnknownHashError,
+        ResourceDownloadError,
+        FileTooLargeError,
+        VmSetupError,
+        MicroVMFailedInit,
+        HostNotFoundError,
+    )
+
+    scheduling_errors: Dict[ItemHash, Exception] = {}
+
+    # Schedule the start of persistent VMs:
+    for vm_hash in allocation.persistent_vms:
+        try:
+            logger.info(f"Starting long running VM '{vm_hash}'")
+            vm_hash = ItemHash(vm_hash)
+            await start_persistent_vm(vm_hash, pubsub)
+        except vm_creation_exceptions as error:
+            logger.exception(error)
+            scheduling_errors[vm_hash] = error
+
+    # Schedule the start of instances:
+    for instance_hash in allocation.instances:
+        logger.info(f"Starting instance '{instance_hash}'")
+        try:
+            instance_hash = ItemHash(instance_hash)
+            await start_persistent_vm(instance_hash, pubsub)
+        except vm_creation_exceptions as error:
+            logger.exception(error)
+            scheduling_errors[instance_hash] = error
+
     # Log unsupported features
     if allocation.on_demand_vms:
         logger.warning("Not supported yet: 'allocation.on_demand_vms'")
     if allocation.jobs:
         logger.warning("Not supported yet: 'allocation.on_demand_vms'")
 
-    return web.json_response(data={"success": True})
+    failing = set(scheduling_errors.keys())
+    successful = allocations - failing
+
+    status_code: int
+    if not failing:
+        status_code = 200  # OK
+    elif not successful:
+        status_code = 503  # Service Unavailable
+    else:
+        status_code = 207  # Multi-Status
+
+    return web.json_response(
+        data={
+            "success": not failing,
+            "successful": list(successful),
+            "failing": list(failing),
+            "errors": {
+                vm_hash: repr(error) for vm_hash, error in scheduling_errors.items()
+            },
+        },
+        status=status_code,
+    )
diff --git a/vm_supervisor/vm/firecracker/program.py b/vm_supervisor/vm/firecracker/program.py
index 9171bad8f..407a1dcc6 100644
--- a/vm_supervisor/vm/firecracker/program.py
+++ b/vm_supervisor/vm/firecracker/program.py
@@ -34,6 +34,7 @@
 from .executable import (
     AlephFirecrackerExecutable,
     AlephFirecrackerResources,
+    ResourceDownloadError,
     VmInitNotConnected,
     VmSetupError,
     Volume,
@@ -46,19 +47,6 @@ class FileTooLargeError(Exception):
     pass
 
 
-class ResourceDownloadError(ClientResponseError):
-    """An error occurred while downloading a VM resource file"""
-
-    def __init__(self, error: ClientResponseError):
-        super().__init__(
-            request_info=error.request_info,
-            history=error.history,
-            status=error.status,
-            message=error.message,
-            headers=error.headers,
-        )
-
-
 def read_input_data(path_to_data: Optional[Path]) -> Optional[bytes]:
     if not path_to_data:
         return None

From d782e4f50dad8b6a9b9e834ffdc99511d2713ee4 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 6 Jul 2023 10:55:28 +0200
Subject: [PATCH 483/990] Fix: ASGI processes did not support lifetime events

ASGI specification https://asgi.readthedocs.io/en/latest/specs/lifespan.html

Fixes #293
Replaces #294
---
 examples/example_fastapi/main.py         | 16 ++++++
 runtimes/aleph-debian-11-python/init1.py | 65 +++++++++++++++++++++---
 vm_supervisor/__main__.py                |  1 +
 vm_supervisor/status.py                  |  8 +++
 vm_supervisor/views/__init__.py          |  1 +
 5 files changed, 84 insertions(+), 7 deletions(-)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index d278b70aa..e2931db6f 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -32,6 +32,14 @@
 app = AlephApp(http_app=http_app)
 cache = VmCache()
 
+startup_lifespan_executed: bool = False
+
+
+@app.on_event("startup")
+async def startup_event():
+    global startup_lifespan_executed
+    startup_lifespan_executed = True
+
 
 @app.get("/")
 async def index():
@@ -59,6 +67,14 @@ async def index():
     }
 
 
+@app.get("/lifespan")
+async def check_lifespan():
+    """
+    Check that ASGI lifespan startup signal has been received
+    """
+    return {"Lifetime": startup_lifespan_executed}
+
+
 @app.get("/environ")
 async def environ() -> Dict[str, str]:
     """List environment variables"""
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index 26747a7b1..01127d09e 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -24,7 +24,18 @@
 from io import StringIO
 from os import system
 from shutil import make_archive
-from typing import Any, AsyncIterable, Dict, List, NewType, Optional, Tuple, Union, cast
+from typing import (
+    Any,
+    AsyncIterable,
+    Dict,
+    List,
+    Literal,
+    NewType,
+    Optional,
+    Tuple,
+    Union,
+    cast,
+)
 
 import aiohttp
 import msgpack
@@ -189,7 +200,40 @@ def setup_volumes(volumes: List[Volume]):
     system("mount")
 
 
-def setup_code_asgi(
+async def wait_for_lifespan_event_completion(
+    application: ASGIApplication, event: Union[Literal["startup", "shutdown"]]
+):
+    """
+    Send the startup lifespan signal to the ASGI app.
+    Specification: https://asgi.readthedocs.io/en/latest/specs/lifespan.html
+    """
+
+    lifespan_completion = asyncio.Event()
+
+    async def receive():
+        return {
+            "type": f"lifespan.{event}",
+        }
+
+    async def send(response: Dict):
+        response_type = response.get("type")
+        if response_type == f"lifespan.{event}.complete":
+            lifespan_completion.set()
+            return
+        else:
+            logger.warning(f"Unexpected response to {event}: {response_type}")
+
+    while not lifespan_completion.is_set():
+        await application(
+            scope={
+                "type": "lifespan",
+            },
+            receive=receive,
+            send=send,
+        )
+
+
+async def setup_code_asgi(
     code: bytes, encoding: Encoding, entrypoint: str
 ) -> ASGIApplication:
     # Allow importing packages from /opt/packages
@@ -225,6 +269,7 @@ def setup_code_asgi(
         app = locals[entrypoint]
     else:
         raise ValueError(f"Unknown encoding '{encoding}'")
+    await wait_for_lifespan_event_completion(application=app, event="startup")
     return ASGIApplication(app)
 
 
@@ -260,14 +305,16 @@ def setup_code_executable(
     return process
 
 
-def setup_code(
+async def setup_code(
     code: bytes,
     encoding: Encoding,
     entrypoint: str,
     interface: Interface,
 ) -> Union[ASGIApplication, subprocess.Popen]:
     if interface == Interface.asgi:
-        return setup_code_asgi(code=code, encoding=encoding, entrypoint=entrypoint)
+        return await setup_code_asgi(
+            code=code, encoding=encoding, entrypoint=entrypoint
+        )
     elif interface == Interface.executable:
         return setup_code_executable(
             code=code, encoding=encoding, entrypoint=entrypoint
@@ -284,7 +331,7 @@ async def run_python_code_http(
         # Execute in the same process, saves ~20ms than a subprocess
 
         # The body should not be part of the ASGI scope itself
-        request_body: bytes = scope.pop("body")
+        scope_body: bytes = scope.pop("body")
 
         async def receive():
             type_ = (
@@ -292,7 +339,7 @@ async def receive():
                 if scope["type"] in ("http", "websocket")
                 else "aleph.message"
             )
-            return {"type": type_, "body": request_body, "more_body": False}
+            return {"type": type_, "body": scope_body, "more_body": False}
 
         send_queue: asyncio.Queue = asyncio.Queue()
 
@@ -402,6 +449,10 @@ async def process_instruction(
             application.terminate()
             logger.debug("Application terminated")
             # application.communicate()
+        else:
+            await wait_for_lifespan_event_completion(
+                application=application, event="shutdown"
+            )
         yield b"STOP\n"
         logger.debug("Supervisor informed of halt")
         raise ShutdownException
@@ -521,7 +572,7 @@ async def main() -> None:
     setup_system(config)
 
     try:
-        app: Union[ASGIApplication, subprocess.Popen] = setup_code(
+        app: Union[ASGIApplication, subprocess.Popen] = await setup_code(
             config.code, config.encoding, config.entrypoint, config.interface
         )
         client.send(msgpack.dumps({"success": True}))
diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index b20d32e1a..2f78d9471 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -203,6 +203,7 @@ async def fake_read() -> bytes:
     settings.REUSE_TIMEOUT = 0.1
     for path in (
         "/",
+        "/lifespan",
         "/environ",
         "/messages",
         "/internet",
diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py
index 4f4cf5e79..aaaeda5a2 100644
--- a/vm_supervisor/status.py
+++ b/vm_supervisor/status.py
@@ -30,6 +30,14 @@ async def check_index(session: ClientSession) -> bool:
         return False
 
 
+async def check_lifespan(session: ClientSession) -> bool:
+    try:
+        result: Dict = await get_json_from_vm(session, "/lifespan")
+        return result["Lifetime"] is True
+    except ClientResponseError:
+        return False
+
+
 async def check_environ(session: ClientSession) -> bool:
     try:
         result: Dict = await get_json_from_vm(session, "/environ")
diff --git a/vm_supervisor/views/__init__.py b/vm_supervisor/views/__init__.py
index 8d0cd0c38..fbf385a62 100644
--- a/vm_supervisor/views/__init__.py
+++ b/vm_supervisor/views/__init__.py
@@ -149,6 +149,7 @@ async def status_check_fastapi(request: web.Request):
     async with aiohttp.ClientSession() as session:
         result = {
             "index": await status.check_index(session),
+            "lifespan": await status.check_lifespan(session),
             "environ": await status.check_environ(session),
             "messages": await status.check_messages(session),
             "dns": await status.check_dns(session),

From ea233547f06c596048f1447d7c90039ac356678f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 19 Sep 2023 11:27:04 +0200
Subject: [PATCH 484/990] Fix: Do not check for lifespan yet

Lifespan is a new feature that requires a new runtime to be deployed before it can be enforced.
---
 vm_supervisor/views/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/vm_supervisor/views/__init__.py b/vm_supervisor/views/__init__.py
index fbf385a62..2f0c89bc7 100644
--- a/vm_supervisor/views/__init__.py
+++ b/vm_supervisor/views/__init__.py
@@ -149,7 +149,8 @@ async def status_check_fastapi(request: web.Request):
     async with aiohttp.ClientSession() as session:
         result = {
             "index": await status.check_index(session),
-            "lifespan": await status.check_lifespan(session),
+            # TODO: lifespan is a new feature that requires a new runtime to be deployed
+            # "lifespan": await status.check_lifespan(session),
             "environ": await status.check_environ(session),
             "messages": await status.check_messages(session),
             "dns": await status.check_dns(session),

From f73068de7c6dd9e154ebe6433854a7e0882b53ae Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 11 Jul 2023 12:32:55 +0200
Subject: [PATCH 485/990] Fix: Program runtime could use newer Debian 12

---
 .../create_disk_image.sh                      | 102 ++++++++++++++++++
 runtimes/aleph-debian-12-python/init0.sh      |  58 ++++++++++
 runtimes/aleph-debian-12-python/init1.py      |   1 +
 runtimes/aleph-debian-12-python/loading.html  |   1 +
 .../aleph-debian-12-python/update_inits.sh    |   1 +
 5 files changed, 163 insertions(+)
 create mode 100755 runtimes/aleph-debian-12-python/create_disk_image.sh
 create mode 100644 runtimes/aleph-debian-12-python/init0.sh
 create mode 120000 runtimes/aleph-debian-12-python/init1.py
 create mode 120000 runtimes/aleph-debian-12-python/loading.html
 create mode 120000 runtimes/aleph-debian-12-python/update_inits.sh

diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
new file mode 100755
index 000000000..6f1d37348
--- /dev/null
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -0,0 +1,102 @@
+#!/bin/sh
+
+rm ./rootfs.squashfs
+
+set -euf
+
+rm -fr ./rootfs
+mkdir ./rootfs
+
+debootstrap --variant=minbase bookworm ./rootfs http://deb.debian.org/debian/
+
+chroot ./rootfs /bin/sh <> /etc/ssh/sshd_config
+echo "PasswordAuthentication no" >> /etc/ssh/sshd_config
+echo "ChallengeResponseAuthentication no" >> /etc/ssh/sshd_config
+echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
+
+mkdir -p /overlay
+
+# Set up a login terminal on the serial console (ttyS0):
+ln -s agetty /etc/init.d/agetty.ttyS0
+echo ttyS0 > /etc/securetty
+EOT
+
+
+# Generate SSH host keys
+#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
+#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key
+#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key
+#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ed25519 -f /etc/ssh/ssh_host_ed25519_key
+
+cat < ./rootfs/etc/inittab
+# /etc/inittab
+
+::sysinit:/sbin/init sysinit
+::sysinit:/sbin/init boot
+::wait:/sbin/init default
+
+# Set up a couple of getty's
+tty1::respawn:/sbin/getty 38400 tty1
+tty2::respawn:/sbin/getty 38400 tty2
+tty3::respawn:/sbin/getty 38400 tty3
+tty4::respawn:/sbin/getty 38400 tty4
+tty5::respawn:/sbin/getty 38400 tty5
+tty6::respawn:/sbin/getty 38400 tty6
+
+# Put a getty on the serial port
+ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100
+
+# Stuff to do for the 3-finger salute
+::ctrlaltdel:/sbin/reboot
+
+# Stuff to do before rebooting
+::shutdown:/sbin/init shutdown
+EOT
+
+# Reduce size
+rm -fr ./rootfs/root/.cache
+rm -fr ./rootfs/var/cache
+mkdir -p ./rootfs/var/cache/apt/archives/partial
+rm -fr ./rootfs/usr/share/doc
+rm -fr ./rootfs/usr/share/man
+rm -fr ./rootfs/var/lib/apt/lists/
+
+# Custom init
+cp ./init0.sh ./rootfs/sbin/init
+cp ./init1.py ./rootfs/root/init1.py
+cp ./loading.html ./rootfs/root/loading.html
+chmod +x ./rootfs/sbin/init
+chmod +x ./rootfs/root/init1.py
+
+mksquashfs ./rootfs/ ./rootfs.squashfs
diff --git a/runtimes/aleph-debian-12-python/init0.sh b/runtimes/aleph-debian-12-python/init0.sh
new file mode 100644
index 000000000..7acdca08f
--- /dev/null
+++ b/runtimes/aleph-debian-12-python/init0.sh
@@ -0,0 +1,58 @@
+#!/bin/sh
+
+set -euf
+
+mount -t proc proc /proc -o nosuid,noexec,nodev
+
+log() {
+    echo "$(awk '{print $1}' /proc/uptime)" '|S' "$@"
+}
+log "init0.sh is launching"
+
+# Switch root from read-only ext4 to to read-write overlay
+mkdir -p /overlay
+/bin/mount -t tmpfs -o noatime,mode=0755 tmpfs /overlay
+mkdir -p /overlay/root /overlay/work
+/bin/mount -o noatime,lowerdir=/,upperdir=/overlay/root,workdir=/overlay/work -t overlay "overlayfs:/overlay/root" /mnt
+mkdir -p /mnt/rom
+pivot_root /mnt /mnt/rom
+
+mount --move /rom/proc /proc
+mount --move /rom/dev /dev
+
+mkdir -p /dev/pts
+mkdir -p /dev/shm
+
+mount -t sysfs sys /sys -o nosuid,noexec,nodev
+mount -t tmpfs run /run -o mode=0755,nosuid,nodev
+#mount -t devtmpfs dev /dev -o mode=0755,nosuid
+mount -t devpts devpts /dev/pts -o mode=0620,gid=5,nosuid,noexec
+mount -t tmpfs shm /dev/shm -omode=1777,nosuid,nodev
+
+# Required by Docker
+cgroupfs-mount
+update-alternatives --set iptables /usr/sbin/iptables-legacy
+update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+
+# Enable the following to force the storage driver used by Docker.
+# See https://docs.docker.com/storage/storagedriver/select-storage-driver/
+#echo '{\n"storage-driver": "overlay2"\n}\n' > /etc/docker/daemon.json
+
+# List block devices
+lsblk
+
+#cat /proc/sys/kernel/random/entropy_avail
+
+# TODO: Move in init1
+mkdir -p /run/sshd
+/usr/sbin/sshd &
+log "SSH UP"
+
+log "Setup socat"
+socat UNIX-LISTEN:/tmp/socat-socket,fork,reuseaddr VSOCK-CONNECT:2:53 &
+log "Socat ready"
+
+export PYTHONPATH=/opt/aleph/libs
+
+# Replace this script with the manager
+exec /root/init1.py
diff --git a/runtimes/aleph-debian-12-python/init1.py b/runtimes/aleph-debian-12-python/init1.py
new file mode 120000
index 000000000..7f48acafa
--- /dev/null
+++ b/runtimes/aleph-debian-12-python/init1.py
@@ -0,0 +1 @@
+../aleph-debian-11-python/init1.py
\ No newline at end of file
diff --git a/runtimes/aleph-debian-12-python/loading.html b/runtimes/aleph-debian-12-python/loading.html
new file mode 120000
index 000000000..926fba036
--- /dev/null
+++ b/runtimes/aleph-debian-12-python/loading.html
@@ -0,0 +1 @@
+../aleph-debian-11-python/loading.html
\ No newline at end of file
diff --git a/runtimes/aleph-debian-12-python/update_inits.sh b/runtimes/aleph-debian-12-python/update_inits.sh
new file mode 120000
index 000000000..757431761
--- /dev/null
+++ b/runtimes/aleph-debian-12-python/update_inits.sh
@@ -0,0 +1 @@
+../aleph-debian-11-python/update_inits.sh
\ No newline at end of file

From e8ff4e30df0fe0351fe38195ded7a61377583620 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 18 Sep 2023 15:33:39 +0200
Subject: [PATCH 486/990] Use Debian 12 by default in CI

---
 .github/workflows/build-deb-package.yml       | 42 ++++++++++++++++++-
 .github/workflows/test-build-examples.yml     |  2 +-
 .../workflows/test-new-runtime-examples.yml   | 20 ++++-----
 .../create_disk_image.sh                      |  7 ----
 4 files changed, 52 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index ef9d04b67..57ee24a71 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -23,6 +23,26 @@ jobs:
           name: aleph-vm.debian-11.deb
           path: packaging/target/aleph-vm.debian-11.deb
 
+  build_deb_debian_12:
+    name: "Build Debian Package"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # Fetch the whole history for all tags and branches (required for aleph.__version__)
+          fetch-depth: 0
+
+      - run: |
+          cd packaging && make all-podman-debian-12 && cd ..
+          ls packaging/target
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: aleph-vm.debian-12.deb
+          path: packaging/target/aleph-vm.debian-12.deb
+
   build_deb_ubuntu_22_04:
     name: "Build Ubuntu Package"
     runs-on: ubuntu-latest
@@ -43,7 +63,7 @@ jobs:
           name: aleph-vm.ubuntu-22.04.deb
           path: packaging/target/aleph-vm.ubuntu-22.04.deb
 
-  build_rootfs:
+  build_rootfs_debian_11:
     name: "Build runtime aleph-debian-11-python"
     runs-on: ubuntu-latest
 
@@ -64,6 +84,26 @@ jobs:
           name: aleph-debian-11-python.squashfs
           path: runtimes/aleph-debian-11-python/rootfs.squashfs
 
+  build_rootfs_debian_12:
+    name: "Build runtime aleph-debian-12-python"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
+      - run: |
+          sudo apt update
+          sudo apt install -y debootstrap
+          cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: aleph-debian-12-python.squashfs
+          path: runtimes/aleph-debian-12-python/rootfs.squashfs
 
   build_example_venv_volume:
     name: "Build example squashfs volume using Docker"
diff --git a/.github/workflows/test-build-examples.yml b/.github/workflows/test-build-examples.yml
index 220b579f4..66e382d49 100644
--- a/.github/workflows/test-build-examples.yml
+++ b/.github/workflows/test-build-examples.yml
@@ -25,7 +25,7 @@ jobs:
           sudo chown $(whoami) /opt/packages
 
       - run: |
-          pip3 install aleph-client
+          pip3 install aleph-sdk-python
 
       - run: |
           ls -la
diff --git a/.github/workflows/test-new-runtime-examples.yml b/.github/workflows/test-new-runtime-examples.yml
index 43f77f8c2..3b2982577 100644
--- a/.github/workflows/test-new-runtime-examples.yml
+++ b/.github/workflows/test-new-runtime-examples.yml
@@ -3,8 +3,8 @@ on:
   push
 
 jobs:
-  run_debian_11:
-    name: "Test new runtime on Droplet with Debian 11"
+  run_debian_12:
+    name: "Test new runtime on Droplet with Debian 12"
     runs-on: ubuntu-latest
     concurrency: droplet-aleph-vm-runtime
 
@@ -35,7 +35,7 @@ jobs:
       - name: Create the Droplet
         run: |
           doctl compute droplet create \
-          --image debian-11-x64 \
+          --image debian-12-x64 \
           --size c-2 \
           --region fra1 \
           --vpc-uuid 992896c8-c089-4da3-9288-f81e28c095a4 \
@@ -47,16 +47,16 @@ jobs:
         run: |
           sudo apt update
           sudo apt install -y debootstrap
-          cd runtimes/aleph-debian-11-python && sudo ./create_disk_image.sh && cd ../..
+          cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
 
       - uses: actions/upload-artifact@v3
         with:
-          name: aleph-debian-11-python.squashfs
-          path: runtimes/aleph-debian-11-python/rootfs.squashfs
+          name: aleph-debian-12-python.squashfs
+          path: runtimes/aleph-debian-12-python/rootfs.squashfs
 
       - name: Build Debian Package
         run: |
-          cd packaging && make all-podman-debian-11 && cd ..
+          cd packaging && make all-podman-debian-12 && cd ..
           ls packaging/target
 
       - name: Wait for the system to setup and boot
@@ -68,7 +68,7 @@ jobs:
         run: |
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
-          scp runtimes/aleph-debian-11-python/rootfs.squashfs root@${DROPLET_IPV4}:/opt
+          scp runtimes/aleph-debian-12-python/rootfs.squashfs root@${DROPLET_IPV4}:/opt
 
       - name: Install Aleph-VM on the Droplet
         run: |
@@ -80,9 +80,9 @@ jobs:
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y docker.io apparmor-profiles"
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
           
-          scp packaging/target/aleph-vm.debian-11.deb root@${DROPLET_IPV4}:/opt
+          scp packaging/target/aleph-vm.debian-12.deb root@${DROPLET_IPV4}:/opt
           scp -pr ./examples root@${DROPLET_IPV4}:/opt/
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt -o DPkg::Lock::Timeout=60 install -y /opt/aleph-vm.debian-11.deb"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt -o DPkg::Lock::Timeout=60 install -y /opt/aleph-vm.debian-12.deb"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_FAKE_DATA_PROGRAM=/opt/examples/example_fastapi >> /etc/aleph-vm/supervisor.env"
           ssh root@${DROPLET_IPV4} "echo ALEPH_VM_FAKE_DATA_RUNTIME=/opt/rootfs.squashfs >> /etc/aleph-vm/supervisor.env"
diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index 6f1d37348..1d969f69c 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -52,13 +52,6 @@ ln -s agetty /etc/init.d/agetty.ttyS0
 echo ttyS0 > /etc/securetty
 EOT
 
-
-# Generate SSH host keys
-#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
-#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key
-#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key
-#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ed25519 -f /etc/ssh/ssh_host_ed25519_key
-
 cat < ./rootfs/etc/inittab
 # /etc/inittab
 

From 9117b016131f8544963d857e69897da2128e5ae5 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 25 Sep 2023 18:26:35 +0200
Subject: [PATCH 487/990] Fix: Sentry SDK was absent from most installs

Problem: Setting a value for `SENTRY_DNS` often did not work because the Sentry SDK was not installed.

The Sentry SDK has a very limited number of dependencies.

Solution: Install the Sentry SDK by default from the Debian packages.
---
 packaging/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index e5bc51eca..a5a6c5811 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -17,7 +17,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.0' 'jwskate==0.8.0' 'eth-account==0.9.0'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.0' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo

From 9157805cfd5caa600c71bf7736a6ce718e062bf6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 25 Aug 2023 17:20:15 +0200
Subject: [PATCH 488/990] Fix: The Kubo IPFS service was not hardened

Systemd provides hardening tools to improve the security of the host.

This uses the hardened service file from the Kubo repository.
---
 packaging/aleph-vm/DEBIAN/postinst            |  8 ++++
 packaging/aleph-vm/DEBIAN/postrm              |  5 +++
 packaging/aleph-vm/DEBIAN/preinst             |  6 +++
 .../aleph-vm/etc/systemd/system/ipfs.service  | 39 +++++++++++++++++--
 4 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/packaging/aleph-vm/DEBIAN/postinst b/packaging/aleph-vm/DEBIAN/postinst
index 99af91c2d..913e8c411 100755
--- a/packaging/aleph-vm/DEBIAN/postinst
+++ b/packaging/aleph-vm/DEBIAN/postinst
@@ -9,10 +9,18 @@ rm -fr /srv/jailer  # Upgrade from < 0.1.11
 rm -fr /tmp/aleph   # Upgrade from < 0.1.11
 mkdir -p /var/lib/aleph/vm/jailer
 
+# Create the IPFS directory if it does not exist
+if [ ! -d "/var/lib/ipfs" ]; then
+    mkdir -p /var/lib/ipfs
+    # Set appropriate permissions if needed
+    chown ipfs:ipfs /var/lib/ipfs
+fi
+
 # Systemd is absent from containers
 if ! [[ -v container ]]; then
   systemctl daemon-reload
   systemctl enable ipfs.service
+  systemctl restart ipfs.service
   systemctl enable aleph-vm-supervisor.service
   systemctl restart aleph-vm-supervisor.service
 fi
diff --git a/packaging/aleph-vm/DEBIAN/postrm b/packaging/aleph-vm/DEBIAN/postrm
index 5da106c35..a2bdda35d 100755
--- a/packaging/aleph-vm/DEBIAN/postrm
+++ b/packaging/aleph-vm/DEBIAN/postrm
@@ -5,4 +5,9 @@ rm -fr /srv/jailer  # Upgrade from < 0.1.11
 rm -fr /tmp/aleph/  # Upgrade from < 0.1.11
 rm -r /var/lib/aleph/vm/jailer
 
+if [ "$1" = "purge" ]; then
+    # Remove the directory when the package is purged
+    rm -rf /var/lib/ipfs
+fi
+
 systemctl daemon-reload
diff --git a/packaging/aleph-vm/DEBIAN/preinst b/packaging/aleph-vm/DEBIAN/preinst
index c76d9f6a4..4fb97a8be 100755
--- a/packaging/aleph-vm/DEBIAN/preinst
+++ b/packaging/aleph-vm/DEBIAN/preinst
@@ -1,6 +1,8 @@
 #!/bin/bash
 set -uf -o pipefail
 
+# Documentation: https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html
+
 # Systemd is absent from containers
 if ! [[ -v container ]]; then
   # Stop the service during an upgrade.
@@ -9,3 +11,7 @@ if ! [[ -v container ]]; then
 fi
 
 set -e
+
+# We will not delete this user on uninstall since there may be files owned by that user in /var/lib/ipfs
+addgroup --system ipfs
+adduser --system --ingroup ipfs ipfs
diff --git a/packaging/aleph-vm/etc/systemd/system/ipfs.service b/packaging/aleph-vm/etc/systemd/system/ipfs.service
index 914593bcf..6f4a8c359 100644
--- a/packaging/aleph-vm/etc/systemd/system/ipfs.service
+++ b/packaging/aleph-vm/etc/systemd/system/ipfs.service
@@ -1,3 +1,5 @@
+# Source: https://github.com/ipfs/kubo/blob/master/misc/systemd/ipfs-hardened.service
+
 # This file will be overwritten on package upgrades, avoid customizations here.
 #
 # To make persistent changes, create file in
@@ -12,12 +14,42 @@
 #
 # For more info about custom unit files see systemd.unit(5).
 
+# This service file enables systemd-hardening features compatible with IPFS,
+# while breaking compatibility with the fuse-mount function. Use this one only
+# if you don't need the fuse-mount functionality.
+
 [Unit]
 Description=InterPlanetary File System (IPFS) daemon
 Documentation=https://docs.ipfs.tech/
 After=network.target
 
 [Service]
+# hardening
+ReadOnlyPaths="/opt/kubo/"
+ReadWritePaths="/var/lib/ipfs/"
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectKernelLogs=true
+PrivateDevices=true
+DevicePolicy=closed
+ProtectControlGroups=true
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
+ProtectHostname=true
+PrivateTmp=true
+ProtectClock=true
+LockPersonality=true
+RestrictNamespaces=true
+RestrictRealtime=true
+MemoryDenyWriteExecute=true
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallFilter=~@privileged
+ProtectHome=true
+RemoveIPC=true
+RestrictSUIDSGID=true
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
 
 # enable for 1-1024 port listening
 #AmbientCapabilities=CAP_NET_BIND_SERVICE
@@ -37,9 +69,10 @@ MemorySwapMax=0
 TimeoutStartSec=infinity
 
 Type=notify
-StateDirectory=ipfs
-Environment=IPFS_PATH="${HOME}"
-ExecStart=/opt/kubo/ipfs daemon --init --init-profile=server --migrate
+User=ipfs
+Group=ipfs
+Environment=IPFS_PATH="/var/lib/ipfs"
+ExecStart=/opt/kubo/ipfs daemon --init --migrate --init-profile=server
 Restart=on-failure
 KillSignal=SIGINT
 

From 11204adf0b37c678e104e0187584a5056cdcdbc2 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Thu, 28 Sep 2023 00:01:35 +0200
Subject: [PATCH 489/990] Chore: bump FastAPI to 0.103.1 in runtime

---
 runtimes/aleph-debian-11-python/create_disk_image.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
index 65f33d7c0..716f344bb 100755
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-11-python/create_disk_image.sh
@@ -31,7 +31,7 @@ apt-get install -y --no-install-recommends --no-install-suggests \
   \
   iputils-ping curl
 
-pip3 install 'fastapi~=0.95.1'
+pip3 install 'fastapi~=0.103.1'
 
 echo "Pip installing aleph-client"
 pip3 install 'aleph-sdk-python==0.7.0'

From 046c82293ff2237ce71e9d4500c053e401df01c6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 27 Sep 2023 14:03:06 +0200
Subject: [PATCH 490/990] Fix: Unclear stacktrace when port in used already

Solution: Display a clear error message when the port is already in use.
---
 vm_supervisor/supervisor.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/vm_supervisor/supervisor.py b/vm_supervisor/supervisor.py
index 9a95dec9d..959e306e6 100644
--- a/vm_supervisor/supervisor.py
+++ b/vm_supervisor/supervisor.py
@@ -96,6 +96,14 @@ def run():
             app.on_cleanup.append(stop_all_vms)
 
         web.run_app(app, host=settings.SUPERVISOR_HOST, port=settings.SUPERVISOR_PORT)
+    except OSError as e:
+        if e.errno == 98:
+            logger.error(
+                f"Port {settings.SUPERVISOR_PORT} already in use. "
+                f"Please check that no other instance of Aleph-VM is running."
+            )
+        else:
+            raise
     finally:
         if settings.ALLOW_VM_NETWORKING:
             pool.network.teardown()

From dc5f204cc86d8227d291a5e02e03d69f7c02f13b Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Thu, 28 Sep 2023 11:11:56 +0200
Subject: [PATCH 491/990] Fix: install FastAPI from PyPI in Debian 12 runtime

Problem: Debian 12 runtime has FastAPI 0.92.0 (installed from apt) while
Debian 11 runtime has FastAPI 0.95.1. Some VMs previously running on the
Debian 11 runtime do not run on the Debian 12 runtime because of this
issue.

Solution: install the latest version of FastAPI from PyPI.
---
 runtimes/aleph-debian-11-python/create_disk_image.sh | 3 ---
 runtimes/aleph-debian-12-python/create_disk_image.sh | 5 ++---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
index 716f344bb..2387398d5 100755
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-11-python/create_disk_image.sh
@@ -17,18 +17,15 @@ apt-get install -y --no-install-recommends --no-install-suggests \
   python3-minimal \
   openssh-server \
   socat libsecp256k1-0 \
-  \
   python3-aiohttp python3-msgpack \
   python3-setuptools \
   python3-pip python3-cytoolz python3-pydantic \
   iproute2 unzip \
   nodejs npm \
   build-essential python3-dev \
-  \
   docker.io \
   cgroupfs-mount \
   nftables \
-  \
   iputils-ping curl
 
 pip3 install 'fastapi~=0.103.1'
diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index 1d969f69c..44006b9e9 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -17,7 +17,6 @@ apt-get install -y --no-install-recommends --no-install-suggests \
   python3-minimal \
   openssh-server \
   socat libsecp256k1-1 \
-  \
   python3-aiohttp python3-msgpack \
   python3-setuptools python3-venv \
   python3-pip python3-cytoolz python3-pydantic \
@@ -25,13 +24,13 @@ apt-get install -y --no-install-recommends --no-install-suggests \
   nodejs npm \
   build-essential python3-dev \
   python3-fastapi \
-  \
   docker.io \
   cgroupfs-mount \
   nftables \
-  \
   iputils-ping curl
 
+pip3 install --break-system-packages 'fastapi~=0.103.1'
+
 echo "Pip installing aleph-sdk-python"
 mkdir -p /opt/aleph/libs
 pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.7.0'

From 9368a1a185c2b820e2f6cb976c9d6016e02acd4c Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Tue, 3 Oct 2023 15:39:22 +0200
Subject: [PATCH 492/990] Update
 runtimes/aleph-debian-12-python/create_disk_image.sh

Co-authored-by: Hugo Herter 
---
 runtimes/aleph-debian-12-python/create_disk_image.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index 44006b9e9..2fb98c470 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -33,7 +33,7 @@ pip3 install --break-system-packages 'fastapi~=0.103.1'
 
 echo "Pip installing aleph-sdk-python"
 mkdir -p /opt/aleph/libs
-pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.7.0'
+pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.7.0' 'fastapi~=0.103.1'
 
 # Compile Python code to bytecode for faster execution
 python3 -m compileall -f /usr/local/lib/python3.11

From d19a4147d53feacf7989897d27e4e2cda086ac26 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Tue, 3 Oct 2023 15:40:27 +0200
Subject: [PATCH 493/990] avoid double install

---
 runtimes/aleph-debian-12-python/create_disk_image.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index 2fb98c470..8ba75ec19 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -29,8 +29,6 @@ apt-get install -y --no-install-recommends --no-install-suggests \
   nftables \
   iputils-ping curl
 
-pip3 install --break-system-packages 'fastapi~=0.103.1'
-
 echo "Pip installing aleph-sdk-python"
 mkdir -p /opt/aleph/libs
 pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.7.0' 'fastapi~=0.103.1'

From 14096a8ced9d879cef6abfbb4980e94a605a49d1 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 26 Sep 2023 19:19:31 +0200
Subject: [PATCH 494/990] Fix: Diagnostic did not expose platform information

Solution: Expose os-release, Python version and installed Python packages via the example_fastapi diagnostic VM.
---
 .github/workflows/code-quality.yml |  4 ++++
 examples/example_fastapi/main.py   | 35 ++++++++++++++++++++----------
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
index 117ee7ef6..738a6ed27 100644
--- a/.github/workflows/code-quality.yml
+++ b/.github/workflows/code-quality.yml
@@ -27,21 +27,25 @@ jobs:
         run: |
           black --check ./vm_supervisor
           black --check ./runtimes/aleph-debian-11-python/init1.py
+          black --check ./examples/example_fastapi/
 
       - name: Test with isort
         run: |
           isort --check-only --profile=black ./vm_supervisor
           isort --check-only --profile=black ./runtimes/aleph-debian-11-python/init1.py
+          isort --check-only --profile=black ./examples/example_fastapi/
 
       - name: Test with MyPy
         run: |
           mypy --ignore-missing-imports ./vm_supervisor
           mypy --ignore-missing-imports ./runtimes/aleph-debian-11-python/init1.py
+          mypy --ignore-missing-imports ./examples/example_fastapi/
 
       - name: Test with flake8
         run: |
           flake8 --extend-ignore E501 ./vm_supervisor
           flake8 --extend-ignore E501,E402 ./runtimes/aleph-debian-11-python/init1.py
+          flake8 --extend-ignore E501,E402 ./examples/example_fastapi/
 
   code-quality-shell:
     runs-on: ubuntu-22.04
diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index e2931db6f..866fb6ac4 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -6,26 +6,21 @@
 import sys
 from datetime import datetime
 from os import listdir
+from pathlib import Path
 from typing import Dict
 
-from pydantic import BaseModel
-
-logger = logging.getLogger(__name__)
-
-logger.debug("import aiohttp")
 import aiohttp
-
-logger.debug("import aleph_client")
-from aleph.sdk.client import AlephClient, AuthenticatedAlephClient
 from aleph.sdk.chains.remote import RemoteAccount
+from aleph.sdk.client import AlephClient, AuthenticatedAlephClient
 from aleph.sdk.types import StorageEnum
 from aleph.sdk.vm.app import AlephApp
 from aleph.sdk.vm.cache import VmCache
-
-logger.debug("import fastapi")
 from fastapi import FastAPI
 from fastapi.responses import PlainTextResponse
+from pip._internal.operations.freeze import freeze
+from pydantic import BaseModel
 
+logger = logging.getLogger(__name__)
 logger.debug("imports done")
 
 http_app = FastAPI()
@@ -60,6 +55,9 @@ async def index():
             "/post_a_message",
             "/state/increment",
             "/wait-for/{delay}",
+            "/platform/os",
+            "/platform/python",
+            "/platform/pip-freeze",
         ],
         "files_in_volumes": {
             "/opt/venv": opt_venv,
@@ -72,7 +70,7 @@ async def check_lifespan():
     """
     Check that ASGI lifespan startup signal has been received
     """
-    return {"Lifetime": startup_lifespan_executed}
+    return {"Lifespan": startup_lifespan_executed}
 
 
 @app.get("/environ")
@@ -252,6 +250,21 @@ def crash():
 ]
 
 
+@app.get("/platform/os")
+def platform_os():
+    return PlainTextResponse(content=Path("/etc/os-release").read_text())
+
+
+@app.get("/platform/python")
+def platform_python():
+    return PlainTextResponse(content=sys.version)
+
+
+@app.get("/platform/pip-freeze")
+def platform_pip_freeze():
+    return list(freeze())
+
+
 @app.event(filters=filters)
 async def aleph_event(event):
     print("aleph_event", event)

From b2d6314f4965a9be47a3f1bacc787cc8742aa0b7 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 3 Oct 2023 16:43:39 +0200
Subject: [PATCH 495/990] Fix: Concurrent calls to stop() caused issues. (#415)

Fix: Concurrent calls to stop() caused issues.

In particular, when recording the resource usage of the VM, the database raised the error `UNIQUE constraint failed: records.uuid
External`.
---
 vm_supervisor/models.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/vm_supervisor/models.py b/vm_supervisor/models.py
index 912fa0796..626c20560 100644
--- a/vm_supervisor/models.py
+++ b/vm_supervisor/models.py
@@ -68,6 +68,7 @@ class VmExecution:
     ready_event: asyncio.Event
     concurrent_runs: int
     runs_done_event: asyncio.Event
+    stop_pending_lock: asyncio.Lock
     expire_task: Optional[asyncio.Task] = None
     update_task: Optional[asyncio.Task] = None
 
@@ -108,6 +109,7 @@ def __init__(
         self.ready_event = asyncio.Event()
         self.concurrent_runs = 0
         self.runs_done_event = asyncio.Event()
+        self.stop_pending_lock = asyncio.Lock()
         self.snapshot_manager = snapshot_manager
 
     def to_dict(self) -> Dict:
@@ -215,19 +217,23 @@ def cancel_update(self) -> bool:
             return False
 
     async def stop(self):
-        if self.times.stopped_at is not None:
-            logger.debug(f"VM={self.vm.vm_id} already stopped")
-            return
-        await self.all_runs_complete()
-        self.times.stopping_at = datetime.now()
-        await self.record_usage()
-        await self.vm.teardown()
-        self.times.stopped_at = datetime.now()
-        self.cancel_expiration()
-        self.cancel_update()
-
-        if isinstance(self.message, InstanceContent):
-            await self.snapshot_manager.stop_for(self.vm_hash)
+        """Stop the VM and release resources"""
+
+        # Prevent concurrent calls to stop() using a Lock
+        async with self.stop_pending_lock:
+            if self.times.stopped_at is not None:
+                logger.debug(f"VM={self.vm.vm_id} already stopped")
+                return
+            await self.all_runs_complete()
+            self.times.stopping_at = datetime.now()
+            await self.record_usage()
+            await self.vm.teardown()
+            self.times.stopped_at = datetime.now()
+            self.cancel_expiration()
+            self.cancel_update()
+
+            if isinstance(self.message, InstanceContent):
+                await self.snapshot_manager.stop_for(self.vm_hash)
 
     def start_watching_for_updates(self, pubsub: PubSub):
         if not self.update_task:

From 144b1faed63442a7a48f13133eba5f4304617043 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 3 Oct 2023 15:43:14 +0200
Subject: [PATCH 496/990] Fix: System packages had priority over user installed

By appending the path `/opt/packages` to the Python Path, system packages were loaded in priority. This prevented users from shipping their own versions of packages also present on the OS.
---
 runtimes/aleph-debian-11-python/init1.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index 01127d09e..a368552a7 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -236,13 +236,13 @@ async def send(response: Dict):
 async def setup_code_asgi(
     code: bytes, encoding: Encoding, entrypoint: str
 ) -> ASGIApplication:
-    # Allow importing packages from /opt/packages
-    sys.path.append("/opt/packages")
+    # Allow importing packages from /opt/packages, give it priority
+    sys.path.insert(0, "/opt/packages")
 
     logger.debug("Extracting code")
     app: ASGIApplication
     if encoding == Encoding.squashfs:
-        sys.path.append("/opt/code")
+        sys.path.insert(0, "/opt/code")
         module_name, app_name = entrypoint.split(":", 1)
         logger.debug("import module")
         module = __import__(module_name)
@@ -255,7 +255,7 @@ async def setup_code_asgi(
             open("/opt/archive.zip", "wb").write(code)
             logger.debug("Run unzip")
             os.system("unzip -q /opt/archive.zip -d /opt")
-        sys.path.append("/opt")
+        sys.path.insert(0, "/opt")
         module_name, app_name = entrypoint.split(":", 1)
         logger.debug("import module")
         module = __import__(module_name)

From 0915bfd2f40359ac23c0ef8e2cd4831698508559 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 4 Oct 2023 12:37:24 +0200
Subject: [PATCH 497/990] Fix: Diagnosis VM used the old runtime.

This upgrades it to use the newer Debian 12 runtime.

The new diagnosis VM adds extra enpoints, and supports the newer "Lifespan" ASGI API.
---
 vm_supervisor/conf.py           | 2 +-
 vm_supervisor/status.py         | 2 +-
 vm_supervisor/views/__init__.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/vm_supervisor/conf.py b/vm_supervisor/conf.py
index bfa76e230..6bf5487a5 100644
--- a/vm_supervisor/conf.py
+++ b/vm_supervisor/conf.py
@@ -242,7 +242,7 @@ class Settings(BaseSettings):
     )
 
     CHECK_FASTAPI_VM_ID = (
-        "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
+        "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af"
     )
 
     # Developer options
diff --git a/vm_supervisor/status.py b/vm_supervisor/status.py
index aaaeda5a2..439cde7fb 100644
--- a/vm_supervisor/status.py
+++ b/vm_supervisor/status.py
@@ -33,7 +33,7 @@ async def check_index(session: ClientSession) -> bool:
 async def check_lifespan(session: ClientSession) -> bool:
     try:
         result: Dict = await get_json_from_vm(session, "/lifespan")
-        return result["Lifetime"] is True
+        return result["Lifespan"] is True
     except ClientResponseError:
         return False
 
diff --git a/vm_supervisor/views/__init__.py b/vm_supervisor/views/__init__.py
index 2f0c89bc7..fd60df618 100644
--- a/vm_supervisor/views/__init__.py
+++ b/vm_supervisor/views/__init__.py
@@ -150,7 +150,7 @@ async def status_check_fastapi(request: web.Request):
         result = {
             "index": await status.check_index(session),
             # TODO: lifespan is a new feature that requires a new runtime to be deployed
-            # "lifespan": await status.check_lifespan(session),
+            "lifespan": await status.check_lifespan(session),
             "environ": await status.check_environ(session),
             "messages": await status.check_messages(session),
             "dns": await status.check_dns(session),

From bb6b9b0d7e70b1b888cecf2f50ec38246f577334 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 4 Oct 2023 10:21:08 +0200
Subject: [PATCH 498/990] Fix: Errors in guest_api were not reported on Sentry

---
 guest_api/__main__.py                      | 25 ++++++++++++++++++++--
 vm_supervisor/__main__.py                  |  2 +-
 vm_supervisor/vm/firecracker/executable.py |  3 ++-
 3 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/guest_api/__main__.py b/guest_api/__main__.py
index 829680bbf..1692139a3 100644
--- a/guest_api/__main__.py
+++ b/guest_api/__main__.py
@@ -4,10 +4,15 @@
 from typing import Optional
 
 import aiohttp
-from aiohttp import web
 import aioredis
+from aiohttp import web
 from setproctitle import setproctitle
 
+try:
+    import sentry_sdk
+except ImportError:
+    sentry_sdk = None
+
 logger = logging.getLogger(__name__)
 
 ALEPH_API_SERVER = "https://official.aleph.cloud"
@@ -155,7 +160,23 @@ async def list_keys_from_cache(request: web.Request):
     return web.json_response(keys)
 
 
-def run_guest_api(unix_socket_path, vm_hash: Optional[str] = None):
+def run_guest_api(
+    unix_socket_path,
+    vm_hash: Optional[str] = None,
+    sentry_dsn: Optional[str] = None,
+    server_name: Optional[str] = None,
+):
+    # This function runs in a separate process, requiring to reinitialize the Sentry SDK
+    if sentry_sdk and sentry_dsn:
+        sentry_sdk.init(
+            dsn=sentry_dsn,
+            server_name=server_name,
+            # Set traces_sample_rate to 1.0 to capture 100%
+            # of transactions for performance monitoring.
+            # We recommend adjusting this value in production.
+            traces_sample_rate=1.0,
+        )
+
     setproctitle(f"aleph-vm guest_api on {unix_socket_path}")
     app = web.Application()
     app["meta_vm_hash"] = vm_hash or "_"
diff --git a/vm_supervisor/__main__.py b/vm_supervisor/__main__.py
index 2f78d9471..5342ebeac 100644
--- a/vm_supervisor/__main__.py
+++ b/vm_supervisor/__main__.py
@@ -324,7 +324,7 @@ def main():
                 traces_sample_rate=1.0,
             )
         else:
-            logger.debug("Sentry SDK found with no DNS configured.")
+            logger.debug("Sentry SDK found with no DSN configured.")
     else:
         logger.debug(
             "Sentry SDK not found. \n"
diff --git a/vm_supervisor/vm/firecracker/executable.py b/vm_supervisor/vm/firecracker/executable.py
index 0d3985d53..3bad89bd5 100644
--- a/vm_supervisor/vm/firecracker/executable.py
+++ b/vm_supervisor/vm/firecracker/executable.py
@@ -269,7 +269,8 @@ async def start_guest_api(self):
         vsock_path = f"{self.fvm.vsock_path}_53"
         vm_hash = self.vm_hash
         self.guest_api_process = Process(
-            target=run_guest_api, args=(vsock_path, vm_hash)
+            target=run_guest_api,
+            args=(vsock_path, vm_hash, settings.SENTRY_DSN, settings.DOMAIN_NAME),
         )
         self.guest_api_process.start()
         while not exists(vsock_path):

From 1849b833c41ca7f0c413fa1e7a5bd7e51e5bfdf2 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 5 Oct 2023 13:27:16 +0200
Subject: [PATCH 499/990] Fix: Give in total 60 second to the VM to respond the
 ping.

---
 vm_supervisor/vm/firecracker/instance.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index be4f886ba..680f1074b 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -142,8 +142,8 @@ async def wait_for_init(self) -> None:
 
         ip = ip.split("/", 1)[0]
 
-        attempts = 10
-        timeout_seconds = 1.0
+        attempts = 30
+        timeout_seconds = 2.0
 
         for attempt in range(attempts):
             try:

From e1a5224ee79cb2c549742b7519aadeb972e98d43 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 10 Oct 2023 09:37:23 +0200
Subject: [PATCH 500/990] Fix: Newer kernel is available

This updates the build-kernel.sh script to use the newer 5.10.197 kernel instead of 5.10.124.

Other minor improvements to the build script:
- cleanup existing resources first
- document additional packages required for the build
- detect the number of CPU cores for the build
- update the repository with the config file updated by make
---
 kernels/build-kernel.sh | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/kernels/build-kernel.sh b/kernels/build-kernel.sh
index 99c85c18e..61ac7fbc6 100644
--- a/kernels/build-kernel.sh
+++ b/kernels/build-kernel.sh
@@ -1,18 +1,27 @@
 #!/bin/bash
+
 set -euf -o pipefail
 
-curl -OL "https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.124.tar.xz"
-curl -OL "https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.124.tar.sign"
-unxz linux-5.10.124.tar.xz
+# apt install ncurses-dev flex bison bc
+
+rm -fr linux-5.10.197 linux-5.10.197.tar linux-5.10.197.tar.sign  linux-5.10.197.tar.xz
+
+
+curl -OL "https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.197.tar.xz"
+curl -OL "https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.197.tar.sign"
+unxz linux-5.10.197.tar.xz
 
 gpg --locate-keys torvalds@kernel.org gregkh@kernel.org
-gpg --verify linux-5.10.124.tar.sign linux-5.10.124.tar
+gpg --verify linux-5.10.197.tar.sign linux-5.10.197.tar
 
-tar -xvf linux-5.10.124.tar
+tar -xvf linux-5.10.197.tar
 
-cp microvm-kernel-x86_64-5.10.config linux-5.10.124/.config
+cp microvm-kernel-x86_64-5.10.config linux-5.10.197/.config
 
-cd linux-5.10.124/
+cd linux-5.10.197/
 make menuconfig
 
-make -j32 vmlinux
\ No newline at end of file
+make -j$(nproc) vmlinux
+
+# Copy the updated config locally for documentation
+cp linux-5.10.197/.config ./linux.config

From 1b029f418db968cf3f44ee8b9f35c435cda2ba47 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 10 Oct 2023 09:38:35 +0200
Subject: [PATCH 501/990] Fix: Update linux.config with new kernel build

The content was from a previous kernel build.
---
 kernels/linux.config | 951 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 704 insertions(+), 247 deletions(-)

diff --git a/kernels/linux.config b/kernels/linux.config
index 02d05f2d8..e2c590c7f 100644
--- a/kernels/linux.config
+++ b/kernels/linux.config
@@ -1,16 +1,23 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/x86 4.20.0 Kernel Configuration
-#
-
-#
-# Compiler: gcc (Debian 8.3.0-6) 8.3.0
+# Linux/x86 5.10.197 Kernel Configuration
 #
+CONFIG_CC_VERSION_TEXT="gcc (Debian 12.2.0-14) 12.2.0"
 CONFIG_CC_IS_GCC=y
-CONFIG_GCC_VERSION=80300
+CONFIG_GCC_VERSION=120200
+CONFIG_LD_VERSION=240000000
 CONFIG_CLANG_VERSION=0
+CONFIG_AS_IS_GNU=y
+CONFIG_AS_VERSION=24000
+CONFIG_LLD_VERSION=0
+CONFIG_CC_CAN_LINK=y
+CONFIG_CC_CAN_LINK_STATIC=y
+CONFIG_CC_HAS_ASM_GOTO=y
+CONFIG_CC_HAS_ASM_GOTO_OUTPUT=y
+CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT=y
+CONFIG_CC_HAS_ASM_INLINE=y
 CONFIG_IRQ_WORK=y
-CONFIG_BUILDTIME_EXTABLE_SORT=y
+CONFIG_BUILDTIME_TABLE_SORT=y
 CONFIG_THREAD_INFO_IN_TASK=y
 
 #
@@ -27,25 +34,27 @@ CONFIG_HAVE_KERNEL_LZMA=y
 CONFIG_HAVE_KERNEL_XZ=y
 CONFIG_HAVE_KERNEL_LZO=y
 CONFIG_HAVE_KERNEL_LZ4=y
+CONFIG_HAVE_KERNEL_ZSTD=y
 CONFIG_KERNEL_GZIP=y
 # CONFIG_KERNEL_BZIP2 is not set
 # CONFIG_KERNEL_LZMA is not set
 # CONFIG_KERNEL_XZ is not set
 # CONFIG_KERNEL_LZO is not set
 # CONFIG_KERNEL_LZ4 is not set
+# CONFIG_KERNEL_ZSTD is not set
+CONFIG_DEFAULT_INIT=""
 CONFIG_DEFAULT_HOSTNAME="(none)"
 CONFIG_SWAP=y
 CONFIG_SYSVIPC=y
 CONFIG_SYSVIPC_SYSCTL=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_POSIX_MQUEUE_SYSCTL=y
+# CONFIG_WATCH_QUEUE is not set
 CONFIG_CROSS_MEMORY_ATTACH=y
 # CONFIG_USELIB is not set
 CONFIG_AUDIT=y
 CONFIG_HAVE_ARCH_AUDITSYSCALL=y
 CONFIG_AUDITSYSCALL=y
-CONFIG_AUDIT_WATCH=y
-CONFIG_AUDIT_TREE=y
 
 #
 # IRQ subsystem
@@ -55,6 +64,7 @@ CONFIG_GENERIC_IRQ_SHOW=y
 CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK=y
 CONFIG_GENERIC_PENDING_IRQ=y
 CONFIG_GENERIC_IRQ_MIGRATION=y
+CONFIG_HARDIRQS_SW_RESEND=y
 CONFIG_IRQ_DOMAIN=y
 CONFIG_IRQ_DOMAIN_HIERARCHY=y
 CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR=y
@@ -62,8 +72,9 @@ CONFIG_GENERIC_IRQ_RESERVATION_MODE=y
 CONFIG_IRQ_FORCED_THREADING=y
 CONFIG_SPARSE_IRQ=y
 # CONFIG_GENERIC_IRQ_DEBUGFS is not set
+# end of IRQ subsystem
+
 CONFIG_CLOCKSOURCE_WATCHDOG=y
-CONFIG_ARCH_CLOCKSOURCE_DATA=y
 CONFIG_ARCH_CLOCKSOURCE_INIT=y
 CONFIG_CLOCKSOURCE_VALIDATE_LAST_CYCLE=y
 CONFIG_GENERIC_TIME_VSYSCALL=y
@@ -71,6 +82,8 @@ CONFIG_GENERIC_CLOCKEVENTS=y
 CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y
 CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST=y
 CONFIG_GENERIC_CMOS_UPDATE=y
+CONFIG_HAVE_POSIX_CPU_TIMERS_TASK_WORK=y
+CONFIG_POSIX_CPU_TIMERS_TASK_WORK=y
 
 #
 # Timers subsystem
@@ -82,6 +95,8 @@ CONFIG_NO_HZ_IDLE=y
 # CONFIG_NO_HZ_FULL is not set
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
+# end of Timers subsystem
+
 CONFIG_PREEMPT_NONE=y
 # CONFIG_PREEMPT_VOLUNTARY is not set
 # CONFIG_PREEMPT is not set
@@ -100,6 +115,8 @@ CONFIG_TASK_DELAY_ACCT=y
 CONFIG_TASK_XACCT=y
 CONFIG_TASK_IO_ACCOUNTING=y
 # CONFIG_PSI is not set
+# end of CPU/Task time and stats accounting
+
 CONFIG_CPU_ISOLATION=y
 
 #
@@ -109,16 +126,29 @@ CONFIG_TREE_RCU=y
 # CONFIG_RCU_EXPERT is not set
 CONFIG_SRCU=y
 CONFIG_TREE_SRCU=y
+CONFIG_TASKS_RCU_GENERIC=y
+CONFIG_TASKS_TRACE_RCU=y
 CONFIG_RCU_STALL_COMMON=y
 CONFIG_RCU_NEED_SEGCBLIST=y
+# end of RCU Subsystem
+
 CONFIG_BUILD_BIN2C=y
 # CONFIG_IKCONFIG is not set
+# CONFIG_IKHEADERS is not set
 CONFIG_LOG_BUF_SHIFT=21
 CONFIG_LOG_CPU_MAX_BUF_SHIFT=12
 CONFIG_PRINTK_SAFE_LOG_BUF_SHIFT=13
 CONFIG_HAVE_UNSTABLE_SCHED_CLOCK=y
+
+#
+# Scheduler features
+#
+# CONFIG_UCLAMP_TASK is not set
+# end of Scheduler features
+
 CONFIG_ARCH_SUPPORTS_NUMA_BALANCING=y
 CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH=y
+CONFIG_CC_HAS_INT128=y
 CONFIG_ARCH_SUPPORTS_INT128=y
 CONFIG_NUMA_BALANCING=y
 # CONFIG_NUMA_BALANCING_DEFAULT_ENABLED is not set
@@ -126,10 +156,8 @@ CONFIG_CGROUPS=y
 CONFIG_PAGE_COUNTER=y
 CONFIG_MEMCG=y
 CONFIG_MEMCG_SWAP=y
-CONFIG_MEMCG_SWAP_ENABLED=y
 CONFIG_MEMCG_KMEM=y
 CONFIG_BLK_CGROUP=y
-# CONFIG_DEBUG_BLK_CGROUP is not set
 CONFIG_CGROUP_WRITEBACK=y
 CONFIG_CGROUP_SCHED=y
 CONFIG_FAIR_GROUP_SCHED=y
@@ -145,9 +173,11 @@ CONFIG_CGROUP_DEVICE=y
 CONFIG_CGROUP_CPUACCT=y
 CONFIG_CGROUP_PERF=y
 CONFIG_CGROUP_BPF=y
+# CONFIG_CGROUP_DEBUG is not set
 CONFIG_SOCK_CGROUP_DATA=y
 CONFIG_NAMESPACES=y
 CONFIG_UTS_NS=y
+CONFIG_TIME_NS=y
 CONFIG_IPC_NS=y
 CONFIG_USER_NS=y
 CONFIG_PID_NS=y
@@ -164,10 +194,12 @@ CONFIG_RD_LZMA=y
 CONFIG_RD_XZ=y
 CONFIG_RD_LZO=y
 CONFIG_RD_LZ4=y
+CONFIG_RD_ZSTD=y
+# CONFIG_BOOT_CONFIG is not set
 CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE=y
 # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set
+CONFIG_LD_ORPHAN_WARN=y
 CONFIG_SYSCTL=y
-CONFIG_ANON_INODES=y
 CONFIG_HAVE_UID16=y
 CONFIG_SYSCTL_EXCEPTION_TRACE=y
 CONFIG_HAVE_PCSPKR_PLATFORM=y
@@ -193,13 +225,20 @@ CONFIG_TIMERFD=y
 CONFIG_EVENTFD=y
 CONFIG_SHMEM=y
 CONFIG_AIO=y
+CONFIG_IO_URING=y
 CONFIG_ADVISE_SYSCALLS=y
+CONFIG_HAVE_ARCH_USERFAULTFD_WP=y
 CONFIG_MEMBARRIER=y
 CONFIG_KALLSYMS=y
+# CONFIG_KALLSYMS_ALL is not set
 CONFIG_KALLSYMS_ABSOLUTE_PERCPU=y
 CONFIG_KALLSYMS_BASE_RELATIVE=y
 CONFIG_BPF_SYSCALL=y
+CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
 CONFIG_BPF_JIT_ALWAYS_ON=y
+CONFIG_BPF_JIT_DEFAULT_ON=y
+# CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set
+# CONFIG_BPF_PRELOAD is not set
 CONFIG_USERFAULTFD=y
 CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y
 CONFIG_RSEQ=y
@@ -210,6 +249,9 @@ CONFIG_HAVE_PERF_EVENTS=y
 # Kernel Performance Events And Counters
 #
 CONFIG_PERF_EVENTS=y
+# CONFIG_DEBUG_PERF_USE_VMALLOC is not set
+# end of Kernel Performance Events And Counters
+
 CONFIG_VM_EVENT_COUNTERS=y
 CONFIG_SLUB_DEBUG=y
 # CONFIG_COMPAT_BRK is not set
@@ -218,15 +260,17 @@ CONFIG_SLUB=y
 CONFIG_SLAB_MERGE_DEFAULT=y
 # CONFIG_SLAB_FREELIST_RANDOM is not set
 CONFIG_SLAB_FREELIST_HARDENED=y
+# CONFIG_SHUFFLE_PAGE_ALLOCATOR is not set
 CONFIG_SLUB_CPU_PARTIAL=y
 CONFIG_SYSTEM_DATA_VERIFICATION=y
 CONFIG_PROFILING=y
+# end of General setup
+
 CONFIG_64BIT=y
 CONFIG_X86_64=y
 CONFIG_X86=y
 CONFIG_INSTRUCTION_DECODER=y
 CONFIG_OUTPUT_FORMAT="elf64-x86-64"
-CONFIG_ARCH_DEFCONFIG="arch/x86/configs/x86_64_defconfig"
 CONFIG_LOCKDEP_SUPPORT=y
 CONFIG_STACKTRACE_SUPPORT=y
 CONFIG_MMU=y
@@ -237,9 +281,7 @@ CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX=16
 CONFIG_GENERIC_ISA_DMA=y
 CONFIG_GENERIC_BUG=y
 CONFIG_GENERIC_BUG_RELATIVE_POINTERS=y
-CONFIG_GENERIC_HWEIGHT=y
 CONFIG_ARCH_MAY_HAVE_PC_FDC=y
-CONFIG_RWSEM_XCHGADD_ALGORITHM=y
 CONFIG_GENERIC_CALIBRATE_DELAY=y
 CONFIG_ARCH_HAS_CPU_RELAX=y
 CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
@@ -249,11 +291,9 @@ CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK=y
 CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK=y
 CONFIG_ARCH_HIBERNATION_POSSIBLE=y
 CONFIG_ARCH_SUSPEND_POSSIBLE=y
-CONFIG_ARCH_WANT_HUGE_PMD_SHARE=y
 CONFIG_ARCH_WANT_GENERAL_HUGETLB=y
 CONFIG_ZONE_DMA32=y
 CONFIG_AUDIT_ARCH=y
-CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING=y
 CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
 CONFIG_X86_64_SMP=y
 CONFIG_ARCH_SUPPORTS_UPROBES=y
@@ -271,19 +311,22 @@ CONFIG_X86_FEATURE_NAMES=y
 CONFIG_X86_X2APIC=y
 CONFIG_X86_MPPARSE=y
 # CONFIG_GOLDFISH is not set
-CONFIG_RETPOLINE=y
-# CONFIG_INTEL_RDT is not set
+# CONFIG_X86_CPU_RESCTRL is not set
 # CONFIG_X86_EXTENDED_PLATFORM is not set
+# CONFIG_X86_AMD_PLATFORM_DEVICE is not set
 CONFIG_SCHED_OMIT_FRAME_POINTER=y
 CONFIG_HYPERVISOR_GUEST=y
 CONFIG_PARAVIRT=y
+# CONFIG_PARAVIRT_DEBUG is not set
 CONFIG_PARAVIRT_SPINLOCKS=y
-# CONFIG_QUEUED_LOCK_STAT is not set
+CONFIG_X86_HV_CALLBACK_VECTOR=y
 # CONFIG_XEN is not set
 CONFIG_KVM_GUEST=y
-CONFIG_KVM_DEBUG_FS=y
+CONFIG_ARCH_CPUIDLE_HALTPOLL=y
+# CONFIG_PVH is not set
 CONFIG_PARAVIRT_TIME_ACCOUNTING=y
 CONFIG_PARAVIRT_CLOCK=y
+# CONFIG_ACRN_GUEST is not set
 # CONFIG_MK8 is not set
 # CONFIG_MPSC is not set
 # CONFIG_MCORE2 is not set
@@ -296,12 +339,16 @@ CONFIG_X86_CMPXCHG64=y
 CONFIG_X86_CMOV=y
 CONFIG_X86_MINIMUM_CPU_FAMILY=64
 CONFIG_X86_DEBUGCTLMSR=y
+CONFIG_IA32_FEAT_CTL=y
+CONFIG_X86_VMX_FEATURE_NAMES=y
 CONFIG_CPU_SUP_INTEL=y
 CONFIG_CPU_SUP_AMD=y
 CONFIG_CPU_SUP_HYGON=y
 CONFIG_CPU_SUP_CENTAUR=y
+CONFIG_CPU_SUP_ZHAOXIN=y
 CONFIG_HPET_TIMER=y
 CONFIG_DMI=y
+# CONFIG_MAXSMP is not set
 CONFIG_NR_CPUS_RANGE_BEGIN=2
 CONFIG_NR_CPUS_RANGE_END=512
 CONFIG_NR_CPUS_DEFAULT=64
@@ -318,20 +365,20 @@ CONFIG_X86_REROUTE_FOR_BROKEN_BOOT_IRQS=y
 # Performance monitoring
 #
 # CONFIG_PERF_EVENTS_AMD_POWER is not set
+# end of Performance monitoring
+
 CONFIG_X86_16BIT=y
 CONFIG_X86_ESPFIX64=y
 CONFIG_X86_VSYSCALL_EMULATION=y
-# CONFIG_I8K is not set
+CONFIG_X86_IOPL_IOPERM=y
 # CONFIG_MICROCODE is not set
 CONFIG_X86_MSR=y
 CONFIG_X86_CPUID=y
 # CONFIG_X86_5LEVEL is not set
 CONFIG_X86_DIRECT_GBPAGES=y
 # CONFIG_X86_CPA_STATISTICS is not set
-CONFIG_ARCH_HAS_MEM_ENCRYPT=y
 CONFIG_AMD_MEM_ENCRYPT=y
 # CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT is not set
-CONFIG_ARCH_USE_MEMREMAP_PROT=y
 CONFIG_NUMA=y
 # CONFIG_NUMA_EMU is not set
 CONFIG_NODES_SHIFT=10
@@ -353,10 +400,12 @@ CONFIG_X86_PAT=y
 CONFIG_ARCH_USES_PG_UNCACHED=y
 CONFIG_ARCH_RANDOM=y
 CONFIG_X86_SMAP=y
-CONFIG_X86_INTEL_UMIP=y
-# CONFIG_X86_INTEL_MPX is not set
+CONFIG_X86_UMIP=y
 CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS=y
-CONFIG_SECCOMP=y
+CONFIG_X86_INTEL_TSX_MODE_OFF=y
+# CONFIG_X86_INTEL_TSX_MODE_ON is not set
+# CONFIG_X86_INTEL_TSX_MODE_AUTO is not set
+# CONFIG_EFI is not set
 # CONFIG_HZ_100 is not set
 CONFIG_HZ_250=y
 # CONFIG_HZ_300 is not set
@@ -366,8 +415,7 @@ CONFIG_SCHED_HRTICK=y
 # CONFIG_KEXEC is not set
 CONFIG_KEXEC_FILE=y
 CONFIG_ARCH_HAS_KEXEC_PURGATORY=y
-CONFIG_KEXEC_VERIFY_SIG=y
-CONFIG_KEXEC_BZIMAGE_VERIFY_SIG=y
+# CONFIG_KEXEC_SIG is not set
 # CONFIG_CRASH_DUMP is not set
 CONFIG_PHYSICAL_START=0x1000000
 CONFIG_RELOCATABLE=y
@@ -378,10 +426,25 @@ CONFIG_HOTPLUG_CPU=y
 # CONFIG_DEBUG_HOTPLUG_CPU0 is not set
 # CONFIG_COMPAT_VDSO is not set
 CONFIG_LEGACY_VSYSCALL_EMULATE=y
+# CONFIG_LEGACY_VSYSCALL_XONLY is not set
 # CONFIG_LEGACY_VSYSCALL_NONE is not set
 # CONFIG_CMDLINE_BOOL is not set
 CONFIG_MODIFY_LDT_SYSCALL=y
 CONFIG_HAVE_LIVEPATCH=y
+# end of Processor type and features
+
+CONFIG_CC_HAS_SLS=y
+CONFIG_CC_HAS_RETURN_THUNK=y
+CONFIG_SPECULATION_MITIGATIONS=y
+CONFIG_PAGE_TABLE_ISOLATION=y
+CONFIG_RETPOLINE=y
+CONFIG_RETHUNK=y
+CONFIG_CPU_UNRET_ENTRY=y
+CONFIG_CPU_IBPB_ENTRY=y
+CONFIG_CPU_IBRS_ENTRY=y
+CONFIG_CPU_SRSO=y
+# CONFIG_SLS is not set
+# CONFIG_GDS_FORCE_MITIGATION is not set
 CONFIG_ARCH_HAS_ADD_PAGES=y
 CONFIG_ARCH_ENABLE_MEMORY_HOTPLUG=y
 CONFIG_ARCH_ENABLE_MEMORY_HOTREMOVE=y
@@ -397,6 +460,7 @@ CONFIG_ARCH_HIBERNATION_HEADER=y
 # CONFIG_SUSPEND is not set
 CONFIG_HIBERNATE_CALLBACKS=y
 CONFIG_HIBERNATION=y
+CONFIG_HIBERNATION_SNAPSHOT_DEV=y
 CONFIG_PM_STD_PARTITION=""
 CONFIG_PM_SLEEP=y
 CONFIG_PM_SLEEP_SMP=y
@@ -405,36 +469,81 @@ CONFIG_PM_SLEEP_SMP=y
 CONFIG_PM=y
 # CONFIG_PM_DEBUG is not set
 # CONFIG_WQ_POWER_EFFICIENT_DEFAULT is not set
+# CONFIG_ENERGY_MODEL is not set
 CONFIG_ARCH_SUPPORTS_ACPI=y
+CONFIG_ACPI=y
+CONFIG_ACPI_LEGACY_TABLES_LOOKUP=y
+CONFIG_ARCH_MIGHT_HAVE_ACPI_PDC=y
+CONFIG_ACPI_SYSTEM_POWER_STATES_SUPPORT=y
+# CONFIG_ACPI_DEBUGGER is not set
+CONFIG_ACPI_SPCR_TABLE=y
+CONFIG_ACPI_LPIT=y
+CONFIG_ACPI_SLEEP=y
+CONFIG_ACPI_REV_OVERRIDE_POSSIBLE=y
+# CONFIG_ACPI_EC_DEBUGFS is not set
+CONFIG_ACPI_AC=y
+CONFIG_ACPI_BATTERY=y
+CONFIG_ACPI_BUTTON=y
+CONFIG_ACPI_FAN=y
+# CONFIG_ACPI_TAD is not set
+# CONFIG_ACPI_DOCK is not set
+CONFIG_ACPI_CPU_FREQ_PSS=y
+CONFIG_ACPI_PROCESSOR_CSTATE=y
+CONFIG_ACPI_PROCESSOR_IDLE=y
+CONFIG_ACPI_CPPC_LIB=y
+CONFIG_ACPI_PROCESSOR=y
+CONFIG_ACPI_HOTPLUG_CPU=y
+# CONFIG_ACPI_PROCESSOR_AGGREGATOR is not set
+CONFIG_ACPI_THERMAL=y
+CONFIG_ARCH_HAS_ACPI_TABLE_UPGRADE=y
+CONFIG_ACPI_TABLE_UPGRADE=y
+# CONFIG_ACPI_DEBUG is not set
+CONFIG_ACPI_CONTAINER=y
+# CONFIG_ACPI_HOTPLUG_MEMORY is not set
+# CONFIG_ACPI_SBS is not set
+# CONFIG_ACPI_HED is not set
+# CONFIG_ACPI_CUSTOM_METHOD is not set
+# CONFIG_ACPI_NFIT is not set
+# CONFIG_ACPI_NUMA is not set
+CONFIG_HAVE_ACPI_APEI=y
+CONFIG_HAVE_ACPI_APEI_NMI=y
+# CONFIG_ACPI_APEI is not set
+# CONFIG_ACPI_DPTF is not set
+# CONFIG_ACPI_CONFIGFS is not set
+# CONFIG_PMIC_OPREGION is not set
+CONFIG_X86_PM_TIMER=y
 # CONFIG_SFI is not set
 
 #
 # CPU Frequency scaling
 #
 CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_GOV_ATTR_SET=y
 CONFIG_CPU_FREQ_STAT=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
 # CONFIG_CPU_FREQ_DEFAULT_GOV_POWERSAVE is not set
 # CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE is not set
-# CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND is not set
-# CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE is not set
 # CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL is not set
 CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
 # CONFIG_CPU_FREQ_GOV_POWERSAVE is not set
 # CONFIG_CPU_FREQ_GOV_USERSPACE is not set
 # CONFIG_CPU_FREQ_GOV_ONDEMAND is not set
 # CONFIG_CPU_FREQ_GOV_CONSERVATIVE is not set
-# CONFIG_CPU_FREQ_GOV_SCHEDUTIL is not set
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
 
 #
 # CPU frequency scaling drivers
 #
 CONFIG_X86_INTEL_PSTATE=y
+# CONFIG_X86_PCC_CPUFREQ is not set
+# CONFIG_X86_ACPI_CPUFREQ is not set
+# CONFIG_X86_SPEEDSTEP_CENTRINO is not set
 # CONFIG_X86_P4_CLOCKMOD is not set
 
 #
 # shared options
 #
+# end of CPU Frequency scaling
 
 #
 # CPU Idle
@@ -442,34 +551,31 @@ CONFIG_X86_INTEL_PSTATE=y
 CONFIG_CPU_IDLE=y
 CONFIG_CPU_IDLE_GOV_LADDER=y
 CONFIG_CPU_IDLE_GOV_MENU=y
+# CONFIG_CPU_IDLE_GOV_TEO is not set
+# CONFIG_CPU_IDLE_GOV_HALTPOLL is not set
+CONFIG_HALTPOLL_CPUIDLE=y
+# end of CPU Idle
+
 CONFIG_INTEL_IDLE=y
+# end of Power management and ACPI options
 
 #
 # Bus options (PCI etc.)
 #
-# CONFIG_PCI is not set
-CONFIG_PCI_LOCKLESS_CONFIG=y
-
-#
-# PCI Endpoint
-#
-# CONFIG_PCI_ENDPOINT is not set
 CONFIG_ISA_DMA_API=y
-# CONFIG_PCCARD is not set
 # CONFIG_X86_SYSFB is not set
+# end of Bus options (PCI etc.)
 
 #
 # Binary Emulations
 #
 CONFIG_IA32_EMULATION=y
-# CONFIG_IA32_AOUT is not set
 # CONFIG_X86_X32 is not set
 CONFIG_COMPAT_32=y
 CONFIG_COMPAT=y
 CONFIG_COMPAT_FOR_U64_ALIGNMENT=y
 CONFIG_SYSVIPC_COMPAT=y
-CONFIG_X86_DEV_DMA_OPS=y
-CONFIG_HAVE_GENERIC_GUP=y
+# end of Binary Emulations
 
 #
 # Firmware Drivers
@@ -479,14 +585,22 @@ CONFIG_FIRMWARE_MEMMAP=y
 CONFIG_DMIID=y
 # CONFIG_DMI_SYSFS is not set
 CONFIG_DMI_SCAN_MACHINE_NON_EFI_FALLBACK=y
+# CONFIG_ISCSI_IBFT is not set
 # CONFIG_FW_CFG_SYSFS is not set
 # CONFIG_GOOGLE_FIRMWARE is not set
 
 #
 # Tegra firmware driver
 #
+# end of Tegra firmware driver
+# end of Firmware Drivers
+
 CONFIG_HAVE_KVM=y
 # CONFIG_VIRTUALIZATION is not set
+CONFIG_AS_AVX512=y
+CONFIG_AS_SHA1_NI=y
+CONFIG_AS_SHA256_NI=y
+CONFIG_AS_TPAUSE=y
 
 #
 # General architecture-dependent options
@@ -494,12 +608,14 @@ CONFIG_HAVE_KVM=y
 CONFIG_CRASH_CORE=y
 CONFIG_KEXEC_CORE=y
 CONFIG_HOTPLUG_SMT=y
+CONFIG_GENERIC_ENTRY=y
 # CONFIG_OPROFILE is not set
 CONFIG_HAVE_OPROFILE=y
 CONFIG_OPROFILE_NMI_TIMER=y
 CONFIG_KPROBES=y
 CONFIG_JUMP_LABEL=y
 # CONFIG_STATIC_KEYS_SELFTEST is not set
+# CONFIG_STATIC_CALL_SELFTEST is not set
 CONFIG_OPTPROBES=y
 CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y
 CONFIG_ARCH_USE_BUILTIN_BSWAP=y
@@ -516,8 +632,11 @@ CONFIG_HAVE_DMA_CONTIGUOUS=y
 CONFIG_GENERIC_SMP_IDLE_THREAD=y
 CONFIG_ARCH_HAS_FORTIFY_SOURCE=y
 CONFIG_ARCH_HAS_SET_MEMORY=y
+CONFIG_ARCH_HAS_SET_DIRECT_MAP=y
+CONFIG_ARCH_HAS_CPU_FINALIZE_INIT=y
 CONFIG_HAVE_ARCH_THREAD_STRUCT_WHITELIST=y
 CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT=y
+CONFIG_HAVE_ASM_MODVERSIONS=y
 CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y
 CONFIG_HAVE_RSEQ=y
 CONFIG_HAVE_FUNCTION_ARG_ACCESS_API=y
@@ -530,32 +649,34 @@ CONFIG_HAVE_PERF_REGS=y
 CONFIG_HAVE_PERF_USER_STACK_DUMP=y
 CONFIG_HAVE_ARCH_JUMP_LABEL=y
 CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE=y
-CONFIG_HAVE_RCU_TABLE_FREE=y
-CONFIG_HAVE_RCU_TABLE_INVALIDATE=y
+CONFIG_MMU_GATHER_TABLE_FREE=y
+CONFIG_MMU_GATHER_RCU_TABLE_FREE=y
 CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG=y
 CONFIG_HAVE_ALIGNED_STRUCT_PAGE=y
 CONFIG_HAVE_CMPXCHG_LOCAL=y
 CONFIG_HAVE_CMPXCHG_DOUBLE=y
 CONFIG_ARCH_WANT_COMPAT_IPC_PARSE_VERSION=y
 CONFIG_ARCH_WANT_OLD_COMPAT_IPC=y
+CONFIG_HAVE_ARCH_SECCOMP=y
 CONFIG_HAVE_ARCH_SECCOMP_FILTER=y
+CONFIG_SECCOMP=y
 CONFIG_SECCOMP_FILTER=y
 CONFIG_HAVE_ARCH_STACKLEAK=y
 CONFIG_HAVE_STACKPROTECTOR=y
-CONFIG_CC_HAS_STACKPROTECTOR_NONE=y
 CONFIG_STACKPROTECTOR=y
 CONFIG_STACKPROTECTOR_STRONG=y
 CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES=y
 CONFIG_HAVE_CONTEXT_TRACKING=y
 CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y
 CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y
+CONFIG_HAVE_MOVE_PMD=y
 CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE=y
 CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD=y
 CONFIG_HAVE_ARCH_HUGE_VMAP=y
+CONFIG_ARCH_WANT_HUGE_PMD_SHARE=y
 CONFIG_HAVE_ARCH_SOFT_DIRTY=y
 CONFIG_HAVE_MOD_ARCH_SPECIFIC=y
 CONFIG_MODULES_USE_ELF_RELA=y
-CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK=y
 CONFIG_ARCH_HAS_ELF_RANDOMIZE=y
 CONFIG_HAVE_ARCH_MMAP_RND_BITS=y
 CONFIG_HAVE_EXIT_THREAD=y
@@ -563,7 +684,6 @@ CONFIG_ARCH_MMAP_RND_BITS=28
 CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS=y
 CONFIG_ARCH_MMAP_RND_COMPAT_BITS=8
 CONFIG_HAVE_ARCH_COMPAT_MMAP_BASES=y
-CONFIG_HAVE_COPY_THREAD_TLS=y
 CONFIG_HAVE_STACK_VALIDATION=y
 CONFIG_HAVE_RELIABLE_STACKTRACE=y
 CONFIG_OLD_SIGSUSPEND3=y
@@ -575,24 +695,34 @@ CONFIG_ARCH_HAS_STRICT_KERNEL_RWX=y
 CONFIG_STRICT_KERNEL_RWX=y
 CONFIG_ARCH_HAS_STRICT_MODULE_RWX=y
 CONFIG_STRICT_MODULE_RWX=y
-CONFIG_ARCH_HAS_REFCOUNT=y
-# CONFIG_REFCOUNT_FULL is not set
 CONFIG_HAVE_ARCH_PREL32_RELOCATIONS=y
+CONFIG_ARCH_USE_MEMREMAP_PROT=y
+# CONFIG_LOCK_EVENT_COUNTS is not set
+CONFIG_ARCH_HAS_MEM_ENCRYPT=y
+CONFIG_ARCH_HAS_CC_PLATFORM=y
+CONFIG_HAVE_STATIC_CALL=y
+CONFIG_HAVE_STATIC_CALL_INLINE=y
+CONFIG_ARCH_WANT_LD_ORPHAN_WARN=y
 
 #
 # GCOV-based kernel profiling
 #
 # CONFIG_GCOV_KERNEL is not set
 CONFIG_ARCH_HAS_GCOV_PROFILE_ALL=y
-CONFIG_PLUGIN_HOSTCC=""
+# end of GCOV-based kernel profiling
+
 CONFIG_HAVE_GCC_PLUGINS=y
+# end of General architecture-dependent options
+
 CONFIG_RT_MUTEXES=y
 CONFIG_BASE_SMALL=0
+CONFIG_MODULE_SIG_FORMAT=y
 CONFIG_MODULES=y
 CONFIG_MODULE_FORCE_LOAD=y
 CONFIG_MODULE_UNLOAD=y
 # CONFIG_MODULE_FORCE_UNLOAD is not set
 CONFIG_MODVERSIONS=y
+CONFIG_ASM_MODVERSIONS=y
 CONFIG_MODULE_SRCVERSION_ALL=y
 CONFIG_MODULE_SIG=y
 # CONFIG_MODULE_SIG_FORCE is not set
@@ -604,9 +734,12 @@ CONFIG_MODULE_SIG_ALL=y
 CONFIG_MODULE_SIG_SHA512=y
 CONFIG_MODULE_SIG_HASH="sha512"
 # CONFIG_MODULE_COMPRESS is not set
+# CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set
+CONFIG_UNUSED_SYMBOLS=y
 CONFIG_MODULES_TREE_LOOKUP=y
 CONFIG_BLOCK=y
 CONFIG_BLK_SCSI_REQUEST=y
+CONFIG_BLK_CGROUP_RWSTAT=y
 CONFIG_BLK_DEV_BSG=y
 CONFIG_BLK_DEV_BSGLIB=y
 CONFIG_BLK_DEV_INTEGRITY=y
@@ -616,10 +749,11 @@ CONFIG_BLK_DEV_THROTTLING=y
 CONFIG_BLK_CMDLINE_PARSER=y
 CONFIG_BLK_WBT=y
 # CONFIG_BLK_CGROUP_IOLATENCY is not set
-# CONFIG_BLK_WBT_SQ is not set
+# CONFIG_BLK_CGROUP_IOCOST is not set
 CONFIG_BLK_WBT_MQ=y
 CONFIG_BLK_DEBUG_FS=y
 # CONFIG_BLK_SED_OPAL is not set
+# CONFIG_BLK_INLINE_ENCRYPTION is not set
 
 #
 # Partition Types
@@ -631,7 +765,11 @@ CONFIG_PARTITION_ADVANCED=y
 # CONFIG_AMIGA_PARTITION is not set
 # CONFIG_ATARI_PARTITION is not set
 # CONFIG_MAC_PARTITION is not set
-# CONFIG_MSDOS_PARTITION is not set
+CONFIG_MSDOS_PARTITION=y
+# CONFIG_BSD_DISKLABEL is not set
+# CONFIG_MINIX_SUBPARTITION is not set
+# CONFIG_SOLARIS_X86_PARTITION is not set
+# CONFIG_UNIXWARE_DISKLABEL is not set
 # CONFIG_LDM_PARTITION is not set
 # CONFIG_SGI_PARTITION is not set
 # CONFIG_ULTRIX_PARTITION is not set
@@ -640,6 +778,8 @@ CONFIG_PARTITION_ADVANCED=y
 # CONFIG_EFI_PARTITION is not set
 # CONFIG_SYSV68_PARTITION is not set
 # CONFIG_CMDLINE_PARTITION is not set
+# end of Partition Types
+
 CONFIG_BLOCK_COMPAT=y
 CONFIG_BLK_MQ_VIRTIO=y
 CONFIG_BLK_PM=y
@@ -647,16 +787,11 @@ CONFIG_BLK_PM=y
 #
 # IO Schedulers
 #
-CONFIG_IOSCHED_NOOP=y
-# CONFIG_IOSCHED_DEADLINE is not set
-CONFIG_IOSCHED_CFQ=y
-CONFIG_CFQ_GROUP_IOSCHED=y
-# CONFIG_DEFAULT_CFQ is not set
-CONFIG_DEFAULT_NOOP=y
-CONFIG_DEFAULT_IOSCHED="noop"
 # CONFIG_MQ_IOSCHED_DEADLINE is not set
 # CONFIG_MQ_IOSCHED_KYBER is not set
 # CONFIG_IOSCHED_BFQ is not set
+# end of IO Schedulers
+
 CONFIG_ASN1=y
 CONFIG_INLINE_SPIN_UNLOCK_IRQ=y
 CONFIG_INLINE_READ_UNLOCK=y
@@ -671,6 +806,7 @@ CONFIG_ARCH_USE_QUEUED_SPINLOCKS=y
 CONFIG_QUEUED_SPINLOCKS=y
 CONFIG_ARCH_USE_QUEUED_RWLOCKS=y
 CONFIG_QUEUED_RWLOCKS=y
+CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE=y
 CONFIG_ARCH_HAS_SYNC_CORE_BEFORE_USERMODE=y
 CONFIG_ARCH_HAS_SYSCALL_WRAPPER=y
 CONFIG_FREEZER=y
@@ -685,6 +821,7 @@ CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
 CONFIG_BINFMT_SCRIPT=y
 CONFIG_BINFMT_MISC=y
 CONFIG_COREDUMP=y
+# end of Executable file formats
 
 #
 # Memory Management options
@@ -693,12 +830,11 @@ CONFIG_SELECT_MEMORY_MODEL=y
 CONFIG_SPARSEMEM_MANUAL=y
 CONFIG_SPARSEMEM=y
 CONFIG_NEED_MULTIPLE_NODES=y
-CONFIG_HAVE_MEMORY_PRESENT=y
 CONFIG_SPARSEMEM_EXTREME=y
 CONFIG_SPARSEMEM_VMEMMAP_ENABLE=y
 CONFIG_SPARSEMEM_VMEMMAP=y
-CONFIG_HAVE_MEMBLOCK_NODE_MAP=y
-CONFIG_ARCH_DISCARD_MEMBLOCK=y
+CONFIG_HAVE_FAST_GUP=y
+CONFIG_NUMA_KEEP_MEMINFO=y
 CONFIG_MEMORY_ISOLATION=y
 CONFIG_HAVE_BOOTMEM_INFO_NODE=y
 CONFIG_MEMORY_HOTPLUG=y
@@ -707,9 +843,11 @@ CONFIG_MEMORY_HOTPLUG_SPARSE=y
 CONFIG_MEMORY_HOTREMOVE=y
 CONFIG_SPLIT_PTLOCK_CPUS=4
 CONFIG_MEMORY_BALLOON=y
-CONFIG_BALLOON_COMPACTION=y
+# CONFIG_BALLOON_COMPACTION is not set
 CONFIG_COMPACTION=y
+CONFIG_PAGE_REPORTING=y
 CONFIG_MIGRATION=y
+CONFIG_CONTIG_ALLOC=y
 CONFIG_PHYS_ADDR_T_64BIT=y
 CONFIG_BOUNCE=y
 CONFIG_VIRT_TO_BUS=y
@@ -720,7 +858,6 @@ CONFIG_TRANSPARENT_HUGEPAGE=y
 CONFIG_TRANSPARENT_HUGEPAGE_MADVISE=y
 CONFIG_ARCH_WANTS_THP_SWAP=y
 CONFIG_THP_SWAP=y
-CONFIG_TRANSPARENT_HUGE_PAGECACHE=y
 CONFIG_CLEANCACHE=y
 CONFIG_FRONTSWAP=y
 # CONFIG_CMA is not set
@@ -732,15 +869,19 @@ CONFIG_ZPOOL=y
 CONFIG_GENERIC_EARLY_IOREMAP=y
 # CONFIG_DEFERRED_STRUCT_PAGE_INIT is not set
 # CONFIG_IDLE_PAGE_TRACKING is not set
-CONFIG_ARCH_HAS_ZONE_DEVICE=y
+CONFIG_ARCH_HAS_PTE_DEVMAP=y
 # CONFIG_ZONE_DEVICE is not set
 CONFIG_ARCH_USES_HIGH_VMA_FLAGS=y
 CONFIG_ARCH_HAS_PKEYS=y
 CONFIG_PERCPU_STATS=y
 # CONFIG_GUP_BENCHMARK is not set
+# CONFIG_READ_ONLY_THP_FOR_FS is not set
 CONFIG_ARCH_HAS_PTE_SPECIAL=y
+# end of Memory Management options
+
 CONFIG_NET=y
 CONFIG_NET_INGRESS=y
+CONFIG_SKB_EXTENSIONS=y
 
 #
 # Networking options
@@ -748,11 +889,13 @@ CONFIG_NET_INGRESS=y
 CONFIG_PACKET=y
 # CONFIG_PACKET_DIAG is not set
 CONFIG_UNIX=y
+CONFIG_UNIX_SCM=y
 # CONFIG_UNIX_DIAG is not set
 # CONFIG_TLS is not set
 CONFIG_XFRM=y
 CONFIG_XFRM_ALGO=y
 CONFIG_XFRM_USER=y
+# CONFIG_XFRM_USER_COMPAT is not set
 # CONFIG_XFRM_INTERFACE is not set
 CONFIG_XFRM_SUB_POLICY=y
 CONFIG_XFRM_MIGRATE=y
@@ -778,13 +921,12 @@ CONFIG_IP_MROUTE_MULTIPLE_TABLES=y
 CONFIG_IP_PIMSM_V1=y
 CONFIG_IP_PIMSM_V2=y
 CONFIG_SYN_COOKIES=y
+# CONFIG_NET_IPVTI is not set
 # CONFIG_NET_FOU is not set
 # CONFIG_INET_AH is not set
 # CONFIG_INET_ESP is not set
 # CONFIG_INET_IPCOMP is not set
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
+CONFIG_INET_TABLE_PERTURB_ORDER=16
 # CONFIG_INET_DIAG is not set
 CONFIG_TCP_CONG_ADVANCED=y
 # CONFIG_TCP_CONG_BIC is not set
@@ -816,10 +958,7 @@ CONFIG_IPV6_OPTIMISTIC_DAD=y
 # CONFIG_INET6_IPCOMP is not set
 # CONFIG_IPV6_MIP6 is not set
 # CONFIG_IPV6_ILA is not set
-# CONFIG_INET6_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET6_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET6_XFRM_MODE_BEET is not set
-# CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION is not set
+# CONFIG_IPV6_VTI is not set
 # CONFIG_IPV6_SIT is not set
 # CONFIG_IPV6_TUNNEL is not set
 CONFIG_IPV6_MULTIPLE_TABLES=y
@@ -830,7 +969,9 @@ CONFIG_IPV6_PIMSM_V2=y
 CONFIG_IPV6_SEG6_LWTUNNEL=y
 CONFIG_IPV6_SEG6_HMAC=y
 CONFIG_IPV6_SEG6_BPF=y
+# CONFIG_IPV6_RPL_LWTUNNEL is not set
 CONFIG_NETLABEL=y
+# CONFIG_MPTCP is not set
 CONFIG_NETWORK_SECMARK=y
 CONFIG_NET_PTP_CLASSIFY=y
 CONFIG_NETWORK_PHY_TIMESTAMPING=y
@@ -874,11 +1015,8 @@ CONFIG_NF_CT_PROTO_UDPLITE=y
 # CONFIG_NF_CT_NETLINK is not set
 # CONFIG_NF_CT_NETLINK_TIMEOUT is not set
 CONFIG_NF_NAT=y
-CONFIG_NF_NAT_NEEDED=y
-CONFIG_NF_NAT_PROTO_DCCP=y
-CONFIG_NF_NAT_PROTO_UDPLITE=y
-CONFIG_NF_NAT_PROTO_SCTP=y
 CONFIG_NF_NAT_REDIRECT=y
+CONFIG_NF_NAT_MASQUERADE=y
 CONFIG_NETFILTER_SYNPROXY=y
 # CONFIG_NF_TABLES is not set
 CONFIG_NETFILTER_XTABLES=y
@@ -909,6 +1047,7 @@ CONFIG_NETFILTER_XT_TARGET_NETMAP=y
 # CONFIG_NETFILTER_XT_TARGET_NFQUEUE is not set
 # CONFIG_NETFILTER_XT_TARGET_RATEEST is not set
 CONFIG_NETFILTER_XT_TARGET_REDIRECT=y
+CONFIG_NETFILTER_XT_TARGET_MASQUERADE=y
 # CONFIG_NETFILTER_XT_TARGET_TEE is not set
 # CONFIG_NETFILTER_XT_TARGET_TPROXY is not set
 # CONFIG_NETFILTER_XT_TARGET_SECMARK is not set
@@ -963,6 +1102,8 @@ CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
 # CONFIG_NETFILTER_XT_MATCH_TCPMSS is not set
 # CONFIG_NETFILTER_XT_MATCH_TIME is not set
 # CONFIG_NETFILTER_XT_MATCH_U32 is not set
+# end of Core Netfilter Configuration
+
 # CONFIG_IP_SET is not set
 # CONFIG_IP_VS is not set
 
@@ -976,8 +1117,6 @@ CONFIG_NF_DEFRAG_IPV4=y
 CONFIG_NF_LOG_ARP=y
 CONFIG_NF_LOG_IPV4=y
 CONFIG_NF_REJECT_IPV4=y
-CONFIG_NF_NAT_IPV4=y
-CONFIG_NF_NAT_MASQUERADE_IPV4=y
 CONFIG_IP_NF_IPTABLES=y
 # CONFIG_IP_NF_MATCH_AH is not set
 # CONFIG_IP_NF_MATCH_ECN is not set
@@ -997,6 +1136,7 @@ CONFIG_IP_NF_MANGLE=y
 # CONFIG_IP_NF_RAW is not set
 # CONFIG_IP_NF_SECURITY is not set
 # CONFIG_IP_NF_ARPTABLES is not set
+# end of IP: Netfilter Configuration
 
 #
 # IPv6: Netfilter Configuration
@@ -1006,9 +1146,11 @@ CONFIG_IP_NF_MANGLE=y
 # CONFIG_NF_DUP_IPV6 is not set
 # CONFIG_NF_REJECT_IPV6 is not set
 # CONFIG_NF_LOG_IPV6 is not set
-# CONFIG_NF_NAT_IPV6 is not set
 # CONFIG_IP6_NF_IPTABLES is not set
+# end of IPv6: Netfilter Configuration
+
 CONFIG_NF_DEFRAG_IPV6=y
+# CONFIG_NF_CONNTRACK_BRIDGE is not set
 # CONFIG_BRIDGE_NF_EBTABLES is not set
 # CONFIG_BPFILTER is not set
 # CONFIG_IP_DCCP is not set
@@ -1020,10 +1162,10 @@ CONFIG_NF_DEFRAG_IPV6=y
 CONFIG_STP=y
 CONFIG_BRIDGE=y
 CONFIG_BRIDGE_IGMP_SNOOPING=y
+# CONFIG_BRIDGE_MRP is not set
 CONFIG_HAVE_NET_DSA=y
 # CONFIG_NET_DSA is not set
 # CONFIG_VLAN_8021Q is not set
-# CONFIG_DECNET is not set
 CONFIG_LLC=y
 # CONFIG_LLC2 is not set
 # CONFIG_ATALK is not set
@@ -1066,6 +1208,7 @@ CONFIG_NET_SCHED=y
 # CONFIG_NET_SCH_PIE is not set
 # CONFIG_NET_SCH_INGRESS is not set
 # CONFIG_NET_SCH_PLUG is not set
+# CONFIG_NET_SCH_ETS is not set
 # CONFIG_NET_SCH_DEFAULT is not set
 
 #
@@ -1073,12 +1216,9 @@ CONFIG_NET_SCHED=y
 #
 CONFIG_NET_CLS=y
 # CONFIG_NET_CLS_BASIC is not set
-# CONFIG_NET_CLS_TCINDEX is not set
 # CONFIG_NET_CLS_ROUTE4 is not set
 # CONFIG_NET_CLS_FW is not set
 # CONFIG_NET_CLS_U32 is not set
-# CONFIG_NET_CLS_RSVP is not set
-# CONFIG_NET_CLS_RSVP6 is not set
 # CONFIG_NET_CLS_FLOW is not set
 # CONFIG_NET_CLS_CGROUP is not set
 # CONFIG_NET_CLS_BPF is not set
@@ -1103,12 +1243,16 @@ CONFIG_NET_CLS_ACT=y
 # CONFIG_NET_ACT_SIMP is not set
 # CONFIG_NET_ACT_SKBEDIT is not set
 # CONFIG_NET_ACT_CSUM is not set
+# CONFIG_NET_ACT_MPLS is not set
 # CONFIG_NET_ACT_VLAN is not set
 # CONFIG_NET_ACT_BPF is not set
 # CONFIG_NET_ACT_CONNMARK is not set
+# CONFIG_NET_ACT_CTINFO is not set
 # CONFIG_NET_ACT_SKBMOD is not set
 # CONFIG_NET_ACT_IFE is not set
 # CONFIG_NET_ACT_TUNNEL_KEY is not set
+# CONFIG_NET_ACT_GATE is not set
+# CONFIG_NET_TC_SKB_EXT is not set
 CONFIG_NET_SCH_FIFO=y
 CONFIG_DCB=y
 # CONFIG_DNS_RESOLVER is not set
@@ -1116,6 +1260,7 @@ CONFIG_DCB=y
 # CONFIG_OPENVSWITCH is not set
 CONFIG_VSOCKETS=y
 CONFIG_VSOCKETS_DIAG=y
+CONFIG_VSOCKETS_LOOPBACK=y
 CONFIG_VIRTIO_VSOCKETS=y
 CONFIG_VIRTIO_VSOCKETS_COMMON=y
 # CONFIG_NETLINK_DIAG is not set
@@ -1126,6 +1271,7 @@ CONFIG_MPLS=y
 # CONFIG_HSR is not set
 # CONFIG_NET_SWITCHDEV is not set
 # CONFIG_NET_L3_MASTER_DEV is not set
+# CONFIG_QRTR is not set
 # CONFIG_NET_NCSI is not set
 CONFIG_RPS=y
 CONFIG_RFS_ACCEL=y
@@ -1142,6 +1288,9 @@ CONFIG_NET_FLOW_LIMIT=y
 # Network testing
 #
 # CONFIG_NET_PKTGEN is not set
+# end of Network testing
+# end of Networking options
+
 # CONFIG_HAMRADIO is not set
 # CONFIG_CAN is not set
 # CONFIG_BT is not set
@@ -1163,14 +1312,18 @@ CONFIG_LWTUNNEL_BPF=y
 CONFIG_DST_CACHE=y
 CONFIG_GRO_CELLS=y
 CONFIG_NET_SOCK_MSG=y
-# CONFIG_NET_DEVLINK is not set
-CONFIG_MAY_USE_DEVLINK=y
 CONFIG_FAILOVER=y
+CONFIG_ETHTOOL_NETLINK=y
 CONFIG_HAVE_EBPF_JIT=y
 
 #
 # Device Drivers
 #
+CONFIG_HAVE_EISA=y
+# CONFIG_EISA is not set
+CONFIG_HAVE_PCI=y
+# CONFIG_PCI is not set
+# CONFIG_PCCARD is not set
 
 #
 # Generic Driver Options
@@ -1188,16 +1341,27 @@ CONFIG_PREVENT_FIRMWARE_BUILD=y
 CONFIG_FW_LOADER=y
 CONFIG_EXTRA_FIRMWARE=""
 # CONFIG_FW_LOADER_USER_HELPER is not set
+# CONFIG_FW_LOADER_COMPRESS is not set
+CONFIG_FW_CACHE=y
+# end of Firmware loader
+
 CONFIG_ALLOW_DEV_COREDUMP=y
+# CONFIG_DEBUG_DRIVER is not set
+# CONFIG_DEBUG_DEVRES is not set
+# CONFIG_DEBUG_TEST_DRIVER_REMOVE is not set
 # CONFIG_TEST_ASYNC_DRIVER_PROBE is not set
 CONFIG_GENERIC_CPU_AUTOPROBE=y
 CONFIG_GENERIC_CPU_VULNERABILITIES=y
 CONFIG_DMA_SHARED_BUFFER=y
 # CONFIG_DMA_FENCE_TRACE is not set
+# end of Generic Driver Options
 
 #
 # Bus devices
 #
+# CONFIG_MHI_BUS is not set
+# end of Bus devices
+
 CONFIG_CONNECTOR=y
 CONFIG_PROC_EVENTS=y
 # CONFIG_GNSS is not set
@@ -1205,6 +1369,13 @@ CONFIG_PROC_EVENTS=y
 # CONFIG_OF is not set
 CONFIG_ARCH_MIGHT_HAVE_PC_PARPORT=y
 # CONFIG_PARPORT is not set
+CONFIG_PNP=y
+CONFIG_PNP_DEBUG_MESSAGES=y
+
+#
+# Protocols
+#
+CONFIG_PNPACPI=y
 CONFIG_BLK_DEV=y
 # CONFIG_BLK_DEV_NULL_BLK is not set
 # CONFIG_BLK_DEV_FD is not set
@@ -1217,13 +1388,14 @@ CONFIG_BLK_DEV_LOOP_MIN_COUNT=8
 # CONFIG_CDROM_PKTCDVD is not set
 # CONFIG_ATA_OVER_ETH is not set
 CONFIG_VIRTIO_BLK=y
-# CONFIG_VIRTIO_BLK_SCSI is not set
 # CONFIG_BLK_DEV_RBD is not set
 
 #
 # NVME Support
 #
 # CONFIG_NVME_FC is not set
+# CONFIG_NVME_TCP is not set
+# end of NVME Support
 
 #
 # Misc devices
@@ -1231,57 +1403,27 @@ CONFIG_VIRTIO_BLK=y
 # CONFIG_DUMMY_IRQ is not set
 # CONFIG_ENCLOSURE_SERVICES is not set
 # CONFIG_SRAM is not set
+# CONFIG_XILINX_SDFEC is not set
+# CONFIG_PVPANIC is not set
 # CONFIG_C2PORT is not set
 
 #
 # EEPROM support
 #
 # CONFIG_EEPROM_93CX6 is not set
+# end of EEPROM support
 
 #
 # Texas Instruments shared transport line discipline
 #
+# end of Texas Instruments shared transport line discipline
 
 #
 # Altera FPGA firmware download module (requires I2C)
 #
-
-#
-# Intel MIC & related support
-#
-
-#
-# Intel MIC Bus Driver
-#
-
-#
-# SCIF Bus Driver
-#
-
-#
-# VOP Bus Driver
-#
-
-#
-# Intel MIC Host Driver
-#
-
-#
-# Intel MIC Card Driver
-#
-
-#
-# SCIF Driver
-#
-
-#
-# Intel MIC Coprocessor State Management (COSM) Drivers
-#
-
-#
-# VOP Driver
-#
 # CONFIG_ECHO is not set
+# end of Misc devices
+
 CONFIG_HAVE_IDE=y
 # CONFIG_IDE is not set
 
@@ -1292,7 +1434,6 @@ CONFIG_SCSI_MOD=y
 # CONFIG_RAID_ATTRS is not set
 CONFIG_SCSI=y
 CONFIG_SCSI_DMA=y
-CONFIG_SCSI_MQ_DEFAULT=y
 CONFIG_SCSI_PROC_FS=y
 
 #
@@ -1300,7 +1441,6 @@ CONFIG_SCSI_PROC_FS=y
 #
 # CONFIG_BLK_DEV_SD is not set
 # CONFIG_CHR_DEV_ST is not set
-# CONFIG_CHR_DEV_OSST is not set
 # CONFIG_BLK_DEV_SR is not set
 # CONFIG_CHR_DEV_SG is not set
 # CONFIG_CHR_DEV_SCH is not set
@@ -1317,6 +1457,8 @@ CONFIG_SCSI_ISCSI_ATTRS=y
 # CONFIG_SCSI_SAS_ATTRS is not set
 # CONFIG_SCSI_SAS_LIBSAS is not set
 # CONFIG_SCSI_SRP_ATTRS is not set
+# end of SCSI Transports
+
 CONFIG_SCSI_LOWLEVEL=y
 CONFIG_ISCSI_TCP=y
 # CONFIG_ISCSI_BOOT_SYSFS is not set
@@ -1324,7 +1466,8 @@ CONFIG_ISCSI_TCP=y
 # CONFIG_SCSI_DEBUG is not set
 # CONFIG_SCSI_VIRTIO is not set
 # CONFIG_SCSI_DH is not set
-# CONFIG_SCSI_OSD_INITIATOR is not set
+# end of SCSI device support
+
 # CONFIG_ATA is not set
 # CONFIG_MD is not set
 # CONFIG_TARGET_CORE is not set
@@ -1333,12 +1476,15 @@ CONFIG_NETDEVICES=y
 CONFIG_NET_CORE=y
 # CONFIG_BONDING is not set
 # CONFIG_DUMMY is not set
+# CONFIG_WIREGUARD is not set
 # CONFIG_EQUALIZER is not set
-# CONFIG_IFB is not set
 # CONFIG_NET_TEAM is not set
 # CONFIG_MACVLAN is not set
 # CONFIG_IPVLAN is not set
 # CONFIG_VXLAN is not set
+# CONFIG_GENEVE is not set
+# CONFIG_BAREUDP is not set
+# CONFIG_GTP is not set
 # CONFIG_MACSEC is not set
 # CONFIG_NETCONSOLE is not set
 CONFIG_TUN=y
@@ -1347,16 +1493,21 @@ CONFIG_VETH=y
 CONFIG_VIRTIO_NET=y
 # CONFIG_NLMON is not set
 
-#
-# CAIF transport drivers
-#
-
 #
 # Distributed Switch Architecture drivers
 #
+# end of Distributed Switch Architecture drivers
+
 # CONFIG_ETHERNET is not set
-# CONFIG_MDIO_DEVICE is not set
+# CONFIG_NET_SB1000 is not set
 # CONFIG_PHYLIB is not set
+# CONFIG_MDIO_DEVICE is not set
+
+#
+# PCS device drivers
+#
+# end of PCS device drivers
+
 # CONFIG_PPP is not set
 # CONFIG_SLIP is not set
 
@@ -1369,10 +1520,10 @@ CONFIG_VIRTIO_NET=y
 # Enable WiMAX (Networking options) to see the WiMAX drivers
 #
 # CONFIG_WAN is not set
+# CONFIG_FUJITSU_ES is not set
 # CONFIG_NETDEVSIM is not set
 CONFIG_NET_FAILOVER=y
 # CONFIG_ISDN is not set
-# CONFIG_NVM is not set
 
 #
 # Input device support
@@ -1408,6 +1559,8 @@ CONFIG_INPUT=y
 # CONFIG_SERIO is not set
 CONFIG_ARCH_MIGHT_HAVE_PC_SERIO=y
 # CONFIG_GAMEPORT is not set
+# end of Hardware I/O ports
+# end of Input device support
 
 #
 # Character devices
@@ -1421,11 +1574,7 @@ CONFIG_HW_CONSOLE=y
 CONFIG_VT_HW_CONSOLE_BINDING=y
 CONFIG_UNIX98_PTYS=y
 # CONFIG_LEGACY_PTYS is not set
-# CONFIG_SERIAL_NONSTANDARD is not set
-# CONFIG_N_GSM is not set
-# CONFIG_TRACE_SINK is not set
-CONFIG_DEVMEM=y
-# CONFIG_DEVKMEM is not set
+CONFIG_LDISC_AUTOLOAD=y
 
 #
 # Serial drivers
@@ -1433,6 +1582,8 @@ CONFIG_DEVMEM=y
 CONFIG_SERIAL_EARLYCON=y
 CONFIG_SERIAL_8250=y
 # CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set
+CONFIG_SERIAL_8250_PNP=y
+# CONFIG_SERIAL_8250_16550A_VARIANTS is not set
 # CONFIG_SERIAL_8250_FINTEK is not set
 CONFIG_SERIAL_8250_CONSOLE=y
 CONFIG_SERIAL_8250_DMA=y
@@ -1448,30 +1599,45 @@ CONFIG_SERIAL_8250_RUNTIME_UARTS=1
 # CONFIG_SERIAL_UARTLITE is not set
 CONFIG_SERIAL_CORE=y
 CONFIG_SERIAL_CORE_CONSOLE=y
+# CONFIG_SERIAL_LANTIQ is not set
 # CONFIG_SERIAL_SCCNXP is not set
 # CONFIG_SERIAL_ALTERA_JTAGUART is not set
 # CONFIG_SERIAL_ALTERA_UART is not set
 # CONFIG_SERIAL_ARC is not set
 # CONFIG_SERIAL_FSL_LPUART is not set
+# CONFIG_SERIAL_FSL_LINFLEXUART is not set
+# end of Serial drivers
+
+# CONFIG_SERIAL_NONSTANDARD is not set
+# CONFIG_N_GSM is not set
+# CONFIG_NULL_TTY is not set
+# CONFIG_TRACE_SINK is not set
+CONFIG_HVC_DRIVER=y
 CONFIG_SERIAL_DEV_BUS=y
 CONFIG_SERIAL_DEV_CTRL_TTYPORT=y
-CONFIG_HVC_DRIVER=y
 CONFIG_VIRTIO_CONSOLE=y
 # CONFIG_IPMI_HANDLER is not set
 # CONFIG_HW_RANDOM is not set
-# CONFIG_NVRAM is not set
-# CONFIG_R3964 is not set
 # CONFIG_MWAVE is not set
+CONFIG_DEVMEM=y
+# CONFIG_DEVKMEM is not set
+# CONFIG_NVRAM is not set
 # CONFIG_RAW_DRIVER is not set
+# CONFIG_HPET is not set
 # CONFIG_HANGCHECK_TIMER is not set
 # CONFIG_TCG_TPM is not set
 # CONFIG_TELCLOCK is not set
 # CONFIG_RANDOM_TRUST_CPU is not set
+# CONFIG_RANDOM_TRUST_BOOTLOADER is not set
+# end of Character devices
 
 #
 # I2C support
 #
 # CONFIG_I2C is not set
+# end of I2C support
+
+# CONFIG_I3C is not set
 # CONFIG_SPI is not set
 # CONFIG_SPMI is not set
 # CONFIG_HSI is not set
@@ -1498,10 +1664,12 @@ CONFIG_PTP_1588_CLOCK=y
 # Enable PHYLIB and NETWORK_PHY_TIMESTAMPING to see the additional clocks.
 #
 CONFIG_PTP_1588_CLOCK_KVM=y
+# CONFIG_PTP_1588_CLOCK_VMW is not set
+# end of PTP clock support
+
 # CONFIG_PINCTRL is not set
 # CONFIG_GPIOLIB is not set
 # CONFIG_W1 is not set
-# CONFIG_POWER_AVS is not set
 CONFIG_POWER_RESET=y
 # CONFIG_POWER_RESET_RESTART is not set
 CONFIG_POWER_SUPPLY=y
@@ -1514,24 +1682,30 @@ CONFIG_POWER_SUPPLY=y
 # CONFIG_CHARGER_MAX8903 is not set
 # CONFIG_HWMON is not set
 CONFIG_THERMAL=y
+# CONFIG_THERMAL_NETLINK is not set
 # CONFIG_THERMAL_STATISTICS is not set
 CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=0
 CONFIG_THERMAL_WRITABLE_TRIPS=y
 CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE=y
 # CONFIG_THERMAL_DEFAULT_GOV_FAIR_SHARE is not set
 # CONFIG_THERMAL_DEFAULT_GOV_USER_SPACE is not set
-# CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR is not set
 CONFIG_THERMAL_GOV_FAIR_SHARE=y
 CONFIG_THERMAL_GOV_STEP_WISE=y
 # CONFIG_THERMAL_GOV_BANG_BANG is not set
 CONFIG_THERMAL_GOV_USER_SPACE=y
-# CONFIG_THERMAL_GOV_POWER_ALLOCATOR is not set
 # CONFIG_THERMAL_EMULATION is not set
+
+#
+# Intel thermal drivers
+#
 # CONFIG_INTEL_POWERCLAMP is not set
 
 #
 # ACPI INT340X thermal drivers
 #
+# end of ACPI INT340X thermal drivers
+# end of Intel thermal drivers
+
 # CONFIG_WATCHDOG is not set
 CONFIG_SSB_POSSIBLE=y
 # CONFIG_SSB is not set
@@ -1541,51 +1715,58 @@ CONFIG_BCMA_POSSIBLE=y
 #
 # Multifunction device drivers
 #
-# CONFIG_MFD_AT91_USART is not set
-# CONFIG_MFD_CROS_EC is not set
 # CONFIG_MFD_MADERA is not set
 # CONFIG_HTC_PASIC3 is not set
+# CONFIG_MFD_INTEL_LPSS_ACPI is not set
+# CONFIG_MFD_INTEL_PMC_BXT is not set
 # CONFIG_MFD_KEMPLD is not set
 # CONFIG_MFD_MT6397 is not set
 # CONFIG_MFD_SM501 is not set
 # CONFIG_ABX500_CORE is not set
 # CONFIG_MFD_SYSCON is not set
 # CONFIG_MFD_TI_AM335X_TSCADC is not set
+# CONFIG_MFD_TQMX86 is not set
 # CONFIG_RAVE_SP_CORE is not set
+# end of Multifunction device drivers
+
 # CONFIG_REGULATOR is not set
 # CONFIG_RC_CORE is not set
+# CONFIG_MEDIA_CEC_SUPPORT is not set
 # CONFIG_MEDIA_SUPPORT is not set
 
 #
 # Graphics support
 #
 # CONFIG_DRM is not set
-# CONFIG_DRM_DP_CEC is not set
 
 #
-# ACP (Audio CoProcessor) Configuration
+# ARM devices
 #
+# end of ARM devices
 
 #
-# AMD Library routines
+# Frame buffer Devices
 #
+# CONFIG_FB is not set
+# end of Frame buffer Devices
 
 #
-# Frame buffer Devices
+# Backlight & LCD device support
 #
-# CONFIG_FB is not set
-CONFIG_BACKLIGHT_LCD_SUPPORT=y
 # CONFIG_LCD_CLASS_DEVICE is not set
 # CONFIG_BACKLIGHT_CLASS_DEVICE is not set
+# end of Backlight & LCD device support
 
 #
 # Console display driver support
 #
 CONFIG_VGA_CONSOLE=y
-# CONFIG_VGACON_SOFT_SCROLLBACK is not set
 CONFIG_DUMMY_CONSOLE=y
 CONFIG_DUMMY_CONSOLE_COLUMNS=80
 CONFIG_DUMMY_CONSOLE_ROWS=25
+# end of Console display driver support
+# end of Graphics support
+
 # CONFIG_SOUND is not set
 
 #
@@ -1606,8 +1787,8 @@ CONFIG_HIDRAW=y
 # CONFIG_HID_AUREAL is not set
 # CONFIG_HID_BELKIN is not set
 # CONFIG_HID_CHERRY is not set
-# CONFIG_HID_CHICONY is not set
 # CONFIG_HID_COUGAR is not set
+# CONFIG_HID_MACALLY is not set
 # CONFIG_HID_CMEDIA is not set
 # CONFIG_HID_CYPRESS is not set
 # CONFIG_HID_DRAGONRISE is not set
@@ -1616,9 +1797,12 @@ CONFIG_HIDRAW=y
 # CONFIG_HID_EZKEY is not set
 # CONFIG_HID_GEMBIRD is not set
 # CONFIG_HID_GFRM is not set
+# CONFIG_HID_GLORIOUS is not set
+# CONFIG_HID_VIVALDI is not set
 # CONFIG_HID_KEYTOUCH is not set
 # CONFIG_HID_KYE is not set
 # CONFIG_HID_WALTOP is not set
+# CONFIG_HID_VIEWSONIC is not set
 # CONFIG_HID_GYRATION is not set
 # CONFIG_HID_ICADE is not set
 # CONFIG_HID_ITE is not set
@@ -1627,8 +1811,8 @@ CONFIG_HIDRAW=y
 # CONFIG_HID_KENSINGTON is not set
 # CONFIG_HID_LCPOWER is not set
 # CONFIG_HID_LENOVO is not set
-# CONFIG_HID_LOGITECH is not set
 # CONFIG_HID_MAGICMOUSE is not set
+# CONFIG_HID_MALTRON is not set
 # CONFIG_HID_MAYFLASH is not set
 CONFIG_HID_REDRAGON=y
 # CONFIG_HID_MICROSOFT is not set
@@ -1642,7 +1826,6 @@ CONFIG_HID_REDRAGON=y
 # CONFIG_HID_PLANTRONICS is not set
 # CONFIG_HID_PRIMAX is not set
 # CONFIG_HID_SAITEK is not set
-# CONFIG_HID_SAMSUNG is not set
 # CONFIG_HID_SPEEDLINK is not set
 # CONFIG_HID_STEAM is not set
 # CONFIG_HID_STEELSERIES is not set
@@ -1659,9 +1842,11 @@ CONFIG_HID_REDRAGON=y
 # CONFIG_HID_ZYDACRON is not set
 # CONFIG_HID_SENSOR_HUB is not set
 # CONFIG_HID_ALPS is not set
+# end of Special HID drivers
+# end of HID support
+
 CONFIG_USB_OHCI_LITTLE_ENDIAN=y
 # CONFIG_USB_SUPPORT is not set
-# CONFIG_UWB is not set
 # CONFIG_MMC is not set
 # CONFIG_MEMSTICK is not set
 # CONFIG_NEW_LEDS is not set
@@ -1679,11 +1864,13 @@ CONFIG_DMADEVICES=y
 #
 # DMA Devices
 #
+CONFIG_DMA_ACPI=y
 # CONFIG_ALTERA_MSGDMA is not set
 # CONFIG_INTEL_IDMA64 is not set
 # CONFIG_QCOM_HIDMA_MGMT is not set
 # CONFIG_QCOM_HIDMA is not set
 # CONFIG_DW_DMAC is not set
+# CONFIG_SF_PDMA is not set
 
 #
 # DMABUF options
@@ -1691,51 +1878,89 @@ CONFIG_DMADEVICES=y
 CONFIG_SYNC_FILE=y
 # CONFIG_SW_SYNC is not set
 # CONFIG_UDMABUF is not set
+# CONFIG_DMABUF_MOVE_NOTIFY is not set
+# CONFIG_DMABUF_SELFTESTS is not set
+# CONFIG_DMABUF_HEAPS is not set
+# end of DMABUF options
+
 CONFIG_AUXDISPLAY=y
 # CONFIG_IMG_ASCII_LCD is not set
+# CONFIG_CHARLCD_BL_OFF is not set
+# CONFIG_CHARLCD_BL_ON is not set
+CONFIG_CHARLCD_BL_FLASH=y
 # CONFIG_UIO is not set
+# CONFIG_VFIO is not set
 CONFIG_VIRT_DRIVERS=y
 CONFIG_VIRTIO=y
 CONFIG_VIRTIO_MENU=y
 CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_MEM=m
 # CONFIG_VIRTIO_INPUT is not set
 CONFIG_VIRTIO_MMIO=y
 CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
+# CONFIG_VDPA is not set
+CONFIG_VHOST_MENU=y
+# CONFIG_VHOST_NET is not set
+# CONFIG_VHOST_VSOCK is not set
+# CONFIG_VHOST_CROSS_ENDIAN_LEGACY is not set
 
 #
 # Microsoft Hyper-V guest support
 #
+# CONFIG_HYPERV is not set
+# end of Microsoft Hyper-V guest support
+
+# CONFIG_GREYBUS is not set
 CONFIG_STAGING=y
 # CONFIG_COMEDI is not set
-
-#
-# Speakup console speech
-#
-# CONFIG_SPEAKUP is not set
 # CONFIG_STAGING_MEDIA is not set
 
 #
 # Android
 #
+# end of Android
+
 # CONFIG_GS_FPGABOOT is not set
 # CONFIG_UNISYSSPAR is not set
-# CONFIG_MOST is not set
-# CONFIG_GREYBUS is not set
 
 #
 # Gasket devices
 #
-# CONFIG_XIL_AXIS_FIFO is not set
-# CONFIG_EROFS_FS is not set
+# end of Gasket devices
+
+# CONFIG_FIELDBUS_DEV is not set
 CONFIG_X86_PLATFORM_DEVICES=y
+# CONFIG_ACPI_WMI is not set
+# CONFIG_ACERHDF is not set
+# CONFIG_ACER_WIRELESS is not set
+# CONFIG_ASUS_WIRELESS is not set
 # CONFIG_DCDBAS is not set
 # CONFIG_DELL_SMBIOS is not set
 # CONFIG_DELL_RBU is not set
+# CONFIG_DELL_SMO8800 is not set
+# CONFIG_FUJITSU_TABLET is not set
+# CONFIG_GPD_POCKET_FAN is not set
+# CONFIG_HP_WIRELESS is not set
 # CONFIG_SENSORS_HDAPS is not set
-# CONFIG_INTEL_PUNIT_IPC is not set
+# CONFIG_INTEL_HID_EVENT is not set
+# CONFIG_INTEL_MENLOW is not set
+# CONFIG_INTEL_VBTN is not set
+# CONFIG_SURFACE_PRO3_BUTTON is not set
+# CONFIG_SAMSUNG_Q10 is not set
+# CONFIG_TOSHIBA_BT_RFKILL is not set
+# CONFIG_TOSHIBA_HAPS is not set
+# CONFIG_ACPI_CMPC is not set
+# CONFIG_SYSTEM76_ACPI is not set
+# CONFIG_TOPSTAR_LAPTOP is not set
+# CONFIG_INTEL_RST is not set
+# CONFIG_INTEL_SMARTCONNECT is not set
 CONFIG_INTEL_TURBO_MAX_3=y
+# CONFIG_INTEL_UNCORE_FREQ_CONTROL is not set
+# CONFIG_INTEL_PUNIT_IPC is not set
+# CONFIG_INTEL_SCU_PLATFORM is not set
 # CONFIG_CHROME_PLATFORMS is not set
 # CONFIG_MELLANOX_PLATFORM is not set
+# CONFIG_COMMON_CLK is not set
 # CONFIG_HWSPINLOCK is not set
 
 #
@@ -1744,25 +1969,33 @@ CONFIG_INTEL_TURBO_MAX_3=y
 CONFIG_CLKEVT_I8253=y
 CONFIG_I8253_LOCK=y
 CONFIG_CLKBLD_I8253=y
+# end of Clock Source drivers
+
 CONFIG_MAILBOX=y
+CONFIG_PCC=y
 # CONFIG_ALTERA_MBOX is not set
 CONFIG_IOMMU_SUPPORT=y
 
 #
 # Generic IOMMU Pagetable Support
 #
+# end of Generic IOMMU Pagetable Support
+
 # CONFIG_IOMMU_DEBUGFS is not set
 
 #
 # Remoteproc drivers
 #
 # CONFIG_REMOTEPROC is not set
+# end of Remoteproc drivers
 
 #
 # Rpmsg drivers
 #
 # CONFIG_RPMSG_QCOM_GLINK_RPM is not set
 # CONFIG_RPMSG_VIRTIO is not set
+# end of Rpmsg drivers
+
 # CONFIG_SOUNDWIRE is not set
 
 #
@@ -1772,28 +2005,42 @@ CONFIG_IOMMU_SUPPORT=y
 #
 # Amlogic SoC drivers
 #
+# end of Amlogic SoC drivers
+
+#
+# Aspeed SoC drivers
+#
+# end of Aspeed SoC drivers
 
 #
 # Broadcom SoC drivers
 #
+# end of Broadcom SoC drivers
 
 #
 # NXP/Freescale QorIQ SoC drivers
 #
+# end of NXP/Freescale QorIQ SoC drivers
 
 #
 # i.MX SoC drivers
 #
+# end of i.MX SoC drivers
 
 #
 # Qualcomm SoC drivers
 #
+# end of Qualcomm SoC drivers
+
 # CONFIG_SOC_TI is not set
 
 #
 # Xilinx SoC drivers
 #
 # CONFIG_XILINX_VCU is not set
+# end of Xilinx SoC drivers
+# end of SOC (System On Chip) specific Drivers
+
 # CONFIG_PM_DEVFREQ is not set
 # CONFIG_EXTCON is not set
 # CONFIG_MEMORY is not set
@@ -1803,10 +2050,10 @@ CONFIG_IOMMU_SUPPORT=y
 #
 # IRQ chip support
 #
-CONFIG_ARM_GIC_MAX_NR=1
+# end of IRQ chip support
+
 # CONFIG_IPACK_BUS is not set
 # CONFIG_RESET_CONTROLLER is not set
-# CONFIG_FMC is not set
 
 #
 # PHY Subsystem
@@ -1815,18 +2062,25 @@ CONFIG_ARM_GIC_MAX_NR=1
 # CONFIG_BCM_KONA_USB2_PHY is not set
 # CONFIG_PHY_PXA_28NM_HSIC is not set
 # CONFIG_PHY_PXA_28NM_USB2 is not set
+# CONFIG_PHY_INTEL_LGM_EMMC is not set
+# end of PHY Subsystem
+
 # CONFIG_POWERCAP is not set
 # CONFIG_MCB is not set
 
 #
 # Performance monitor support
 #
+# end of Performance monitor support
+
 CONFIG_RAS=y
 
 #
 # Android
 #
 # CONFIG_ANDROID is not set
+# end of Android
+
 # CONFIG_LIBNVDIMM is not set
 # CONFIG_DAX is not set
 # CONFIG_NVMEM is not set
@@ -1836,14 +2090,22 @@ CONFIG_RAS=y
 #
 # CONFIG_STM is not set
 # CONFIG_INTEL_TH is not set
+# end of HW tracing support
+
 # CONFIG_FPGA is not set
+# CONFIG_TEE is not set
+# CONFIG_UNISYS_VISORBUS is not set
 # CONFIG_SIOX is not set
 # CONFIG_SLIMBUS is not set
+# CONFIG_INTERCONNECT is not set
+# CONFIG_COUNTER is not set
+# end of Device Drivers
 
 #
 # File systems
 #
 CONFIG_DCACHE_WORD_ACCESS=y
+# CONFIG_VALIDATE_FS_PARSER is not set
 CONFIG_FS_IOMAP=y
 # CONFIG_EXT2_FS is not set
 # CONFIG_EXT3_FS is not set
@@ -1851,8 +2113,6 @@ CONFIG_EXT4_FS=y
 CONFIG_EXT4_USE_FOR_EXT2=y
 CONFIG_EXT4_FS_POSIX_ACL=y
 CONFIG_EXT4_FS_SECURITY=y
-CONFIG_EXT4_ENCRYPTION=y
-CONFIG_EXT4_FS_ENCRYPTION=y
 CONFIG_EXT4_DEBUG=y
 CONFIG_JBD2=y
 CONFIG_JBD2_DEBUG=y
@@ -1861,7 +2121,13 @@ CONFIG_FS_MBCACHE=y
 # CONFIG_JFS_FS is not set
 # CONFIG_XFS_FS is not set
 # CONFIG_GFS2_FS is not set
-# CONFIG_BTRFS_FS is not set
+CONFIG_BTRFS_FS=y
+# CONFIG_BTRFS_FS_POSIX_ACL is not set
+# CONFIG_BTRFS_FS_CHECK_INTEGRITY is not set
+# CONFIG_BTRFS_FS_RUN_SANITY_TESTS is not set
+# CONFIG_BTRFS_DEBUG is not set
+# CONFIG_BTRFS_ASSERT is not set
+# CONFIG_BTRFS_FS_REF_VERIFY is not set
 # CONFIG_NILFS2_FS is not set
 # CONFIG_F2FS_FS is not set
 # CONFIG_FS_DAX is not set
@@ -1871,6 +2137,8 @@ CONFIG_EXPORTFS=y
 CONFIG_FILE_LOCKING=y
 CONFIG_MANDATORY_FILE_LOCKING=y
 CONFIG_FS_ENCRYPTION=y
+CONFIG_FS_ENCRYPTION_ALGS=y
+# CONFIG_FS_VERITY is not set
 CONFIG_FSNOTIFY=y
 CONFIG_DNOTIFY=y
 CONFIG_INOTIFY_USER=y
@@ -1883,7 +2151,6 @@ CONFIG_QUOTA_NETLINK_INTERFACE=y
 # CONFIG_QFMT_V1 is not set
 # CONFIG_QFMT_V2 is not set
 CONFIG_QUOTACTL=y
-CONFIG_QUOTACTL_COMPAT=y
 # CONFIG_AUTOFS4_FS is not set
 # CONFIG_AUTOFS_FS is not set
 # CONFIG_FUSE_FS is not set
@@ -1898,19 +2165,25 @@ CONFIG_OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW=y
 # Caches
 #
 # CONFIG_FSCACHE is not set
+# end of Caches
 
 #
 # CD-ROM/DVD Filesystems
 #
-# CONFIG_ISO9660_FS is not set
+CONFIG_ISO9660_FS=y
+# CONFIG_JOLIET is not set
+# CONFIG_ZISOFS is not set
 # CONFIG_UDF_FS is not set
+# end of CD-ROM/DVD Filesystems
 
 #
-# DOS/FAT/NT Filesystems
+# DOS/FAT/EXFAT/NT Filesystems
 #
 # CONFIG_MSDOS_FS is not set
 # CONFIG_VFAT_FS is not set
+# CONFIG_EXFAT_FS is not set
 # CONFIG_NTFS_FS is not set
+# end of DOS/FAT/EXFAT/NT Filesystems
 
 #
 # Pseudo filesystems
@@ -1920,16 +2193,20 @@ CONFIG_PROC_KCORE=y
 CONFIG_PROC_SYSCTL=y
 CONFIG_PROC_PAGE_MONITOR=y
 CONFIG_PROC_CHILDREN=y
+CONFIG_PROC_PID_ARCH_STATUS=y
 CONFIG_KERNFS=y
 CONFIG_SYSFS=y
 CONFIG_TMPFS=y
 CONFIG_TMPFS_POSIX_ACL=y
 CONFIG_TMPFS_XATTR=y
+# CONFIG_TMPFS_INODE64 is not set
 CONFIG_HUGETLBFS=y
 CONFIG_HUGETLB_PAGE=y
 CONFIG_MEMFD_CREATE=y
 CONFIG_ARCH_HAS_GIGANTIC_PAGE=y
 # CONFIG_CONFIGFS_FS is not set
+# end of Pseudo filesystems
+
 CONFIG_MISC_FILESYSTEMS=y
 # CONFIG_ORANGEFS_FS is not set
 # CONFIG_ADFS_FS is not set
@@ -1978,6 +2255,7 @@ CONFIG_PSTORE_COMPRESS_DEFAULT="deflate"
 # CONFIG_PSTORE_RAM is not set
 # CONFIG_SYSV_FS is not set
 # CONFIG_UFS_FS is not set
+# CONFIG_EROFS_FS is not set
 CONFIG_NETWORK_FILESYSTEMS=y
 # CONFIG_NFS_FS is not set
 # CONFIG_NFSD is not set
@@ -2036,14 +2314,16 @@ CONFIG_NLS_DEFAULT="utf8"
 # CONFIG_NLS_MAC_ROMANIAN is not set
 # CONFIG_NLS_MAC_TURKISH is not set
 # CONFIG_NLS_UTF8 is not set
+# CONFIG_UNICODE is not set
+CONFIG_IO_WQ=y
+# end of File systems
 
 #
 # Security options
 #
 CONFIG_KEYS=y
-CONFIG_KEYS_COMPAT=y
+# CONFIG_KEYS_REQUEST_CACHE is not set
 CONFIG_PERSISTENT_KEYRINGS=y
-# CONFIG_BIG_KEYS is not set
 CONFIG_ENCRYPTED_KEYS=y
 # CONFIG_KEY_DH_OPERATIONS is not set
 # CONFIG_SECURITY_DMESG_RESTRICT is not set
@@ -2051,7 +2331,6 @@ CONFIG_SECURITY=y
 CONFIG_SECURITY_WRITABLE_HOOKS=y
 CONFIG_SECURITYFS=y
 CONFIG_SECURITY_NETWORK=y
-CONFIG_PAGE_TABLE_ISOLATION=y
 CONFIG_SECURITY_NETWORK_XFRM=y
 # CONFIG_SECURITY_PATH is not set
 CONFIG_LSM_MMAP_MIN_ADDR=65536
@@ -2061,16 +2340,19 @@ CONFIG_FORTIFY_SOURCE=y
 # CONFIG_STATIC_USERMODEHELPER is not set
 CONFIG_SECURITY_SELINUX=y
 CONFIG_SECURITY_SELINUX_BOOTPARAM=y
-CONFIG_SECURITY_SELINUX_BOOTPARAM_VALUE=1
 CONFIG_SECURITY_SELINUX_DISABLE=y
 CONFIG_SECURITY_SELINUX_DEVELOP=y
 CONFIG_SECURITY_SELINUX_AVC_STATS=y
 CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE=1
+CONFIG_SECURITY_SELINUX_SIDTAB_HASH_BITS=9
+CONFIG_SECURITY_SELINUX_SID2STR_CACHE_SIZE=256
 # CONFIG_SECURITY_SMACK is not set
 # CONFIG_SECURITY_TOMOYO is not set
 # CONFIG_SECURITY_APPARMOR is not set
 # CONFIG_SECURITY_LOADPIN is not set
 # CONFIG_SECURITY_YAMA is not set
+# CONFIG_SECURITY_SAFESETID is not set
+# CONFIG_SECURITY_LOCKDOWN_LSM is not set
 CONFIG_INTEGRITY=y
 # CONFIG_INTEGRITY_SIGNATURE is not set
 CONFIG_INTEGRITY_AUDIT=y
@@ -2078,7 +2360,28 @@ CONFIG_INTEGRITY_AUDIT=y
 # CONFIG_EVM is not set
 CONFIG_DEFAULT_SECURITY_SELINUX=y
 # CONFIG_DEFAULT_SECURITY_DAC is not set
-CONFIG_DEFAULT_SECURITY="selinux"
+CONFIG_LSM="lockdown,yama,loadpin,safesetid,integrity,selinux,smack,tomoyo,apparmor,bpf"
+
+#
+# Kernel hardening options
+#
+
+#
+# Memory initialization
+#
+CONFIG_CC_HAS_AUTO_VAR_INIT_PATTERN=y
+CONFIG_CC_HAS_AUTO_VAR_INIT_ZERO_BARE=y
+CONFIG_CC_HAS_AUTO_VAR_INIT_ZERO=y
+CONFIG_INIT_STACK_NONE=y
+# CONFIG_INIT_STACK_ALL_PATTERN is not set
+# CONFIG_INIT_STACK_ALL_ZERO is not set
+# CONFIG_INIT_ON_ALLOC_DEFAULT_ON is not set
+# CONFIG_INIT_ON_FREE_DEFAULT_ON is not set
+# end of Memory initialization
+# end of Kernel hardening options
+# end of Security options
+
+CONFIG_XOR_BLOCKS=y
 CONFIG_CRYPTO=y
 
 #
@@ -2088,8 +2391,8 @@ CONFIG_CRYPTO_ALGAPI=y
 CONFIG_CRYPTO_ALGAPI2=y
 CONFIG_CRYPTO_AEAD=y
 CONFIG_CRYPTO_AEAD2=y
-CONFIG_CRYPTO_BLKCIPHER=y
-CONFIG_CRYPTO_BLKCIPHER2=y
+CONFIG_CRYPTO_SKCIPHER=y
+CONFIG_CRYPTO_SKCIPHER2=y
 CONFIG_CRYPTO_HASH=y
 CONFIG_CRYPTO_HASH2=y
 CONFIG_CRYPTO_RNG=y
@@ -2100,22 +2403,29 @@ CONFIG_CRYPTO_AKCIPHER=y
 CONFIG_CRYPTO_KPP2=y
 CONFIG_CRYPTO_KPP=y
 CONFIG_CRYPTO_ACOMP2=y
-CONFIG_CRYPTO_RSA=y
-CONFIG_CRYPTO_DH=y
-CONFIG_CRYPTO_ECDH=y
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_MANAGER2=y
 # CONFIG_CRYPTO_USER is not set
 CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=y
-# CONFIG_CRYPTO_GF128MUL is not set
 CONFIG_CRYPTO_NULL=y
 CONFIG_CRYPTO_NULL2=y
 # CONFIG_CRYPTO_PCRYPT is not set
-CONFIG_CRYPTO_WORKQUEUE=y
 # CONFIG_CRYPTO_CRYPTD is not set
 # CONFIG_CRYPTO_AUTHENC is not set
 # CONFIG_CRYPTO_TEST is not set
 
+#
+# Public-key cryptography
+#
+CONFIG_CRYPTO_RSA=y
+CONFIG_CRYPTO_DH=y
+CONFIG_CRYPTO_ECC=y
+CONFIG_CRYPTO_ECDH=y
+# CONFIG_CRYPTO_ECRDSA is not set
+# CONFIG_CRYPTO_SM2 is not set
+# CONFIG_CRYPTO_CURVE25519 is not set
+# CONFIG_CRYPTO_CURVE25519_X86 is not set
+
 #
 # Authenticated Encryption with Associated Data
 #
@@ -2123,16 +2433,7 @@ CONFIG_CRYPTO_WORKQUEUE=y
 # CONFIG_CRYPTO_GCM is not set
 # CONFIG_CRYPTO_CHACHA20POLY1305 is not set
 # CONFIG_CRYPTO_AEGIS128 is not set
-# CONFIG_CRYPTO_AEGIS128L is not set
-# CONFIG_CRYPTO_AEGIS256 is not set
 # CONFIG_CRYPTO_AEGIS128_AESNI_SSE2 is not set
-# CONFIG_CRYPTO_AEGIS128L_AESNI_SSE2 is not set
-# CONFIG_CRYPTO_AEGIS256_AESNI_SSE2 is not set
-# CONFIG_CRYPTO_MORUS640 is not set
-# CONFIG_CRYPTO_MORUS640_SSE2 is not set
-# CONFIG_CRYPTO_MORUS1280 is not set
-# CONFIG_CRYPTO_MORUS1280_SSE2 is not set
-# CONFIG_CRYPTO_MORUS1280_AVX2 is not set
 CONFIG_CRYPTO_SEQIV=y
 # CONFIG_CRYPTO_ECHAINIV is not set
 
@@ -2149,6 +2450,10 @@ CONFIG_CRYPTO_ECB=y
 # CONFIG_CRYPTO_PCBC is not set
 CONFIG_CRYPTO_XTS=y
 # CONFIG_CRYPTO_KEYWRAP is not set
+# CONFIG_CRYPTO_NHPOLY1305_SSE2 is not set
+# CONFIG_CRYPTO_NHPOLY1305_AVX2 is not set
+# CONFIG_CRYPTO_ADIANTUM is not set
+# CONFIG_CRYPTO_ESSIV is not set
 
 #
 # Hash modes
@@ -2165,6 +2470,10 @@ CONFIG_CRYPTO_CRC32C=y
 # CONFIG_CRYPTO_CRC32C_INTEL is not set
 # CONFIG_CRYPTO_CRC32 is not set
 # CONFIG_CRYPTO_CRC32_PCLMUL is not set
+CONFIG_CRYPTO_XXHASH=y
+CONFIG_CRYPTO_BLAKE2B=y
+# CONFIG_CRYPTO_BLAKE2S is not set
+# CONFIG_CRYPTO_BLAKE2S_X86 is not set
 CONFIG_CRYPTO_CRCT10DIF=y
 CONFIG_CRYPTO_CRCT10DIF_PCLMUL=y
 # CONFIG_CRYPTO_GHASH is not set
@@ -2185,6 +2494,7 @@ CONFIG_CRYPTO_SHA256=y
 CONFIG_CRYPTO_SHA512=y
 # CONFIG_CRYPTO_SHA3 is not set
 # CONFIG_CRYPTO_SM3 is not set
+# CONFIG_CRYPTO_STREEBOG is not set
 # CONFIG_CRYPTO_TGR192 is not set
 # CONFIG_CRYPTO_WP512 is not set
 # CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL is not set
@@ -2194,10 +2504,7 @@ CONFIG_CRYPTO_SHA512=y
 #
 CONFIG_CRYPTO_AES=y
 CONFIG_CRYPTO_AES_TI=y
-# CONFIG_CRYPTO_AES_X86_64 is not set
 # CONFIG_CRYPTO_AES_NI_INTEL is not set
-# CONFIG_CRYPTO_ANUBIS is not set
-# CONFIG_CRYPTO_ARC4 is not set
 # CONFIG_CRYPTO_BLOWFISH is not set
 # CONFIG_CRYPTO_BLOWFISH_X86_64 is not set
 # CONFIG_CRYPTO_CAMELLIA is not set
@@ -2211,17 +2518,14 @@ CONFIG_CRYPTO_AES_TI=y
 # CONFIG_CRYPTO_DES is not set
 # CONFIG_CRYPTO_DES3_EDE_X86_64 is not set
 # CONFIG_CRYPTO_FCRYPT is not set
-# CONFIG_CRYPTO_KHAZAD is not set
 # CONFIG_CRYPTO_SALSA20 is not set
 # CONFIG_CRYPTO_CHACHA20 is not set
 # CONFIG_CRYPTO_CHACHA20_X86_64 is not set
-# CONFIG_CRYPTO_SEED is not set
 # CONFIG_CRYPTO_SERPENT is not set
 # CONFIG_CRYPTO_SERPENT_SSE2_X86_64 is not set
 # CONFIG_CRYPTO_SERPENT_AVX_X86_64 is not set
 # CONFIG_CRYPTO_SERPENT_AVX2_X86_64 is not set
 # CONFIG_CRYPTO_SM4 is not set
-# CONFIG_CRYPTO_TEA is not set
 # CONFIG_CRYPTO_TWOFISH is not set
 # CONFIG_CRYPTO_TWOFISH_X86_64 is not set
 # CONFIG_CRYPTO_TWOFISH_X86_64_3WAY is not set
@@ -2271,19 +2575,42 @@ CONFIG_SYSTEM_TRUSTED_KEYS=""
 # CONFIG_SECONDARY_TRUSTED_KEYRING is not set
 CONFIG_SYSTEM_BLACKLIST_KEYRING=y
 CONFIG_SYSTEM_BLACKLIST_HASH_LIST=""
+# CONFIG_SYSTEM_REVOCATION_LIST is not set
+# end of Certificates for signature checking
 
 #
 # Library routines
 #
+CONFIG_RAID6_PQ=y
+CONFIG_RAID6_PQ_BENCHMARK=y
+# CONFIG_PACKING is not set
 CONFIG_BITREVERSE=y
 CONFIG_GENERIC_STRNCPY_FROM_USER=y
 CONFIG_GENERIC_STRNLEN_USER=y
 CONFIG_GENERIC_NET_UTILS=y
 CONFIG_GENERIC_FIND_FIRST_BIT=y
+# CONFIG_CORDIC is not set
+# CONFIG_PRIME_NUMBERS is not set
 CONFIG_GENERIC_PCI_IOMAP=y
 CONFIG_GENERIC_IOMAP=y
 CONFIG_ARCH_USE_CMPXCHG_LOCKREF=y
 CONFIG_ARCH_HAS_FAST_MULTIPLIER=y
+CONFIG_ARCH_USE_SYM_ANNOTATIONS=y
+
+#
+# Crypto library routines
+#
+CONFIG_CRYPTO_LIB_AES=y
+CONFIG_CRYPTO_LIB_BLAKE2S_GENERIC=y
+# CONFIG_CRYPTO_LIB_CHACHA is not set
+# CONFIG_CRYPTO_LIB_CURVE25519 is not set
+CONFIG_CRYPTO_LIB_POLY1305_RSIZE=11
+# CONFIG_CRYPTO_LIB_POLY1305 is not set
+# CONFIG_CRYPTO_LIB_CHACHA20POLY1305 is not set
+CONFIG_CRYPTO_LIB_SHA256=y
+# end of Crypto library routines
+
+CONFIG_LIB_MEMNEQ=y
 CONFIG_CRC_CCITT=y
 CONFIG_CRC16=y
 CONFIG_CRC_T10DIF=y
@@ -2306,6 +2633,7 @@ CONFIG_ZLIB_DEFLATE=y
 CONFIG_LZO_COMPRESS=y
 CONFIG_LZO_DECOMPRESS=y
 CONFIG_LZ4_DECOMPRESS=y
+CONFIG_ZSTD_COMPRESS=y
 CONFIG_ZSTD_DECOMPRESS=y
 CONFIG_XZ_DEC=y
 CONFIG_XZ_DEC_X86=y
@@ -2322,6 +2650,8 @@ CONFIG_DECOMPRESS_LZMA=y
 CONFIG_DECOMPRESS_XZ=y
 CONFIG_DECOMPRESS_LZO=y
 CONFIG_DECOMPRESS_LZ4=y
+CONFIG_DECOMPRESS_ZSTD=y
+CONFIG_GENERIC_ALLOCATOR=y
 CONFIG_XARRAY_MULTI=y
 CONFIG_ASSOCIATIVE_ARRAY=y
 CONFIG_HAS_IOMEM=y
@@ -2330,24 +2660,29 @@ CONFIG_HAS_DMA=y
 CONFIG_NEED_SG_DMA_LENGTH=y
 CONFIG_NEED_DMA_MAP_STATE=y
 CONFIG_ARCH_DMA_ADDR_T_64BIT=y
-CONFIG_DMA_DIRECT_OPS=y
+CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED=y
 CONFIG_SWIOTLB=y
+CONFIG_DMA_COHERENT_POOL=y
+# CONFIG_DMA_API_DEBUG is not set
 CONFIG_SGL_ALLOC=y
 CONFIG_CPU_RMAP=y
 CONFIG_DQL=y
 CONFIG_NLATTR=y
 CONFIG_CLZ_TAB=y
-# CONFIG_CORDIC is not set
-# CONFIG_DDR is not set
 CONFIG_IRQ_POLL=y
 CONFIG_MPILIB=y
 CONFIG_OID_REGISTRY=y
+CONFIG_HAVE_GENERIC_VDSO=y
+CONFIG_GENERIC_GETTIMEOFDAY=y
+CONFIG_GENERIC_VDSO_TIME_NS=y
 CONFIG_SG_POOL=y
-CONFIG_ARCH_HAS_SG_CHAIN=y
 CONFIG_ARCH_HAS_PMEM_API=y
 CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE=y
+CONFIG_ARCH_HAS_COPY_MC=y
+CONFIG_ARCH_STACKWALK=y
 CONFIG_SBITMAP=y
 # CONFIG_STRING_SELFTEST is not set
+# end of Library routines
 
 #
 # Kernel hacking
@@ -2357,95 +2692,234 @@ CONFIG_SBITMAP=y
 # printk and dmesg options
 #
 CONFIG_PRINTK_TIME=y
+# CONFIG_PRINTK_CALLER is not set
 CONFIG_CONSOLE_LOGLEVEL_DEFAULT=7
 CONFIG_CONSOLE_LOGLEVEL_QUIET=4
 CONFIG_MESSAGE_LOGLEVEL_DEFAULT=4
+# CONFIG_BOOT_PRINTK_DELAY is not set
 CONFIG_DYNAMIC_DEBUG=y
+CONFIG_DYNAMIC_DEBUG_CORE=y
+CONFIG_SYMBOLIC_ERRNAME=y
+CONFIG_DEBUG_BUGVERBOSE=y
+# end of printk and dmesg options
 
 #
 # Compile-time checks and compiler options
 #
+# CONFIG_DEBUG_INFO is not set
 # CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_FRAME_WARN=2048
 CONFIG_STRIP_ASM_SYMS=y
-CONFIG_UNUSED_SYMBOLS=y
-CONFIG_DEBUG_FS=y
-# CONFIG_HEADERS_CHECK is not set
+# CONFIG_READABLE_ASM is not set
+# CONFIG_HEADERS_INSTALL is not set
 CONFIG_DEBUG_SECTION_MISMATCH=y
 CONFIG_SECTION_MISMATCH_WARN_ONLY=y
 CONFIG_FRAME_POINTER=y
 CONFIG_STACK_VALIDATION=y
+# CONFIG_DEBUG_FORCE_WEAK_PER_CPU is not set
+# end of Compile-time checks and compiler options
+
+#
+# Generic Kernel Debugging Instruments
+#
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE=0x1
 CONFIG_MAGIC_SYSRQ_SERIAL=y
-# CONFIG_DEBUG_KERNEL is not set
+CONFIG_MAGIC_SYSRQ_SERIAL_SEQUENCE=""
+CONFIG_DEBUG_FS=y
+CONFIG_DEBUG_FS_ALLOW_ALL=y
+# CONFIG_DEBUG_FS_DISALLOW_MOUNT is not set
+# CONFIG_DEBUG_FS_ALLOW_NONE is not set
+CONFIG_HAVE_ARCH_KGDB=y
+# CONFIG_KGDB is not set
+CONFIG_ARCH_HAS_UBSAN_SANITIZE_ALL=y
+# CONFIG_UBSAN is not set
+CONFIG_HAVE_ARCH_KCSAN=y
+CONFIG_HAVE_KCSAN_COMPILER=y
+# CONFIG_KCSAN is not set
+# end of Generic Kernel Debugging Instruments
+
+CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_MISC=y
 
 #
 # Memory Debugging
 #
 # CONFIG_PAGE_EXTENSION is not set
+# CONFIG_DEBUG_PAGEALLOC is not set
+# CONFIG_PAGE_OWNER is not set
 # CONFIG_PAGE_POISONING is not set
 # CONFIG_DEBUG_RODATA_TEST is not set
+CONFIG_ARCH_HAS_DEBUG_WX=y
+# CONFIG_DEBUG_WX is not set
+CONFIG_GENERIC_PTDUMP=y
+# CONFIG_PTDUMP_DEBUGFS is not set
+# CONFIG_DEBUG_OBJECTS is not set
 # CONFIG_SLUB_DEBUG_ON is not set
 # CONFIG_SLUB_STATS is not set
 CONFIG_HAVE_DEBUG_KMEMLEAK=y
+# CONFIG_DEBUG_KMEMLEAK is not set
+# CONFIG_DEBUG_STACK_USAGE is not set
+# CONFIG_SCHED_STACK_END_CHECK is not set
+CONFIG_ARCH_HAS_DEBUG_VM_PGTABLE=y
+# CONFIG_DEBUG_VM is not set
+# CONFIG_DEBUG_VM_PGTABLE is not set
 CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y
+# CONFIG_DEBUG_VIRTUAL is not set
 CONFIG_DEBUG_MEMORY_INIT=y
-CONFIG_HAVE_DEBUG_STACKOVERFLOW=y
+# CONFIG_DEBUG_PER_CPU_MAPS is not set
 CONFIG_HAVE_ARCH_KASAN=y
+CONFIG_HAVE_ARCH_KASAN_VMALLOC=y
+CONFIG_CC_HAS_KASAN_GENERIC=y
+CONFIG_CC_HAS_WORKING_NOSANITIZE_ADDRESS=y
 # CONFIG_KASAN is not set
-CONFIG_ARCH_HAS_KCOV=y
-CONFIG_CC_HAS_SANCOV_TRACE_PC=y
-# CONFIG_KCOV is not set
+# end of Memory Debugging
+
+# CONFIG_DEBUG_SHIRQ is not set
 
 #
-# Debug Lockups and Hangs
+# Debug Oops, Lockups and Hangs
 #
-CONFIG_HARDLOCKUP_CHECK_TIMESTAMP=y
 # CONFIG_PANIC_ON_OOPS is not set
 CONFIG_PANIC_ON_OOPS_VALUE=0
 CONFIG_PANIC_TIMEOUT=0
+# CONFIG_SOFTLOCKUP_DETECTOR is not set
+CONFIG_HARDLOCKUP_CHECK_TIMESTAMP=y
+# CONFIG_HARDLOCKUP_DETECTOR is not set
+# CONFIG_DETECT_HUNG_TASK is not set
+# CONFIG_WQ_WATCHDOG is not set
+# CONFIG_TEST_LOCKUP is not set
+# end of Debug Oops, Lockups and Hangs
+
+#
+# Scheduler Debugging
+#
+# CONFIG_SCHED_DEBUG is not set
 CONFIG_SCHED_INFO=y
+# CONFIG_SCHEDSTATS is not set
+# end of Scheduler Debugging
+
 # CONFIG_DEBUG_TIMEKEEPING is not set
 
 #
 # Lock Debugging (spinlocks, mutexes, etc...)
 #
 CONFIG_LOCK_DEBUGGING_SUPPORT=y
+# CONFIG_PROVE_LOCKING is not set
+# CONFIG_LOCK_STAT is not set
+# CONFIG_DEBUG_RT_MUTEXES is not set
+# CONFIG_DEBUG_SPINLOCK is not set
+# CONFIG_DEBUG_MUTEXES is not set
+# CONFIG_DEBUG_WW_MUTEX_SLOWPATH is not set
+# CONFIG_DEBUG_RWSEMS is not set
+# CONFIG_DEBUG_LOCK_ALLOC is not set
+# CONFIG_DEBUG_ATOMIC_SLEEP is not set
+# CONFIG_DEBUG_LOCKING_API_SELFTESTS is not set
+# CONFIG_LOCK_TORTURE_TEST is not set
 # CONFIG_WW_MUTEX_SELFTEST is not set
+# CONFIG_SCF_TORTURE_TEST is not set
+# CONFIG_CSD_LOCK_WAIT_DEBUG is not set
+# end of Lock Debugging (spinlocks, mutexes, etc...)
+
 CONFIG_STACKTRACE=y
 # CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set
-CONFIG_DEBUG_BUGVERBOSE=y
+# CONFIG_DEBUG_KOBJECT is not set
+
+#
+# Debug kernel data structures
+#
 CONFIG_DEBUG_LIST=y
+# CONFIG_DEBUG_PLIST is not set
+# CONFIG_DEBUG_SG is not set
+# CONFIG_DEBUG_NOTIFIERS is not set
+CONFIG_BUG_ON_DATA_CORRUPTION=y
+# end of Debug kernel data structures
+
+# CONFIG_DEBUG_CREDENTIALS is not set
 
 #
 # RCU Debugging
 #
+# CONFIG_RCU_SCALE_TEST is not set
+# CONFIG_RCU_TORTURE_TEST is not set
+# CONFIG_RCU_REF_SCALE_TEST is not set
 CONFIG_RCU_CPU_STALL_TIMEOUT=59
-CONFIG_FUNCTION_ERROR_INJECTION=y
+# CONFIG_RCU_TRACE is not set
+# CONFIG_RCU_EQS_DEBUG is not set
+# end of RCU Debugging
+
+# CONFIG_DEBUG_WQ_FORCE_RR_CPU is not set
+# CONFIG_DEBUG_BLOCK_EXT_DEVT is not set
+# CONFIG_CPU_HOTPLUG_STATE_CONTROL is not set
+# CONFIG_LATENCYTOP is not set
 CONFIG_USER_STACKTRACE_SUPPORT=y
 CONFIG_HAVE_FUNCTION_TRACER=y
 CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
 CONFIG_HAVE_DYNAMIC_FTRACE=y
 CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS=y
+CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS=y
 CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
 CONFIG_HAVE_SYSCALL_TRACEPOINTS=y
 CONFIG_HAVE_FENTRY=y
 CONFIG_HAVE_C_RECORDMCOUNT=y
 CONFIG_TRACING_SUPPORT=y
 # CONFIG_FTRACE is not set
-# CONFIG_DMA_API_DEBUG is not set
+# CONFIG_SAMPLES is not set
+CONFIG_ARCH_HAS_DEVMEM_IS_ALLOWED=y
+CONFIG_STRICT_DEVMEM=y
+# CONFIG_IO_STRICT_DEVMEM is not set
+
+#
+# x86 Debugging
+#
+CONFIG_TRACE_IRQFLAGS_SUPPORT=y
+CONFIG_TRACE_IRQFLAGS_NMI_SUPPORT=y
+CONFIG_X86_VERBOSE_BOOTUP=y
+CONFIG_EARLY_PRINTK=y
+# CONFIG_DEBUG_TLBFLUSH is not set
+CONFIG_HAVE_MMIOTRACE_SUPPORT=y
+# CONFIG_X86_DECODER_SELFTEST is not set
+CONFIG_IO_DELAY_0X80=y
+# CONFIG_IO_DELAY_0XED is not set
+# CONFIG_IO_DELAY_UDELAY is not set
+# CONFIG_IO_DELAY_NONE is not set
+# CONFIG_DEBUG_BOOT_PARAMS is not set
+# CONFIG_CPA_DEBUG is not set
+# CONFIG_DEBUG_ENTRY is not set
+# CONFIG_DEBUG_NMI_SELFTEST is not set
+# CONFIG_X86_DEBUG_FPU is not set
+# CONFIG_UNWINDER_ORC is not set
+CONFIG_UNWINDER_FRAME_POINTER=y
+# end of x86 Debugging
+
+#
+# Kernel Testing and Coverage
+#
+# CONFIG_KUNIT is not set
+# CONFIG_NOTIFIER_ERROR_INJECTION is not set
+CONFIG_FUNCTION_ERROR_INJECTION=y
+# CONFIG_FAULT_INJECTION is not set
+CONFIG_ARCH_HAS_KCOV=y
+CONFIG_CC_HAS_SANCOV_TRACE_PC=y
+# CONFIG_KCOV is not set
 CONFIG_RUNTIME_TESTING_MENU=y
 # CONFIG_LKDTM is not set
 # CONFIG_TEST_LIST_SORT is not set
+# CONFIG_TEST_MIN_HEAP is not set
 # CONFIG_TEST_SORT is not set
+# CONFIG_KPROBES_SANITY_TEST is not set
+# CONFIG_BACKTRACE_SELF_TEST is not set
+# CONFIG_RBTREE_TEST is not set
+# CONFIG_REED_SOLOMON_TEST is not set
+# CONFIG_INTERVAL_TREE_TEST is not set
+# CONFIG_PERCPU_TEST is not set
 # CONFIG_ATOMIC64_SELFTEST is not set
 # CONFIG_TEST_HEXDUMP is not set
 # CONFIG_TEST_STRING_HELPERS is not set
+# CONFIG_TEST_STRSCPY is not set
 # CONFIG_TEST_KSTRTOX is not set
 # CONFIG_TEST_PRINTF is not set
 # CONFIG_TEST_BITMAP is not set
-# CONFIG_TEST_BITFIELD is not set
 # CONFIG_TEST_UUID is not set
 # CONFIG_TEST_XARRAY is not set
 # CONFIG_TEST_OVERFLOW is not set
@@ -2453,8 +2927,11 @@ CONFIG_RUNTIME_TESTING_MENU=y
 # CONFIG_TEST_HASH is not set
 # CONFIG_TEST_IDA is not set
 # CONFIG_TEST_LKM is not set
+# CONFIG_TEST_BITOPS is not set
+# CONFIG_TEST_VMALLOC is not set
 # CONFIG_TEST_USER_COPY is not set
 # CONFIG_TEST_BPF is not set
+# CONFIG_TEST_BLACKHOLE_DEV is not set
 # CONFIG_FIND_BIT_BENCHMARK is not set
 # CONFIG_TEST_FIRMWARE is not set
 # CONFIG_TEST_SYSCTL is not set
@@ -2462,30 +2939,10 @@ CONFIG_RUNTIME_TESTING_MENU=y
 # CONFIG_TEST_STATIC_KEYS is not set
 # CONFIG_TEST_KMOD is not set
 # CONFIG_TEST_MEMCAT_P is not set
+# CONFIG_TEST_STACKINIT is not set
+# CONFIG_TEST_MEMINIT is not set
+# CONFIG_TEST_FREE_PAGES is not set
+# CONFIG_TEST_FPU is not set
 # CONFIG_MEMTEST is not set
-CONFIG_BUG_ON_DATA_CORRUPTION=y
-# CONFIG_SAMPLES is not set
-CONFIG_HAVE_ARCH_KGDB=y
-CONFIG_ARCH_HAS_UBSAN_SANITIZE_ALL=y
-# CONFIG_UBSAN is not set
-CONFIG_ARCH_HAS_DEVMEM_IS_ALLOWED=y
-CONFIG_STRICT_DEVMEM=y
-# CONFIG_IO_STRICT_DEVMEM is not set
-CONFIG_TRACE_IRQFLAGS_SUPPORT=y
-CONFIG_X86_VERBOSE_BOOTUP=y
-CONFIG_EARLY_PRINTK=y
-# CONFIG_DEBUG_WX is not set
-CONFIG_DOUBLEFAULT=y
-CONFIG_HAVE_MMIOTRACE_SUPPORT=y
-CONFIG_IO_DELAY_TYPE_0X80=0
-CONFIG_IO_DELAY_TYPE_0XED=1
-CONFIG_IO_DELAY_TYPE_UDELAY=2
-CONFIG_IO_DELAY_TYPE_NONE=3
-CONFIG_IO_DELAY_0X80=y
-# CONFIG_IO_DELAY_0XED is not set
-# CONFIG_IO_DELAY_UDELAY is not set
-# CONFIG_IO_DELAY_NONE is not set
-CONFIG_DEFAULT_IO_DELAY_TYPE=0
-CONFIG_OPTIMIZE_INLINING=y
-# CONFIG_UNWINDER_ORC is not set
-CONFIG_UNWINDER_FRAME_POINTER=y
+# end of Kernel Testing and Coverage
+# end of Kernel hacking

From eab7262dadda44c18896172437cb82206a6c5fd2 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 10 Oct 2023 09:45:22 +0200
Subject: [PATCH 502/990] Fix: Kernel did not support BTRFS and was outdated

This updates the Linux kernel used in VMs from 5.10.124 to 5.10.197.

The kernel is now downloaded from IPFS.
---
 packaging/Makefile      | 2 +-
 vm_supervisor/README.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index a5a6c5811..48a156a74 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -40,7 +40,7 @@ firecracker-bins: target-dir build-dir
 
 vmlinux:
 	#curl -fsSL -o ./target/vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin
-	curl -fsSL -o ./target/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.2.2/vmlinux.bin
+	curl -fsSL -o ./target/vmlinux.bin https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy
 	#cp ../kernels/vmlinux.bin ./target/vmlinux.bin
 
 download-ipfs-kubo: target-dir build-dir
diff --git a/vm_supervisor/README.md b/vm_supervisor/README.md
index 16a4a1414..ab6fc699d 100644
--- a/vm_supervisor/README.md
+++ b/vm_supervisor/README.md
@@ -104,7 +104,7 @@ A more optimized kernel may be made available in the future.
 See section _Compile your kernel_ below to build your own.
 
 ```shell
-curl -fsSL -o /opt/firecracker/vmlinux.bin https://github.com/aleph-im/aleph-vm/releases/download/0.2.2/vmlinux.bin
+curl -fsSL -o ./target/vmlinux.bin https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy
 ```
 
 ## 3. Running

From a440f0bee3d83ce13af1cf2c1344a813270b7855 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Tue, 10 Oct 2023 16:22:08 +0200
Subject: [PATCH 503/990] Fix: Set hostname encoding as string bytes (#427)

Fix: Set Item hash of the VM as string and enconding as bytes to be able to use it on b32encode function.

Co-authored-by: Andres D. Molins 
---
 vm_supervisor/vm/firecracker/instance.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/vm_supervisor/vm/firecracker/instance.py b/vm_supervisor/vm/firecracker/instance.py
index 680f1074b..8c50aaf80 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/vm_supervisor/vm/firecracker/instance.py
@@ -181,13 +181,17 @@ async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
         self.latest_snapshot = snapshot
         return compressed_snapshot
 
+    def _get_hostname(self) -> str:
+        item_hash_binary: bytes = base64.b16decode(self.vm_hash.encode().upper())
+        return base64.b32encode(item_hash_binary).decode().strip("=").lower()
+
     def _encode_user_data(self) -> bytes:
         """Creates user data configuration file for cloud-init tool"""
 
         ssh_authorized_keys = self.resources.message_content.authorized_keys or []
 
         config: Dict[str, Union[str, bool, List[str]]] = {
-            "hostname": str(self.vm_hash),
+            "hostname": self._get_hostname(),
             "disable_root": False,
             "ssh_pwauth": False,
             "ssh_authorized_keys": ssh_authorized_keys,
@@ -237,11 +241,9 @@ def _create_network_file(self) -> bytes:
     def _create_metadata_file(self) -> bytes:
         """Creates metadata configuration file for cloud-init tool"""
 
-        hostname = base64.b32encode(self.vm_hash).decode().strip("=").lower()
-
         metadata = {
             "instance-id": f"iid-instance-{self.vm_id}",
-            "local-hostname": hostname,
+            "local-hostname": self._get_hostname(),
         }
 
         return json.dumps(metadata).encode()

From 4b4260b93baaa86b29b0c5f9b7cbd90910adfe65 Mon Sep 17 00:00:00 2001
From: Olivier Desenfans 
Date: Tue, 3 Oct 2023 11:05:12 +0200
Subject: [PATCH 504/990] Release candidate: 0.2.9-rc1

---
 doc/INSTALL-Debian-11.md    | 2 +-
 doc/INSTALL-Debian-12.md    | 2 +-
 doc/INSTALL-Ubuntu-22.04.md | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index 03c940f96..ccc8ae7c8 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.7/aleph-vm.debian-11.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.9-rc1/aleph-vm.debian-11.deb
 apt install /opt/aleph-vm.debian-11.deb
 ```
 
diff --git a/doc/INSTALL-Debian-12.md b/doc/INSTALL-Debian-12.md
index 6bb612e8f..8cba3f6d6 100644
--- a/doc/INSTALL-Debian-12.md
+++ b/doc/INSTALL-Debian-12.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.7/aleph-vm.debian-12.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.9-rc1/aleph-vm.debian-12.deb
 apt install /opt/aleph-vm.debian-12.deb
 ```
 
diff --git a/doc/INSTALL-Ubuntu-22.04.md b/doc/INSTALL-Ubuntu-22.04.md
index bb62fc3e1..52a59848b 100644
--- a/doc/INSTALL-Ubuntu-22.04.md
+++ b/doc/INSTALL-Ubuntu-22.04.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.7/aleph-vm.ubuntu-22.04.deb
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.9-rc1/aleph-vm.ubuntu-22.04.deb
 sudo apt install /opt/aleph-vm.ubuntu-22.04.deb
 ```
 

From a836ca60964243416fc8402159fa6b7ae98ab4fe Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 13 Oct 2023 11:01:15 +0200
Subject: [PATCH 505/990] Fix: Workflow duplication made updates difficult

This replaces the workflows with a "Matrix Strategy" to test a matrix of combinations in parallel.

A new query argument can be used when using old runtimes in order to maintain retro-compatibility: `/status/check/fastapi?retro-compatibility=true`.

Co-authored-by: Andres D. Molins 
---
 .github/workflows/build-deb-package.yml       | 105 +++++---------
 .../workflows/test-new-runtime-examples.yml   |   8 +-
 .../workflows/test-on-droplet-debian-11.yml   |  88 ------------
 .../workflows/test-on-droplet-debian-12.yml   |  88 ------------
 .../test-on-droplet-ubuntu-22.04.yml          |  90 ------------
 .github/workflows/test-on-droplets-matrix.yml | 131 ++++++++++++++++++
 vm_supervisor/views/__init__.py               |  15 +-
 7 files changed, 178 insertions(+), 347 deletions(-)
 delete mode 100644 .github/workflows/test-on-droplet-debian-11.yml
 delete mode 100644 .github/workflows/test-on-droplet-debian-12.yml
 delete mode 100644 .github/workflows/test-on-droplet-ubuntu-22.04.yml
 create mode 100644 .github/workflows/test-on-droplets-matrix.yml

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 57ee24a71..aaac94502 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -3,10 +3,22 @@ on:
   push
 
 jobs:
-  build_deb_debian_11:
-    name: "Build Debian Package"
+  build_deb:
+    name: "Build ${{ matrix.os }} Package"
     runs-on: ubuntu-latest
-
+    strategy:
+      matrix:
+        os: ["debian-11", "debian-12", "ubuntu-22.04"]
+        include:
+          - os: "debian-11"
+            make_target: "all-podman-debian-11"
+            artifact_name: "aleph-vm.debian-11.deb"
+          - os: "debian-12"
+            make_target: "all-podman-debian-12"
+            artifact_name: "aleph-vm.debian-12.deb"
+          - os: "ubuntu-22.04"
+            make_target: "all-podman-ubuntu-2204"
+            artifact_name: "aleph-vm.ubuntu-22.04.deb"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -15,79 +27,25 @@ jobs:
           fetch-depth: 0
 
       - run: |
-          cd packaging && make all-podman-debian-11 && cd ..
+          cd packaging && make ${{ matrix.make_target }} && cd ..
           ls packaging/target
 
       - uses: actions/upload-artifact@v3
         with:
-          name: aleph-vm.debian-11.deb
-          path: packaging/target/aleph-vm.debian-11.deb
+          name: ${{ matrix.artifact_name }}
+          path: packaging/target/${{ matrix.artifact_name }}
 
-  build_deb_debian_12:
-    name: "Build Debian Package"
+  build_rootfs:
+    name: "Build runtime aleph-${{ matrix.os }}-python"
     runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          # Fetch the whole history for all tags and branches (required for aleph.__version__)
-          fetch-depth: 0
-
-      - run: |
-          cd packaging && make all-podman-debian-12 && cd ..
-          ls packaging/target
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: aleph-vm.debian-12.deb
-          path: packaging/target/aleph-vm.debian-12.deb
-
-  build_deb_ubuntu_22_04:
-    name: "Build Ubuntu Package"
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          # Fetch the whole history for all tags and branches (required for aleph.__version__)
-          fetch-depth: 0
-
-      - run: |
-          cd packaging && make all-podman-ubuntu-2204 && cd ..
-          ls packaging/target
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: aleph-vm.ubuntu-22.04.deb
-          path: packaging/target/aleph-vm.ubuntu-22.04.deb
-
-  build_rootfs_debian_11:
-    name: "Build runtime aleph-debian-11-python"
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
-        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
-
-      - run: |
-          sudo apt update
-          sudo apt install -y debootstrap
-          cd runtimes/aleph-debian-11-python && sudo ./create_disk_image.sh && cd ../..
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: aleph-debian-11-python.squashfs
-          path: runtimes/aleph-debian-11-python/rootfs.squashfs
-
-  build_rootfs_debian_12:
-    name: "Build runtime aleph-debian-12-python"
-    runs-on: ubuntu-latest
-
+    strategy:
+      matrix:
+        os: ["debian-11", "debian-12"]
+        include:
+          - os: "debian-11"
+            artifact_name: "aleph-debian-11-python.squashfs"
+          - os: "debian-12"
+            artifact_name: "aleph-debian-12-python.squashfs"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -98,17 +56,16 @@ jobs:
       - run: |
           sudo apt update
           sudo apt install -y debootstrap
-          cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
+          cd runtimes/aleph-${{ matrix.os }}-python && sudo ./create_disk_image.sh && cd ../..
 
       - uses: actions/upload-artifact@v3
         with:
-          name: aleph-debian-12-python.squashfs
-          path: runtimes/aleph-debian-12-python/rootfs.squashfs
+          name: ${{ matrix.artifact_name }}
+          path: runtimes/aleph-${{ matrix.os }}-python/rootfs.squashfs
 
   build_example_venv_volume:
     name: "Build example squashfs volume using Docker"
     runs-on: ubuntu-latest
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
diff --git a/.github/workflows/test-new-runtime-examples.yml b/.github/workflows/test-new-runtime-examples.yml
index 3b2982577..cd0df6978 100644
--- a/.github/workflows/test-new-runtime-examples.yml
+++ b/.github/workflows/test-new-runtime-examples.yml
@@ -36,11 +36,11 @@ jobs:
         run: |
           doctl compute droplet create \
           --image debian-12-x64 \
-          --size c-2 \
-          --region fra1 \
-          --vpc-uuid 992896c8-c089-4da3-9288-f81e28c095a4 \
+          --size c-4 \
+          --region ams3 \
+          --vpc-uuid 5976b7bd-4417-49e8-8522-672aaa920c30 \
           --enable-ipv6 \
-          --ssh-keys b3:ff:08:7f:57:00:fd:7a:14:00:f2:35:0a:f6:e8:55 \
+          --ssh-keys ab:2b:25:16:46:6f:25:d0:80:63:e5:be:67:04:cb:64 \
           aleph-vm-ci-runtime
 
       - name: "Build custom runtime"
diff --git a/.github/workflows/test-on-droplet-debian-11.yml b/.github/workflows/test-on-droplet-debian-11.yml
deleted file mode 100644
index 3dc052c84..000000000
--- a/.github/workflows/test-on-droplet-debian-11.yml
+++ /dev/null
@@ -1,88 +0,0 @@
-name: "Test DigitalOcean Droplet Bullseye"
-on:
-  push
-
-jobs:
-  run_debian_11:
-    name: "Run in DigitalOcean Droplet with Debian 11"
-    runs-on: ubuntu-latest
-    concurrency: droplet-aleph-vm-debian-11
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          # Fetch the whole history for all tags and branches (required for aleph.__version__)
-          fetch-depth: 0
-
-      - name: Install doctl
-        uses: digitalocean/action-doctl@v2
-        with:
-          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
-
-      - name: Setup SSH private key
-        run: |
-          mkdir ~/.ssh
-          echo $DIGITALOCEAN_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519
-          chmod 0700 ~/.ssh
-          chmod 0600 ~/.ssh/id_ed25519
-        env:
-          DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }}
-
-      - name: Create the Droplet
-        run: |
-          doctl compute droplet create \
-          --image debian-11-x64 \
-          --size c-2 \
-          --region fra1 \
-          --vpc-uuid 992896c8-c089-4da3-9288-f81e28c095a4 \
-          --enable-ipv6 \
-          --ssh-keys b3:ff:08:7f:57:00:fd:7a:14:00:f2:35:0a:f6:e8:55 \
-          aleph-vm-ci-debian-11
-
-      - name: Build Debian Package
-        run: |
-          cd packaging && make all-podman-debian-11 && cd ..
-          ls packaging/target
-
-      - name: Wait for the system to setup and boot
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
-
-      - name: Install Aleph-VM on the Droplet
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
-          
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
-          ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
-          
-          scp packaging/target/aleph-vm.debian-11.deb root@${DROPLET_IPV4}:/opt
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.debian-11.deb"
-          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
-          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> /etc/aleph-vm/supervisor.env"
-          ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
-
-      - name: Test Aleph-VM on the Droplet
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          
-          sleep 3
-          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
-          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
-
-      - name: Schedule an instance on the Droplet by faking a call from the scheduler
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-11 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          curl --retry 5 --max-time 10 --fail -X POST -H "Content-Type: application/json" \
-            -H "X-Auth-Signature: test" \
-            -d '{"persistent_vms": [], "instances": ["67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"]}' \
-            "http://${DROPLET_IPV4}:4020/control/allocations"
-
-      - name: Cleanup
-        if: always()
-        run: |
-          doctl compute droplet delete -f aleph-vm-ci-debian-11
diff --git a/.github/workflows/test-on-droplet-debian-12.yml b/.github/workflows/test-on-droplet-debian-12.yml
deleted file mode 100644
index 0d59003b9..000000000
--- a/.github/workflows/test-on-droplet-debian-12.yml
+++ /dev/null
@@ -1,88 +0,0 @@
-name: "Test on DigitalOcean Droplet Bookworm"
-on:
-  push
-
-jobs:
-  run_debian_12:
-    name: "Run in DigitalOcean Droplet with Debian 12"
-    runs-on: ubuntu-latest
-    concurrency: droplet-aleph-vm-debian-12
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          # Fetch the whole history for all tags and branches (required for aleph.__version__)
-          fetch-depth: 0
-
-      - name: Install doctl
-        uses: digitalocean/action-doctl@v2
-        with:
-          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
-
-      - name: Setup SSH private key
-        run: |
-          mkdir ~/.ssh
-          echo $DIGITALOCEAN_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519
-          chmod 0700 ~/.ssh
-          chmod 0600 ~/.ssh/id_ed25519
-        env:
-          DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }}
-
-      - name: Create the Droplet
-        run: |
-          doctl compute droplet create \
-          --image debian-12-x64 \
-          --size c-2 \
-          --region fra1 \
-          --vpc-uuid 992896c8-c089-4da3-9288-f81e28c095a4 \
-          --enable-ipv6 \
-          --ssh-keys b3:ff:08:7f:57:00:fd:7a:14:00:f2:35:0a:f6:e8:55 \
-          aleph-vm-ci-debian-12
-
-      - name: Build Debian Package
-        run: |
-          cd packaging && make all-podman-debian-12 && cd ..
-          ls packaging/target
-
-      - name: Wait for the system to setup and boot
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-12 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
-
-      - name: Install Aleph-VM on the Droplet
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-12 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
-          
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
-          ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
-          
-          scp packaging/target/aleph-vm.debian-12.deb root@${DROPLET_IPV4}:/opt
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.debian-12.deb"
-          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
-          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> /etc/aleph-vm/supervisor.env"
-          ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
-
-      - name: Test Aleph-VM on the Droplet
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-12 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          
-          sleep 3
-          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
-          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
-
-      - name: Schedule an instance on the Droplet by faking a call from the scheduler
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-debian-12 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          curl --retry 5 --max-time 10 --fail -X POST -H "Content-Type: application/json" \
-            -H "X-Auth-Signature: test" \
-            -d '{"persistent_vms": [], "instances": ["67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"]}' \
-            "http://${DROPLET_IPV4}:4020/control/allocations"
-
-      - name: Cleanup
-        if: always()
-        run: |
-          doctl compute droplet delete -f aleph-vm-ci-debian-12
diff --git a/.github/workflows/test-on-droplet-ubuntu-22.04.yml b/.github/workflows/test-on-droplet-ubuntu-22.04.yml
deleted file mode 100644
index 622b1db10..000000000
--- a/.github/workflows/test-on-droplet-ubuntu-22.04.yml
+++ /dev/null
@@ -1,90 +0,0 @@
-name: "Test on DigitalOcean Droplet Jammy"
-on:
-  push
-
-jobs:
-  run_ubuntu_22_04:
-    name: "Run in DigitalOcean Droplet with Ubuntu 22.04"
-    runs-on: ubuntu-latest
-    concurrency: droplet-aleph-vm-ubuntu-22-04
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          # Fetch the whole history for all tags and branches (required for aleph.__version__)
-          fetch-depth: 0
-
-      - name: Install doctl
-        uses: digitalocean/action-doctl@v2
-        with:
-          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
-
-      - name: Setup SSH private key
-        run: |
-          mkdir ~/.ssh
-          echo $DIGITALOCEAN_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519
-          chmod 0700 ~/.ssh
-          chmod 0600 ~/.ssh/id_ed25519
-        env:
-          DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }}
-
-      - name: Create the Droplet
-        run: |
-          doctl compute droplet create \
-          --image ubuntu-22-04-x64 \
-          --size c-4 \
-          --region fra1 \
-          --vpc-uuid 992896c8-c089-4da3-9288-f81e28c095a4 \
-          --enable-ipv6 \
-          --ssh-keys b3:ff:08:7f:57:00:fd:7a:14:00:f2:35:0a:f6:e8:55 \
-          aleph-vm-ci-ubuntu-22-04
-
-      - name: Build Ubuntu Package
-        run: |
-          cd packaging && make all-podman-ubuntu-2204 && cd ..
-          ls packaging/target
-
-      - name: Wait for the system to setup and boot
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-ubuntu-22-04 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
-
-      - name: Install Aleph-VM on the Droplet
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-ubuntu-22-04 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
-          
-          # Ubuntu droplets run upgrades at boot, which locks apt-get
-          sleep 30
-          until ssh root@${DROPLET_IPV4} "apt-get update" > /dev/null; do sleep 1; echo "Waiting for apt/dpkg lock..."; done          
-          until ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y" > /dev/null; do sleep 1; echo "Waiting for apt/dpkg lock..."; done
-          until ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles" > /dev/null; do sleep 1; echo "Waiting for apt/dpkg lock..."; done
-          ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
-          
-          scp packaging/target/aleph-vm.ubuntu-22.04.deb root@${DROPLET_IPV4}:/opt
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/aleph-vm.ubuntu-22.04.deb"
-          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> /etc/aleph-vm/supervisor.env"
-          ssh root@${DROPLET_IPV4} "echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> /etc/aleph-vm/supervisor.env"
-          ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
-
-      - name: Test Aleph-VM on the Droplet
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-ubuntu-22-04 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          
-          sleep 3
-          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
-          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
-
-      - name: Schedule an instance on the Droplet by faking a call from the scheduler
-        run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-ubuntu-22-04 --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          curl --retry 5 --max-time 10 --fail -X POST -H "Content-Type: application/json" \
-            -H "X-Auth-Signature: test" \
-            -d '{"persistent_vms": [], "instances": ["67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"]}' \
-            "http://${DROPLET_IPV4}:4020/control/allocations"
-
-      - name: Cleanup
-        if: always()
-        run: |
-          doctl compute droplet delete -f aleph-vm-ci-ubuntu-22-04
diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
new file mode 100644
index 000000000..5b9af4259
--- /dev/null
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -0,0 +1,131 @@
+name: "Test on DigitalOcean Droplets"
+
+on:
+  push
+
+jobs:
+  run_on_droplet:
+    name: "Test Droplet with ${{ matrix.os_config.os_name }}-${{ matrix.check_vm.alias }}"
+    runs-on: ubuntu-latest
+    concurrency: "${{ matrix.os_config.concurrency_group }}-${{ matrix.check_vm.alias }}"
+    timeout-minutes: 10
+
+    strategy:
+      matrix:
+
+        # Check compatibility with all supported OSes.
+        os_config:
+          - os_name: "Debian 11"
+            os_image: "debian-11-x64"
+            alias: "debian-11"
+            package_build_command: "all-podman-debian-11"
+            package_name: "aleph-vm.debian-11.deb"
+            concurrency_group: "droplet-aleph-vm-debian-11"
+
+          - os_name: "Debian 12"
+            os_image: "debian-12-x64"
+            alias: "debian-12"
+            package_build_command: "all-podman-debian-12"
+            package_name: "aleph-vm.debian-12.deb"
+            concurrency_group: "droplet-aleph-vm-debian-12"
+
+          - os_name: "Ubuntu 22.04"
+            os_image: "ubuntu-22-04-x64"
+            alias: "ubuntu-22-04"
+            package_build_command: "all-podman-ubuntu-2204"
+            package_name: "aleph-vm.ubuntu-22.04.deb"
+            concurrency_group: "droplet-aleph-vm-ubuntu-22-04"
+
+        # Check compatibility with all supported runtimes.
+        check_vm:
+          - alias: "runtime-6770"  # Old runtime, using Debian 11
+            item_hash: "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
+            query_params: "?retro-compatibility=true"
+          - alias: "runtime-3fc0"  # New runtime, using Debian 12
+            item_hash: "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af"
+            query_params: ""
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install doctl
+        uses: digitalocean/action-doctl@v2
+        with:
+          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+
+      - name: Setup SSH private key
+        run: |
+          mkdir ~/.ssh
+          echo $DIGITALOCEAN_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519
+          chmod 0700 ~/.ssh
+          chmod 0600 ~/.ssh/id_ed25519
+        env:
+          DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }}
+
+      - name: Create the Droplet
+        run: |
+          doctl compute droplet create \
+          --image ${{ matrix.os_config.os_image }} \
+          --size c-4 \
+          --region ams3 \
+          --vpc-uuid 5976b7bd-4417-49e8-8522-672aaa920c30 \
+          --enable-ipv6 \
+          --ssh-keys ab:2b:25:16:46:6f:25:d0:80:63:e5:be:67:04:cb:64 \
+          aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }}
+
+      - name: Build Package
+        run: |
+          echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> packaging/aleph-vm/etc/aleph-vm/supervisor.env
+          echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> packaging/aleph-vm/etc/aleph-vm/supervisor.env
+          echo ALEPH_VM_CHECK_FASTAPI_VM_ID=${{ matrix.check_vm.item_hash }} >> packaging/aleph-vm/etc/aleph-vm/supervisor.env
+          echo ALEPH_VM_SENTRY_DSN=${{ secrets.SENTRY_DSN }} >> packaging/aleph-vm/etc/aleph-vm/supervisor.env
+          cd packaging && make ${{ matrix.os_config.package_build_command }} && cd ..
+          ls packaging/target
+
+      - name: Wait for the system to setup and boot
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
+        timeout-minutes: 3
+
+      - name: Install Aleph-VM on the Droplet
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
+
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get update"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get upgrade -y"
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get install -y docker.io apparmor-profiles"
+          ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
+
+          scp packaging/target/${{ matrix.os_config.package_name }} root@${DROPLET_IPV4}:/opt
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt install -y /opt/${{ matrix.os_config.package_name }}"
+
+          # Allow some time for IPFS Kubo to start
+          sleep 5
+
+      - name: Test Aleph-VM on the Droplet
+        id: test-aleph-vm
+        if: always()
+        continue-on-error: true
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+
+          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
+          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi${{ matrix.check_vm.query_params }}"
+
+      - name: Schedule an instance on the Droplet by faking a call from the scheduler
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          curl --retry 5 --max-time 10 --fail -X POST -H "Content-Type: application/json" \
+            -H "X-Auth-Signature: test" \
+            -d '{"persistent_vms": [], "instances": ["67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"]}' \
+            "http://${DROPLET_IPV4}:4020/control/allocations"
+
+      - name: Cleanup
+        if: always()
+        run: |
+          doctl compute droplet delete -f aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }}
diff --git a/vm_supervisor/views/__init__.py b/vm_supervisor/views/__init__.py
index fd60df618..a60327d95 100644
--- a/vm_supervisor/views/__init__.py
+++ b/vm_supervisor/views/__init__.py
@@ -146,21 +146,30 @@ async def index(request: web.Request):
 
 
 async def status_check_fastapi(request: web.Request):
+    retro_compatibility: bool = (
+        request.rel_url.query.get("retro-compatibility", "false") == "true"
+    )
+
     async with aiohttp.ClientSession() as session:
         result = {
             "index": await status.check_index(session),
-            # TODO: lifespan is a new feature that requires a new runtime to be deployed
-            "lifespan": await status.check_lifespan(session),
             "environ": await status.check_environ(session),
             "messages": await status.check_messages(session),
             "dns": await status.check_dns(session),
             "ipv4": await status.check_ipv4(session),
-            # "ipv6": await status.check_ipv6(session),
             "internet": await status.check_internet(session),
             "cache": await status.check_cache(session),
             "persistent_storage": await status.check_persistent_storage(session),
             "error_handling": await status.check_error_raised(session),
         }
+        if not retro_compatibility:
+            # These fields were added in the runtime running Debian 12.
+            result = result | {
+                "lifespan": await status.check_lifespan(session),
+                # IPv6 requires extra work from node operators and is not required yet.
+                # "ipv6": await status.check_ipv6(session),
+            }
+
         return web.json_response(result, status=200 if all(result.values()) else 503)
 
 

From 79764379f3005fcfa0dc90f8b8ce512bfad2ee1f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 13 Oct 2023 11:15:41 +0200
Subject: [PATCH 506/990] Fix: Diagnosis VM would not run on some CI
 configuration

Solution: Try againa after restarting the supervisor.
---
 .github/workflows/test-on-droplets-matrix.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index 5b9af4259..353c47dc4 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -117,6 +117,17 @@ jobs:
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi${{ matrix.check_vm.query_params }}"
 
+      - name: Test Aleph-VM on the Droplet again restarting the server first
+        if: steps.test-aleph-vm.outcome == 'failure'
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+
+          # If the first execution fails, restart supervisor and try again
+          ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
+          sleep 5
+
+          curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi${{ matrix.check_vm.query_params }}"
+
       - name: Schedule an instance on the Droplet by faking a call from the scheduler
         run: |
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"

From 3da670f17b0f9e1b66ff631978657e99f1e2b7f7 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 13 Oct 2023 11:03:43 +0200
Subject: [PATCH 507/990] Update: Firecracker 1.3.3 -> 1.5.0

See changelogs on https://github.com/firecracker-microvm/firecracker/releases
---
 packaging/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 48a156a74..88e7b59ce 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -31,7 +31,7 @@ debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo
 firecracker-bins: target-dir build-dir
 	mkdir -p ./build/firecracker-release
 	# Download latest release
-	curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.3.3/firecracker-v1.3.3-x86_64.tgz | tar -xz --no-same-owner --directory ./build/firecracker-release
+	curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory ./build/firecracker-release
 	# Copy binaries:
 	cp ./build/firecracker-release/release-v*/firecracker-v*[!.debug] ./target/firecracker
 	cp ./build/firecracker-release/release-v*/jailer-v*[!.debug] ./target/jailer

From cebca0e2f6ac67efa23060eccad81465972b90b6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 11 Oct 2023 14:38:03 +0200
Subject: [PATCH 508/990] Refactoring: Project could not be "pip installed"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This refactors the codebase in order to make it possile to "pip install"
it from other projects and reuse its components.

Based on the proposal on:
https://community.aleph.im/t/aleph-vm-refactoring-proposal/84

```
.
├── aleph
│   └── vm
│       ├── controller
│       │   ├── firecracker
│       │   │   ├── executable
│       │   │   ├── instance
│       │   │   └── program
│       │   └── qemu
│       ├── guest_api
│       ├── metrics
│       ├── orchestrator
│       │   ├── network
│       │   └── views
│       ├── snapthots
│       └── tests
├── examples
├── kernels
├── packaging
└── runtimes
```
---
 .github/workflows/test-build-examples.yml     |   4 +-
 README.md                                     |   4 +-
 doc/INSTALL-Debian-11.md                      |   4 +-
 doc/INSTALL-Debian-12.md                      |   4 +-
 doc/INSTALL-Ubuntu-22.04.md                   |   4 +-
 docker/run_vm_supervisor.sh                   |   2 +-
 docker/vm_supervisor-dev.dockerfile           |   2 +-
 .../blog/migrations/0001_initial.py           |  11 +-
 .../migrations/0002_auto_20210702_1331.py     |  18 +-
 examples/example_django/blog/views.py         |   4 +-
 .../example_django/example_django/asgi.py     |   3 +-
 .../example_django/example_django/urls.py     |   2 +-
 examples/example_fastapi/main.py              |  33 ++--
 examples/example_pip/main.py                  |   3 +-
 firecracker/__init__.py                       |   2 -
 packaging/Makefile                            |   6 +-
 .../system/aleph-vm-supervisor.service        |   2 +-
 packaging/debian-11.dockerfile                |   4 +-
 packaging/debian-12.dockerfile                |   4 +-
 packaging/ubuntu-22.04.dockerfile             |   4 +-
 packaging/version_from_git.py                 |  38 ++--
 runtimes/aleph-debian-11-python/init1.py      |  50 ++---
 {guest_api => src/aleph}/__init__.py          |   0
 .../migrations => src/aleph/vm}/__init__.py   |   0
 .../aleph/vm/controllers}/__init__.py         |   0
 .../vm/controllers}/firecracker/__init__.py   |   0
 .../vm/controllers}/firecracker/executable.py |  37 ++--
 .../vm/controllers}/firecracker/instance.py   |  68 +++----
 .../vm/controllers}/firecracker/program.py    | 185 ++++++++----------
 src/aleph/vm/guest_api/__init__.py            |   0
 .../aleph/vm/guest_api}/__main__.py           |  20 +-
 src/aleph/vm/hypervisors/__init__.py          |   0
 .../vm/hypervisors/firecracker/__init__.py    |   0
 .../vm/hypervisors/firecracker}/config.py     |  15 +-
 .../vm/hypervisors/firecracker}/microvm.py    |  62 +++---
 .../aleph/vm/orchestrator}/INSTANCES.md       |   0
 .../aleph/vm/orchestrator}/README.md          |  10 +-
 .../aleph/vm/orchestrator}/__init__.py        |   0
 src/aleph/vm/orchestrator/__main__.py         |   4 +
 .../aleph/vm/orchestrator}/alembic.ini        |   2 +-
 .../aleph/vm/orchestrator/cli.py              |  47 ++---
 .../aleph/vm/orchestrator}/conf.py            |  83 +++-----
 .../aleph/vm/orchestrator}/messages.py        |  17 +-
 .../aleph/vm/orchestrator}/metrics.py         |  15 +-
 .../vm/orchestrator/migrations/__init__.py    |   0
 .../aleph/vm/orchestrator}/migrations/env.py  |   4 +-
 .../orchestrator}/migrations/script.py.mako   |   0
 .../0001_bbb12a12372e_execution_records.py    |   2 +-
 .../migrations/versions/__init__.py           |   0
 .../aleph/vm/orchestrator}/models.py          |  65 +++---
 src/aleph/vm/orchestrator/network/__init__.py |   0
 .../vm/orchestrator}/network/firewall.py      |  45 ++---
 .../vm/orchestrator}/network/hostnetwork.py   |  52 ++---
 .../vm/orchestrator}/network/interfaces.py    |   0
 .../vm/orchestrator}/network/ipaddresses.py   |   8 +-
 .../vm/orchestrator}/network/ndp_proxy.py     |   5 +-
 .../aleph/vm/orchestrator}/pool.py            |  27 ++-
 .../aleph/vm/orchestrator}/pubsub.py          |  11 +-
 .../aleph/vm/orchestrator}/reactor.py         |  15 +-
 .../aleph/vm/orchestrator}/resources.py       |  18 +-
 .../aleph/vm/orchestrator}/run.py             |  49 ++---
 .../vm/orchestrator}/snapshot_manager.py      |  30 ++-
 .../aleph/vm/orchestrator}/snapshots.py       |   8 +-
 .../aleph/vm/orchestrator}/status.py          |  26 +--
 .../aleph/vm/orchestrator}/storage.py         |  55 +++---
 .../aleph/vm/orchestrator}/supervisor.py      |   7 +-
 .../aleph/vm/orchestrator}/tasks.py           |  19 +-
 .../aleph/vm/orchestrator}/utils.py           |  27 ++-
 .../aleph/vm/orchestrator}/version.py         |   0
 .../aleph/vm/orchestrator}/views/__init__.py  |  64 +++---
 .../aleph/vm/orchestrator}/views/operator.py  |  15 +-
 .../orchestrator}/views/templates/index.html  |   0
 src/aleph/vm/orchestrator/vm/__init__.py      |   9 +
 .../aleph/vm/orchestrator}/vm/vm_type.py      |   3 +-
 tests/supervisor/test_ipv6_allocator.py       |  13 +-
 tests/supervisor/test_jwk.py                  |  12 +-
 .../supervisor/test_resolvectl_dns_servers.py |  12 +-
 tutorials/TESTING.md                          |   6 +-
 vm_connector/README.md                        |   2 +-
 vm_connector/conf.py                          |   3 +-
 vm_connector/main.py                          |   9 +-
 vm_supervisor/vm/__init__.py                  |   6 -
 82 files changed, 547 insertions(+), 853 deletions(-)
 delete mode 100644 firecracker/__init__.py
 rename {guest_api => src/aleph}/__init__.py (100%)
 rename {vm_supervisor/migrations => src/aleph/vm}/__init__.py (100%)
 rename {vm_supervisor/network => src/aleph/vm/controllers}/__init__.py (100%)
 rename {vm_supervisor/vm => src/aleph/vm/controllers}/firecracker/__init__.py (100%)
 rename {vm_supervisor/vm => src/aleph/vm/controllers}/firecracker/executable.py (89%)
 rename {vm_supervisor/vm => src/aleph/vm/controllers}/firecracker/instance.py (79%)
 rename {vm_supervisor/vm => src/aleph/vm/controllers}/firecracker/program.py (71%)
 create mode 100644 src/aleph/vm/guest_api/__init__.py
 rename {guest_api => src/aleph/vm/guest_api}/__main__.py (91%)
 create mode 100644 src/aleph/vm/hypervisors/__init__.py
 create mode 100644 src/aleph/vm/hypervisors/firecracker/__init__.py
 rename {firecracker => src/aleph/vm/hypervisors/firecracker}/config.py (81%)
 rename {firecracker => src/aleph/vm/hypervisors/firecracker}/microvm.py (89%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/INSTANCES.md (100%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/README.md (95%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/__init__.py (100%)
 create mode 100644 src/aleph/vm/orchestrator/__main__.py
 rename {vm_supervisor => src/aleph/vm/orchestrator}/alembic.ini (98%)
 rename vm_supervisor/__main__.py => src/aleph/vm/orchestrator/cli.py (88%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/conf.py (81%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/messages.py (81%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/metrics.py (88%)
 create mode 100644 src/aleph/vm/orchestrator/migrations/__init__.py
 rename {vm_supervisor => src/aleph/vm/orchestrator}/migrations/env.py (94%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/migrations/script.py.mako (100%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/migrations/versions/0001_bbb12a12372e_execution_records.py (97%)
 create mode 100644 src/aleph/vm/orchestrator/migrations/versions/__init__.py
 rename {vm_supervisor => src/aleph/vm/orchestrator}/models.py (88%)
 create mode 100644 src/aleph/vm/orchestrator/network/__init__.py
 rename {vm_supervisor => src/aleph/vm/orchestrator}/network/firewall.py (91%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/network/hostnetwork.py (83%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/network/interfaces.py (100%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/network/ipaddresses.py (78%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/network/ndp_proxy.py (93%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/pool.py (87%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/pubsub.py (86%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/reactor.py (88%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/resources.py (88%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/run.py (89%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/snapshot_manager.py (77%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/snapshots.py (85%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/status.py (83%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/storage.py (89%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/supervisor.py (95%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/tasks.py (90%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/utils.py (80%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/version.py (100%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/views/__init__.py (84%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/views/operator.py (95%)
 rename {vm_supervisor => src/aleph/vm/orchestrator}/views/templates/index.html (100%)
 create mode 100644 src/aleph/vm/orchestrator/vm/__init__.py
 rename {vm_supervisor => src/aleph/vm/orchestrator}/vm/vm_type.py (84%)
 delete mode 100644 vm_supervisor/vm/__init__.py

diff --git a/.github/workflows/test-build-examples.yml b/.github/workflows/test-build-examples.yml
index 66e382d49..c37e74e8c 100644
--- a/.github/workflows/test-build-examples.yml
+++ b/.github/workflows/test-build-examples.yml
@@ -19,7 +19,7 @@ jobs:
       - run: |
           sudo apt-get -y update
           sudo apt-get -y upgrade
-          sudo apt-get -y install python3-pip python3-venv squashfs-tools build-essential
+          sudo apt-get -y install python3-pip python3-venv squashfs-tools build-essential python3-nftables
 
           sudo mkdir /opt/packages
           sudo chown $(whoami) /opt/packages
@@ -28,7 +28,7 @@ jobs:
           pip3 install aleph-sdk-python
 
       - run: |
-          ls -la
+           hatch build
 
       - run: |
           ls
diff --git a/README.md b/README.md
index 18854c83f..e6226d8b0 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@ Install Aleph-VM to run an Aleph.im Compute Resource Node easily from official p
 For development and testing, install Aleph-VM from source.
 
 1. Install the [VM-Connector](./vm_connector/README.md)
-2. Install the [VM-Supervisor](./vm_supervisor/README.md).
+2. Install the [VM-Supervisor](src/aleph/vm/orchestrator/README.md).
 3. Install and [configure a reverse-proxy such as [Caddy](./CONFIGURE_CADDY.md)
 
 ## 3. Create and run an Aleph Program 
@@ -43,7 +43,7 @@ The rest of this document focuses on how to run an Aleph-VM node that hosts and
 
 Actually runs the programs in a secure environment on virtualization enabled systems. 
 
-See [vm_supervisor/README.md](./vm_supervisor/README.md).
+See [vm_supervisor/README.md](src/aleph/vm/orchestrator/README.md).
 
 ### VM Connector
 
diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index ccc8ae7c8..ee297c835 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -6,7 +6,7 @@ For production using official Debian packages.
 
 ## 1. Requirements
 
-- A [supported Linux server](../vm_supervisor/README.md#1-supported-platforms)
+- A [supported Linux server](../src/aleph/vm/orchestrator/README.md#1-supported-platforms)
 - A public domain name from a registrar and top level domain you trust. 
 
 In order to run an official Aleph.im Compute Resource Node (CRN), you will also need the following resources:
@@ -34,7 +34,7 @@ apt install -y docker.io apparmor-profiles
 docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
 ```
 
-Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
+Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
 wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.9-rc1/aleph-vm.debian-11.deb
diff --git a/doc/INSTALL-Debian-12.md b/doc/INSTALL-Debian-12.md
index 8cba3f6d6..b840259f2 100644
--- a/doc/INSTALL-Debian-12.md
+++ b/doc/INSTALL-Debian-12.md
@@ -6,7 +6,7 @@ For production using official Debian packages.
 
 ## 1. Requirements
 
-- A [supported Linux server](../vm_supervisor/README.md#1-supported-platforms)
+- A [supported Linux server](../src/aleph/vm/orchestrator/README.md#1-supported-platforms)
 - A public domain name from a registrar and top level domain you trust. 
 
 In order to run an official Aleph.im Compute Resource Node (CRN), you will also need the following resources:
@@ -34,7 +34,7 @@ apt install -y docker.io apparmor-profiles
 docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
 ```
 
-Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
+Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
 wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.9-rc1/aleph-vm.debian-12.deb
diff --git a/doc/INSTALL-Ubuntu-22.04.md b/doc/INSTALL-Ubuntu-22.04.md
index 52a59848b..efd5693f1 100644
--- a/doc/INSTALL-Ubuntu-22.04.md
+++ b/doc/INSTALL-Ubuntu-22.04.md
@@ -6,7 +6,7 @@ For production using official Debian packages.
 
 ## 1. Requirements
 
-- A [supported Linux server](../vm_supervisor/README.md#1-supported-platforms)
+- A [supported Linux server](../src/aleph/vm/orchestrator/README.md#1-supported-platforms)
 - A public domain name from a registrar and top level domain you trust. 
 
 In order to run an official Aleph.im Compute Resource Node (CRN), you will also need the following resources:
@@ -34,7 +34,7 @@ sudo apt install -y docker.io
 docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
 ```
 
-Then install the [VM-Supervisor](../vm_supervisor/README.md) using the official Debian package.
+Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
 sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.9-rc1/aleph-vm.ubuntu-22.04.deb
diff --git a/docker/run_vm_supervisor.sh b/docker/run_vm_supervisor.sh
index 5499dc324..00c558989 100755
--- a/docker/run_vm_supervisor.sh
+++ b/docker/run_vm_supervisor.sh
@@ -10,7 +10,7 @@ else
   DOCKER_COMMAND=docker
 fi
 
-$DOCKER_COMMAND build -t alephim/vm-supervisor-dev -f docker/vm_supervisor-dev.dockerfile .
+$DOCKER_COMMAND build -t alephim/vm-supervisor-dev -f docker/orchestrator-dev.dockerfile .
 
 $DOCKER_COMMAND run -ti --rm \
   -v "$(pwd)/runtimes/aleph-debian-11-python/rootfs.squashfs:/opt/aleph-vm/runtimes/aleph-debian-11-python/rootfs.squashfs:ro" \
diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index ee457230d..a840385ab 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -1,4 +1,4 @@
-# This is mainly a copy of the installation instructions from [vm_supervisor/README.md]
+# This is mainly a copy of the installation instructions from [orchestrator/README.md]
 
 FROM debian:bullseye
 
diff --git a/examples/example_django/blog/migrations/0001_initial.py b/examples/example_django/blog/migrations/0001_initial.py
index 214f95af3..d8dacd8dc 100644
--- a/examples/example_django/blog/migrations/0001_initial.py
+++ b/examples/example_django/blog/migrations/0001_initial.py
@@ -1,11 +1,10 @@
 # Generated by Django 3.2.4 on 2021-07-02 09:35
 
-from django.db import migrations, models
 import django.db.models.deletion
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = []
@@ -18,9 +17,7 @@ class Migration(migrations.Migration):
                 ("id", models.UUIDField(primary_key=True, serialize=False)),
                 (
                     "title",
-                    models.CharField(
-                        help_text="Title of the blog article", max_length=256
-                    ),
+                    models.CharField(help_text="Title of the blog article", max_length=256),
                 ),
                 ("body", models.TextField(help_text="Body of the blog article")),
             ],
@@ -33,9 +30,7 @@ class Migration(migrations.Migration):
                 ("text", models.CharField(max_length=1024)),
                 (
                     "article",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="blog.article"
-                    ),
+                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="blog.article"),
                 ),
             ],
         ),
diff --git a/examples/example_django/blog/migrations/0002_auto_20210702_1331.py b/examples/example_django/blog/migrations/0002_auto_20210702_1331.py
index 7d72238e4..d5c813fb6 100644
--- a/examples/example_django/blog/migrations/0002_auto_20210702_1331.py
+++ b/examples/example_django/blog/migrations/0002_auto_20210702_1331.py
@@ -1,29 +1,29 @@
 # Generated by Django 3.2.4 on 2021-07-02 13:31
 
-from django.db import migrations, models
 import uuid
 
+from django.db import migrations, models
+
 
 class Migration(migrations.Migration):
-
     dependencies = [
-        ('blog', '0001_initial'),
+        ("blog", "0001_initial"),
     ]
 
     operations = [
         migrations.AlterField(
-            model_name='article',
-            name='id',
+            model_name="article",
+            name="id",
             field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
         ),
         migrations.AlterField(
-            model_name='comment',
-            name='date',
+            model_name="comment",
+            name="date",
             field=models.DateTimeField(auto_now_add=True),
         ),
         migrations.AlterField(
-            model_name='comment',
-            name='id',
+            model_name="comment",
+            name="id",
             field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
         ),
     ]
diff --git a/examples/example_django/blog/views.py b/examples/example_django/blog/views.py
index 7d3d3e7f8..1e3ad6d08 100644
--- a/examples/example_django/blog/views.py
+++ b/examples/example_django/blog/views.py
@@ -1,7 +1,5 @@
-import os
-
 from django.http import JsonResponse
-from django.views.generic import ListView, FormView, CreateView
+from django.views.generic import CreateView, ListView
 
 from .forms import CommentForm
 from .models import Article
diff --git a/examples/example_django/example_django/asgi.py b/examples/example_django/example_django/asgi.py
index 07cb2143e..1a2020727 100644
--- a/examples/example_django/example_django/asgi.py
+++ b/examples/example_django/example_django/asgi.py
@@ -17,5 +17,4 @@
 
 os.system("/usr/bin/python3 /opt/code/manage.py migrate")
 
-os.system("/usr/bin/python3 /opt/code/manage.py "
-          "loaddata /opt/code/blog/fixtures/default_articles.json")
+os.system("/usr/bin/python3 /opt/code/manage.py " "loaddata /opt/code/blog/fixtures/default_articles.json")
diff --git a/examples/example_django/example_django/urls.py b/examples/example_django/example_django/urls.py
index 477c80598..948195d46 100644
--- a/examples/example_django/example_django/urls.py
+++ b/examples/example_django/example_django/urls.py
@@ -14,7 +14,7 @@
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
-from django.urls import path, include
+from django.urls import include, path
 
 urlpatterns = [
     path("", include("blog.urls")),
diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index 866fb6ac4..c7c5fc161 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -10,15 +10,16 @@
 from typing import Dict
 
 import aiohttp
+from fastapi import FastAPI
+from fastapi.responses import PlainTextResponse
+from pip._internal.operations.freeze import freeze
+from pydantic import BaseModel
+
 from aleph.sdk.chains.remote import RemoteAccount
 from aleph.sdk.client import AlephClient, AuthenticatedAlephClient
 from aleph.sdk.types import StorageEnum
 from aleph.sdk.vm.app import AlephApp
 from aleph.sdk.vm.cache import VmCache
-from fastapi import FastAPI
-from fastapi.responses import PlainTextResponse
-from pip._internal.operations.freeze import freeze
-from pydantic import BaseModel
 
 logger = logging.getLogger(__name__)
 logger.debug("imports done")
@@ -83,18 +84,14 @@ async def environ() -> Dict[str, str]:
 async def read_aleph_messages():
     """Read data from Aleph using the Aleph Client library."""
     async with AlephClient() as client:
-        data = await client.get_messages(
-            hashes=["f246f873c3e0f637a15c566e7a465d2ecbb83eaa024d54ccb8fb566b549a929e"]
-        )
+        data = await client.get_messages(hashes=["f246f873c3e0f637a15c566e7a465d2ecbb83eaa024d54ccb8fb566b549a929e"])
     return {"Messages": data}
 
 
 @app.get("/dns")
 async def resolve_dns_hostname():
     """Check if DNS resolution is working."""
-    info_inet, info_inet6 = socket.getaddrinfo(
-        "example.org", 80, proto=socket.IPPROTO_TCP
-    )
+    info_inet, info_inet6 = socket.getaddrinfo("example.org", 80, proto=socket.IPPROTO_TCP)
     ipv4 = info_inet[4][0]
     ipv6 = info_inet6[4][0]
     return {
@@ -125,9 +122,7 @@ async def connect_ipv6():
     The webserver on that address returns a 404 error, so we accept that response code.
     """
     timeout = aiohttp.ClientTimeout(total=5)
-    async with aiohttp.ClientSession(
-        connector=aiohttp.TCPConnector(), timeout=timeout
-    ) as session:
+    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
         async with session.get("https://[2620:fe::fe]") as resp:
             # We expect this endpoint to return a 404 error
             if resp.status != 404:
@@ -139,9 +134,7 @@ async def connect_ipv6():
 async def read_internet():
     """Connect the aleph.im official website to check Internet connectivity."""
     timeout = aiohttp.ClientTimeout(total=5)
-    async with aiohttp.ClientSession(
-        connector=aiohttp.TCPConnector(), timeout=timeout
-    ) as session:
+    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
         async with session.get("https://aleph.im/") as resp:
             resp.raise_for_status()
             return {"result": resp.status, "headers": resp.headers}
@@ -151,9 +144,7 @@ async def read_internet():
 async def post_a_message():
     """Post a message on the Aleph network"""
 
-    account = await RemoteAccount.from_crypto_host(
-        host="http://localhost", unix_socket="/tmp/socat-socket"
-    )
+    account = await RemoteAccount.from_crypto_host(host="http://localhost", unix_socket="/tmp/socat-socket")
 
     content = {
         "date": datetime.utcnow().isoformat(),
@@ -269,9 +260,7 @@ def platform_pip_freeze():
 async def aleph_event(event):
     print("aleph_event", event)
     async with aiohttp.ClientSession(connector=aiohttp.TCPConnector()) as session:
-        async with session.get(
-            "https://official.aleph.cloud/api/v0/info/public.json"
-        ) as resp:
+        async with session.get("https://official.aleph.cloud/api/v0/info/public.json") as resp:
             print("RESP", resp)
             resp.raise_for_status()
     return {"result": "Good"}
diff --git a/examples/example_pip/main.py b/examples/example_pip/main.py
index 148941645..ad0b66b5b 100644
--- a/examples/example_pip/main.py
+++ b/examples/example_pip/main.py
@@ -8,5 +8,4 @@
 async def root():
     data = range(10)
     df = pandas.DataFrame(data)
-    return Response(content=df.to_html(),
-                    media_type='text/html')
+    return Response(content=df.to_html(), media_type="text/html")
diff --git a/firecracker/__init__.py b/firecracker/__init__.py
deleted file mode 100644
index 321ad3266..000000000
--- a/firecracker/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from .config import FirecrackerConfig
-from .microvm import MicroVM
diff --git a/packaging/Makefile b/packaging/Makefile
index 88e7b59ce..abc851e2a 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -6,9 +6,7 @@ debian-package: debian-package-resources debian-package-code
 debian-package-code:
 	rm -fr ./aleph-vm/opt/aleph-vm
 	mkdir -p ./aleph-vm/opt/aleph-vm
-	cp -r ../vm_supervisor ./aleph-vm/opt/aleph-vm/
-	cp -r ../guest_api ./aleph-vm/opt/aleph-vm/
-	cp -r ../firecracker ./aleph-vm/opt/aleph-vm/
+	cp -r ../src/aleph ./aleph-vm/opt/aleph-vm/
 
 	# Fake data for diagnostic and benchmarks
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/
@@ -49,7 +47,7 @@ download-ipfs-kubo: target-dir build-dir
 
 version:
 	python3 ./version_from_git.py --inplace deb aleph-vm/DEBIAN/control
-	python3 ./version_from_git.py --inplace __version__ ../vm_supervisor/version.py
+	python3 ./version_from_git.py --inplace __version__ ../src/aleph/vm/orchestrator/version.py
 
 build-dir:
 	mkdir -p target
diff --git a/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service b/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
index e5a904d28..ab4006c2a 100644
--- a/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
+++ b/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
@@ -10,7 +10,7 @@ WorkingDirectory=/opt/aleph-vm
 Environment=PYTHONPATH=/opt/aleph-vm/:$PYTHONPATH
 Environment=PYTHONDONTWRITEBYTECODE="enabled"
 EnvironmentFile=/etc/aleph-vm/supervisor.env
-ExecStart=python3 -m vm_supervisor --print-settings --very-verbose
+ExecStart=python3 -m aleph.vm.orchestrator --print-settings --very-verbose
 Restart=always
 RestartSec=10s
 
diff --git a/packaging/debian-11.dockerfile b/packaging/debian-11.dockerfile
index 7b5465b16..677c28827 100644
--- a/packaging/debian-11.dockerfile
+++ b/packaging/debian-11.dockerfile
@@ -9,9 +9,7 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /opt
-COPY ../vm_supervisor ./vm_supervisor
-COPY ../guest_api ./guest_api
-COPY ../firecracker ./firecracker
+COPY ../src/aleph/ ./src/aleph
 COPY ../packaging ./packaging
 COPY ../kernels ./kernels
 
diff --git a/packaging/debian-12.dockerfile b/packaging/debian-12.dockerfile
index 2e62644dd..f4177b128 100644
--- a/packaging/debian-12.dockerfile
+++ b/packaging/debian-12.dockerfile
@@ -9,9 +9,7 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /opt
-COPY ../vm_supervisor ./vm_supervisor
-COPY ../guest_api ./guest_api
-COPY ../firecracker ./firecracker
+COPY ../src/aleph ./src/aleph
 COPY ../packaging ./packaging
 COPY ../kernels ./kernels
 
diff --git a/packaging/ubuntu-22.04.dockerfile b/packaging/ubuntu-22.04.dockerfile
index 8c42c8637..32467a5e9 100644
--- a/packaging/ubuntu-22.04.dockerfile
+++ b/packaging/ubuntu-22.04.dockerfile
@@ -9,9 +9,7 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /opt
-COPY ../vm_supervisor ./vm_supervisor
-COPY ../guest_api ./guest_api
-COPY ../firecracker ./firecracker
+COPY ../src/aleph ./src/aleph
 COPY ../packaging ./packaging
 COPY ../kernels ./kernels
 
diff --git a/packaging/version_from_git.py b/packaging/version_from_git.py
index f64fc263d..327b2e2f4 100755
--- a/packaging/version_from_git.py
+++ b/packaging/version_from_git.py
@@ -9,22 +9,24 @@
 Pass the path to the target file to edit in argument.
 """
 
-import sys
 import os.path
-import subprocess
 import re
+import subprocess
+import sys
 
 script_path, *args, format_, target_file_path = sys.argv
 
 for arg in args:
-    if arg not in ('--inplace', '--stdout'):
-        print("Usage: version_from_git.py [target FILE PATH] [FORMAT] [OPTION...]\n\n"
-              "set the version number of a Debian package based on the current git commit\n\n"
-              "supported formats are 'deb' and 'setup.py'\n\n"
-              "  --help       print this message\n"
-              "  --inplace    edit file in place\n"
-              "  --inplace    edit file in place\n"
-              "  --stdout     print the result on stdout\n")
+    if arg not in ("--inplace", "--stdout"):
+        print(
+            "Usage: version_from_git.py [target FILE PATH] [FORMAT] [OPTION...]\n\n"
+            "set the version number of a Debian package based on the current git commit\n\n"
+            "supported formats are 'deb' and 'setup.py'\n\n"
+            "  --help       print this message\n"
+            "  --inplace    edit file in place\n"
+            "  --inplace    edit file in place\n"
+            "  --stdout     print the result on stdout\n"
+        )
         sys.exit(1)
 
 if not os.path.isfile(target_file_path):
@@ -33,27 +35,27 @@
 
 
 def get_git_version():
-    output = subprocess.check_output(('git', 'describe', '--tags'))
+    output = subprocess.check_output(("git", "describe", "--tags"))
     return output.decode().strip()
 
 
 version = get_git_version()
 
-with open(target_file_path, 'r') as target_file:
+with open(target_file_path, "r") as target_file:
     target_content = target_file.read()
 
-if format_ == 'deb':
+if format_ == "deb":
     updated_content = re.sub(r"(Version:)\w*(.*)", "\\1 {}".format(version), target_content)
-elif format_ == 'setup.py':
+elif format_ == "setup.py":
     updated_content = re.sub(r"(version)\w*=(.*)'", "\\1='{}'".format(version), target_content)
-elif format_ == '__version__':
+elif format_ == "__version__":
     updated_content = re.sub(r"(__version__)\w*(.*)", "\\1 = '{}'".format(version), target_content)
 else:
     print("Format must be 'deb', 'setup.py' or '__version__', not '{}'".format(format_))
 
-if '--inplace' in args:
-    with open(target_file_path, 'w') as target_file:
+if "--inplace" in args:
+    with open(target_file_path, "w") as target_file:
         target_file.write(updated_content)
 
-if '--stdout' in args:
+if "--stdout" in args:
     print(updated_content)
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index a368552a7..beb7060ea 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -144,9 +144,7 @@ def setup_network(
 
     # Forward compatibility with future supervisors that pass the mask with the IP.
     if ipv4 and ("/" not in ipv4):
-        logger.warning(
-            "Not passing the mask with the IP is deprecated and will be unsupported"
-        )
+        logger.warning("Not passing the mask with the IP is deprecated and will be unsupported")
         ipv4 = f"{ipv4}/24"
 
     addresses = [ip for ip in [ipv4, ipv6] if ip]
@@ -233,9 +231,7 @@ async def send(response: Dict):
         )
 
 
-async def setup_code_asgi(
-    code: bytes, encoding: Encoding, entrypoint: str
-) -> ASGIApplication:
+async def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplication:
     # Allow importing packages from /opt/packages, give it priority
     sys.path.insert(0, "/opt/packages")
 
@@ -273,9 +269,7 @@ async def setup_code_asgi(
     return ASGIApplication(app)
 
 
-def setup_code_executable(
-    code: bytes, encoding: Encoding, entrypoint: str
-) -> subprocess.Popen:
+def setup_code_executable(code: bytes, encoding: Encoding, entrypoint: str) -> subprocess.Popen:
     logger.debug("Extracting code")
     if encoding == Encoding.squashfs:
         path = f"/opt/code/{entrypoint}"
@@ -312,20 +306,14 @@ async def setup_code(
     interface: Interface,
 ) -> Union[ASGIApplication, subprocess.Popen]:
     if interface == Interface.asgi:
-        return await setup_code_asgi(
-            code=code, encoding=encoding, entrypoint=entrypoint
-        )
+        return await setup_code_asgi(code=code, encoding=encoding, entrypoint=entrypoint)
     elif interface == Interface.executable:
-        return setup_code_executable(
-            code=code, encoding=encoding, entrypoint=entrypoint
-        )
+        return setup_code_executable(code=code, encoding=encoding, entrypoint=entrypoint)
     else:
         raise ValueError("Invalid interface. This should never happen.")
 
 
-async def run_python_code_http(
-    application: ASGIApplication, scope: dict
-) -> Tuple[Dict, Dict, str, Optional[bytes]]:
+async def run_python_code_http(application: ASGIApplication, scope: dict) -> Tuple[Dict, Dict, str, Optional[bytes]]:
     logger.debug("Running code")
     with StringIO() as buf, redirect_stdout(buf):
         # Execute in the same process, saves ~20ms than a subprocess
@@ -334,11 +322,7 @@ async def run_python_code_http(
         scope_body: bytes = scope.pop("body")
 
         async def receive():
-            type_ = (
-                "http.request"
-                if scope["type"] in ("http", "websocket")
-                else "aleph.message"
-            )
+            type_ = "http.request" if scope["type"] in ("http", "websocket") else "aleph.message"
             return {"type": type_, "body": scope_body, "more_body": False}
 
         send_queue: asyncio.Queue = asyncio.Queue()
@@ -389,9 +373,7 @@ async def make_request(session, scope):
         data=scope.get("body", None),
     ) as resp:
         headers = {
-            "headers": [
-                (a.encode("utf-8"), b.encode("utf-8")) for a, b in resp.headers.items()
-            ],
+            "headers": [(a.encode("utf-8"), b.encode("utf-8")) for a, b in resp.headers.items()],
             "status": resp.status,
         }
         body = {"body": await resp.content.read()}
@@ -450,9 +432,7 @@ async def process_instruction(
             logger.debug("Application terminated")
             # application.communicate()
         else:
-            await wait_for_lifespan_event_completion(
-                application=application, event="shutdown"
-            )
+            await wait_for_lifespan_event_completion(application=application, event="shutdown")
         yield b"STOP\n"
         logger.debug("Supervisor informed of halt")
         raise ShutdownException
@@ -460,9 +440,7 @@ async def process_instruction(
         # Execute shell commands in the form `!ls /`
         msg = instruction[1:].decode()
         try:
-            process_output = subprocess.check_output(
-                msg, stderr=subprocess.STDOUT, shell=True
-            )
+            process_output = subprocess.check_output(msg, stderr=subprocess.STDOUT, shell=True)
             yield process_output
         except subprocess.CalledProcessError as error:
             yield str(error).encode() + b"\n" + error.output
@@ -485,9 +463,7 @@ async def process_instruction(
                     application=application, scope=payload.scope
                 )
             elif interface == Interface.executable:
-                headers, body, output, output_data = await run_executable_http(
-                    scope=payload.scope
-                )
+                headers, body, output, output_data = await run_executable_http(scope=payload.scope)
             else:
                 raise ValueError("Unknown interface. This should never happen")
 
@@ -604,9 +580,7 @@ async def handle_instruction(reader, writer):
             logger.debug(f"<<<\n\n{data_to_print}\n\n>>>")
 
         try:
-            async for result in process_instruction(
-                instruction=data, interface=config.interface, application=app
-            ):
+            async for result in process_instruction(instruction=data, interface=config.interface, application=app):
                 writer.write(result)
                 await writer.drain()
 
diff --git a/guest_api/__init__.py b/src/aleph/__init__.py
similarity index 100%
rename from guest_api/__init__.py
rename to src/aleph/__init__.py
diff --git a/vm_supervisor/migrations/__init__.py b/src/aleph/vm/__init__.py
similarity index 100%
rename from vm_supervisor/migrations/__init__.py
rename to src/aleph/vm/__init__.py
diff --git a/vm_supervisor/network/__init__.py b/src/aleph/vm/controllers/__init__.py
similarity index 100%
rename from vm_supervisor/network/__init__.py
rename to src/aleph/vm/controllers/__init__.py
diff --git a/vm_supervisor/vm/firecracker/__init__.py b/src/aleph/vm/controllers/firecracker/__init__.py
similarity index 100%
rename from vm_supervisor/vm/firecracker/__init__.py
rename to src/aleph/vm/controllers/firecracker/__init__.py
diff --git a/vm_supervisor/vm/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
similarity index 89%
rename from vm_supervisor/vm/firecracker/executable.py
rename to src/aleph/vm/controllers/firecracker/executable.py
index 3bad89bd5..67fede347 100644
--- a/vm_supervisor/vm/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -9,26 +9,24 @@
 from multiprocessing import Process, set_start_method
 from os.path import exists, isfile
 from pathlib import Path
-from typing import Dict, Generic, List, Optional, TypeVar
+from typing import Generic, Optional, TypeVar
 
 from aiohttp import ClientResponseError
-from aleph_message.models import ItemHash
+from aleph_message.models import ExecutableContent, ItemHash
 from aleph_message.models.execution.environment import MachineResources
 
-from firecracker.config import FirecrackerConfig
-from firecracker.microvm import MicroVM
-from guest_api.__main__ import run_guest_api
-from vm_supervisor.conf import settings
-from vm_supervisor.models import ExecutableContent
-from vm_supervisor.network.firewall import teardown_nftables_for_vm
-from vm_supervisor.network.interfaces import TapInterface
-from vm_supervisor.snapshots import CompressedDiskVolumeSnapshot
-from vm_supervisor.storage import get_volume_path
+from aleph.vm.guest_api.__main__ import run_guest_api
+from aleph.vm.hypervisors.firecracker.microvm import FirecrackerConfig, MicroVM
+from aleph.vm.orchestrator.conf import settings
+from aleph.vm.orchestrator.network.firewall import teardown_nftables_for_vm
+from aleph.vm.orchestrator.network.interfaces import TapInterface
+from aleph.vm.orchestrator.snapshots import CompressedDiskVolumeSnapshot
+from aleph.vm.orchestrator.storage import get_volume_path
 
 try:
     import psutil  # type: ignore [no-redef]
 except ImportError:
-    psutil = None
+    psutil = None  # type: ignore [assignment]
 
 logger = logging.getLogger(__name__)
 set_start_method("spawn")
@@ -66,9 +64,9 @@ class BaseConfiguration:
     vm_hash: ItemHash
     ip: Optional[str] = None
     route: Optional[str] = None
-    dns_servers: List[str] = field(default_factory=list)
-    volumes: List[Volume] = field(default_factory=list)
-    variables: Optional[Dict[str, str]] = None
+    dns_servers: list[str] = field(default_factory=list)
+    volumes: list[Volume] = field(default_factory=list)
+    variables: Optional[dict[str, str]] = None
 
 
 @dataclass
@@ -85,7 +83,7 @@ class AlephFirecrackerResources:
 
     kernel_image_path: Path
     rootfs_path: Path
-    volumes: List[HostVolume]
+    volumes: list[HostVolume]
     namespace: str
 
     def __init__(self, message_content: ExecutableContent, namespace: str):
@@ -107,9 +105,7 @@ async def download_volumes(self):
             volumes.append(
                 HostVolume(
                     mount=volume.mount,
-                    path_on_host=(
-                        await get_volume_path(volume=volume, namespace=self.namespace)
-                    ),
+                    path_on_host=(await get_volume_path(volume=volume, namespace=self.namespace)),
                     read_only=volume.is_read_only(),
                 )
             )
@@ -238,7 +234,8 @@ async def start(self):
         logger.debug(f"Starting VM={self.vm_id}")
 
         if not self.fvm:
-            raise ValueError("No VM found. Call setup() before start()")
+            msg = "No VM found. Call setup() before start()"
+            raise ValueError(msg)
 
         try:
             await self.fvm.start(self._firecracker_config)
diff --git a/vm_supervisor/vm/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
similarity index 79%
rename from vm_supervisor/vm/firecracker/instance.py
rename to src/aleph/vm/controllers/firecracker/instance.py
index 8c50aaf80..7f18d0585 100644
--- a/vm_supervisor/vm/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -4,13 +4,13 @@
 import logging
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Dict, List, Optional, Union
+from typing import Optional, Union
 
 import yaml
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
 
-from firecracker.config import (
+from aleph.vm.hypervisors.firecracker.config import (
     BootSource,
     Drive,
     FirecrackerConfig,
@@ -18,23 +18,22 @@
     NetworkInterface,
     Vsock,
 )
-from firecracker.microvm import setfacl
-from vm_supervisor.conf import settings
-from vm_supervisor.network.interfaces import TapInterface
-from vm_supervisor.snapshots import (
+from aleph.vm.hypervisors.firecracker.microvm import setfacl
+from aleph.vm.orchestrator.conf import settings
+from aleph.vm.orchestrator.network.interfaces import TapInterface
+from aleph.vm.orchestrator.snapshots import (
     CompressedDiskVolumeSnapshot,
     DiskVolume,
     DiskVolumeSnapshot,
 )
-from vm_supervisor.storage import (
+from aleph.vm.orchestrator.storage import (
     NotEnoughDiskSpace,
     check_disk_space,
     create_devmapper,
     create_volume_file,
 )
-from vm_supervisor.utils import HostNotFoundError, ping
+from aleph.vm.orchestrator.utils import HostNotFoundError, ping, run_in_subprocess
 
-from ...utils import run_in_subprocess
 from .executable import (
     AlephFirecrackerExecutable,
     AlephFirecrackerResources,
@@ -46,12 +45,8 @@
 
 class AlephInstanceResources(AlephFirecrackerResources):
     async def download_runtime(self):
-        self.rootfs_path = await create_devmapper(
-            self.message_content.rootfs, self.namespace
-        )
-        assert (
-            self.rootfs_path.is_block_device()
-        ), f"Runtime not found on {self.rootfs_path}"
+        self.rootfs_path = await create_devmapper(self.message_content.rootfs, self.namespace)
+        assert self.rootfs_path.is_block_device(), f"Runtime not found on {self.rootfs_path}"
 
     async def download_all(self):
         await asyncio.gather(
@@ -96,12 +91,8 @@ async def setup(self):
 
         self._firecracker_config = FirecrackerConfig(
             boot_source=BootSource(
-                kernel_image_path=Path(
-                    self.fvm.enable_kernel(self.resources.kernel_image_path)
-                ),
-                boot_args=BootSource.args(
-                    enable_console=self.enable_console, writable=True
-                ),
+                kernel_image_path=Path(self.fvm.enable_kernel(self.resources.kernel_image_path)),
+                boot_args=BootSource.args(enable_console=self.enable_console, writable=True),
             ),
             drives=[
                 Drive(
@@ -121,24 +112,19 @@ async def setup(self):
                 mem_size_mib=self.hardware_resources.memory,
             ),
             vsock=Vsock(),
-            network_interfaces=[
-                NetworkInterface(
-                    iface_id="eth0", host_dev_name=self.tap_interface.device_name
-                )
-            ]
+            network_interfaces=[NetworkInterface(iface_id="eth0", host_dev_name=self.tap_interface.device_name)]
             if self.enable_networking
             else [],
         )
 
     async def wait_for_init(self) -> None:
         """Wait for the init process of the instance to be ready."""
-        assert (
-            self.enable_networking and self.tap_interface
-        ), f"Network not enabled for VM {self.vm_id}"
+        assert self.enable_networking and self.tap_interface, f"Network not enabled for VM {self.vm_id}"
 
         ip = self.get_vm_ip()
         if not ip:
-            raise ValueError("Host IP not available")
+            msg = "Host IP not available"
+            raise ValueError(msg)
 
         ip = ip.split("/", 1)[0]
 
@@ -162,18 +148,14 @@ async def configure(self):
 
     async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
         """Create a VM snapshot"""
-        volume_path = await create_volume_file(
-            self.resources.message_content.rootfs, self.resources.namespace
-        )
+        volume_path = await create_volume_file(self.resources.message_content.rootfs, self.resources.namespace)
         volume = DiskVolume(path=volume_path)
 
         if not check_disk_space(volume.size):
             raise NotEnoughDiskSpace
 
         snapshot = await volume.take_snapshot()
-        compressed_snapshot = await snapshot.compress(
-            settings.SNAPSHOT_COMPRESSION_ALGORITHM
-        )
+        compressed_snapshot = await snapshot.compress(settings.SNAPSHOT_COMPRESSION_ALGORITHM)
 
         if self.latest_snapshot:
             self.latest_snapshot.delete()
@@ -190,7 +172,7 @@ def _encode_user_data(self) -> bytes:
 
         ssh_authorized_keys = self.resources.message_content.authorized_keys or []
 
-        config: Dict[str, Union[str, bool, List[str]]] = {
+        config: dict[str, Union[str, bool, list[str]]] = {
             "hostname": self._get_hostname(),
             "disable_root": False,
             "ssh_pwauth": False,
@@ -200,18 +182,14 @@ def _encode_user_data(self) -> bytes:
         }
 
         cloud_config_header = "#cloud-config\n"
-        config_output = yaml.safe_dump(
-            config, default_flow_style=False, sort_keys=False
-        )
+        config_output = yaml.safe_dump(config, default_flow_style=False, sort_keys=False)
 
         return (cloud_config_header + config_output).encode()
 
     def _create_network_file(self) -> bytes:
         """Creates network configuration file for cloud-init tool"""
 
-        assert (
-            self.enable_networking and self.tap_interface
-        ), f"Network not enabled for VM {self.vm_id}"
+        assert self.enable_networking and self.tap_interface, f"Network not enabled for VM {self.vm_id}"
 
         ip = self.get_vm_ip()
         route = self.get_vm_route()
@@ -234,9 +212,7 @@ def _create_network_file(self) -> bytes:
             "version": 2,
         }
 
-        return yaml.safe_dump(
-            network, default_flow_style=False, sort_keys=False
-        ).encode()
+        return yaml.safe_dump(network, default_flow_style=False, sort_keys=False).encode()
 
     def _create_metadata_file(self) -> bytes:
         """Creates metadata configuration file for cloud-init tool"""
diff --git a/vm_supervisor/vm/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
similarity index 71%
rename from vm_supervisor/vm/firecracker/program.py
rename to src/aleph/vm/controllers/firecracker/program.py
index 407a1dcc6..a1a9aeb37 100644
--- a/vm_supervisor/vm/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -8,15 +8,14 @@
 from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple
 
 import msgpack
 from aiohttp import ClientResponseError
-from aleph_message.models import ItemHash
+from aleph_message.models import ExecutableContent, ItemHash
 from aleph_message.models.execution.base import Encoding
 from aleph_message.models.execution.environment import MachineResources
 
-from firecracker.config import (
+from aleph.vm.hypervisors.firecracker.config import (
     BootSource,
     Drive,
     FirecrackerConfig,
@@ -24,13 +23,12 @@
     NetworkInterface,
     Vsock,
 )
-from firecracker.microvm import RuntimeConfiguration, setfacl
-from vm_supervisor.conf import settings
-from vm_supervisor.models import ExecutableContent
-from vm_supervisor.network.interfaces import TapInterface
-from vm_supervisor.storage import get_code_path, get_data_path, get_runtime_path
+from aleph.vm.hypervisors.firecracker.microvm import RuntimeConfiguration, setfacl
+from aleph.vm.orchestrator.conf import settings
+from aleph.vm.orchestrator.network.interfaces import TapInterface
+from aleph.vm.orchestrator.storage import get_code_path, get_data_path, get_runtime_path
+from aleph.vm.orchestrator.utils import MsgpackSerializable
 
-from ...utils import MsgpackSerializable
 from .executable import (
     AlephFirecrackerExecutable,
     AlephFirecrackerResources,
@@ -47,12 +45,13 @@ class FileTooLargeError(Exception):
     pass
 
 
-def read_input_data(path_to_data: Optional[Path]) -> Optional[bytes]:
+def read_input_data(path_to_data: Path | None) -> bytes | None:
     if not path_to_data:
         return None
 
     if os.path.getsize(path_to_data) > settings.MAX_DATA_ARCHIVE_SIZE:
-        raise FileTooLargeError("Data file too large to pass as an inline zip")
+        msg = "Data file too large to pass as an inline zip"
+        raise FileTooLargeError(msg)
 
     return path_to_data.read_bytes()
 
@@ -76,12 +75,12 @@ def from_entrypoint(cls, entrypoint: str):
 class ProgramVmConfiguration(MsgpackSerializable):
     interface: Interface
     vm_hash: ItemHash
-    ip: Optional[str] = None
-    ipv6: Optional[str] = None
-    route: Optional[str] = None
-    dns_servers: List[str] = field(default_factory=list)
-    volumes: List[Volume] = field(default_factory=list)
-    variables: Optional[Dict[str, str]] = None
+    ip: str | None = None
+    ipv6: str | None = None
+    route: str | None = None
+    dns_servers: list[str] = field(default_factory=list)
+    volumes: list[Volume] = field(default_factory=list)
+    variables: dict[str, str] | None = None
 
 
 @dataclass
@@ -95,33 +94,25 @@ class ConfigurationPayloadV1(ConfigurationPayload):
     Configuration payload for runtime v1.
     """
 
-    input_data: Optional[bytes]
+    input_data: bytes | None
     interface: Interface
     vm_hash: str
     encoding: Encoding
     entrypoint: str
-    code: Optional[bytes]
-    ip: Optional[str]
-    route: Optional[str]
-    dns_servers: List[str]
-    volumes: List[Volume]
-    variables: Optional[Dict[str, str]]
+    code: bytes | None
+    ip: str | None
+    route: str | None
+    dns_servers: list[str]
+    volumes: list[Volume]
+    variables: dict[str, str] | None
 
     @classmethod
-    def from_program_config(
-        cls, program_config: ProgramConfiguration
-    ) -> ConfigurationPayload:
+    def from_program_config(cls, program_config: ProgramConfiguration) -> ConfigurationPayload:
         """Converts a program configuration into a configuration payload
         to be sent to a runtime.
         """
-        field_names = set(f.name for f in dataclasses.fields(cls))
-        return cls(
-            **{
-                k: v
-                for k, v in dataclasses.asdict(program_config).items()
-                if k in field_names
-            }
-        )
+        field_names = {f.name for f in dataclasses.fields(cls)}
+        return cls(**{k: v for k, v in dataclasses.asdict(program_config).items() if k in field_names})
 
 
 @dataclass
@@ -131,40 +122,36 @@ class ConfigurationPayloadV2(ConfigurationPayloadV1):
     Adds support for IPv6.
     """
 
-    ipv6: Optional[str]
-    ipv6_gateway: Optional[str]
-    authorized_keys: Optional[List[str]]
+    ipv6: str | None
+    ipv6_gateway: str | None
+    authorized_keys: list[str] | None
 
 
 @dataclass
 class ProgramConfiguration:
     """Configuration passed to the init of the virtual machine in order to start the program."""
 
-    input_data: Optional[bytes]
+    input_data: bytes | None
     interface: Interface
     vm_hash: str
     encoding: Encoding
     entrypoint: str
-    code: Optional[bytes] = None
-    ip: Optional[str] = None
-    ipv6: Optional[str] = None
-    route: Optional[str] = None
-    ipv6_gateway: Optional[str] = None
-    dns_servers: List[str] = field(default_factory=list)
-    volumes: List[Volume] = field(default_factory=list)
-    variables: Optional[Dict[str, str]] = None
-    authorized_keys: Optional[List[str]] = None
-
-    def to_runtime_format(
-        self, runtime_config: RuntimeConfiguration
-    ) -> ConfigurationPayload:
+    code: bytes | None = None
+    ip: str | None = None
+    ipv6: str | None = None
+    route: str | None = None
+    ipv6_gateway: str | None = None
+    dns_servers: list[str] = field(default_factory=list)
+    volumes: list[Volume] = field(default_factory=list)
+    variables: dict[str, str] | None = None
+    authorized_keys: list[str] | None = None
+
+    def to_runtime_format(self, runtime_config: RuntimeConfiguration) -> ConfigurationPayload:
         if runtime_config.version == "1.0.0":
             return ConfigurationPayloadV1.from_program_config(self)
 
         if runtime_config.version != "2.0.0":
-            logger.warning(
-                "This runtime version may be unsupported: %s", runtime_config.version
-            )
+            logger.warning("This runtime version may be unsupported: %s", runtime_config.version)
 
         return ConfigurationPayloadV2.from_program_config(self)
 
@@ -174,15 +161,15 @@ class ConfigurationResponse:
     """Response received from the virtual machine in response to a request."""
 
     success: bool
-    error: Optional[str] = None
-    traceback: Optional[str] = None
+    error: str | None = None
+    traceback: str | None = None
 
 
 @dataclass
 class RunCodePayload(MsgpackSerializable):
     """Information passed to the init of the virtual machine to launch a function/path of the program."""
 
-    scope: Dict
+    scope: dict
 
 
 class AlephProgramResources(AlephFirecrackerResources):
@@ -192,7 +179,7 @@ class AlephProgramResources(AlephFirecrackerResources):
     code_path: Path
     code_encoding: Encoding
     code_entrypoint: str
-    data_path: Optional[Path]
+    data_path: Path | None
 
     def __init__(self, message_content: ExecutableContent, namespace: str):
         super().__init__(message_content, namespace)
@@ -219,10 +206,11 @@ async def download_data(self) -> None:
         if self.message_content.data:
             data_ref: str = self.message_content.data.ref
             try:
-                self.data_path = await get_data_path(data_ref)
+                data_path = await get_data_path(data_ref)
+                self.data_path = data_path
             except ClientResponseError as error:
                 raise ResourceDownloadError(error)
-            assert self.data_path.is_file(), f"Data not found on {self.data_path}"
+            assert data_path.is_file(), f"Data not found on {data_path}"
         else:
             self.data_path = None
 
@@ -236,11 +224,9 @@ async def download_all(self):
         )
 
 
-def get_volumes_for_program(
-    resources: AlephProgramResources, drives: List[Drive]
-) -> Tuple[Optional[bytes], List[Volume]]:
-    code: Optional[bytes]
-    volumes: List[Volume]
+def get_volumes_for_program(resources: AlephProgramResources, drives: list[Drive]) -> tuple[bytes | None, list[Volume]]:
+    code: bytes | None
+    volumes: list[Volume]
     if resources.code_encoding == Encoding.squashfs:
         code = b""
         volumes = [Volume(mount="/opt/code", device="vdb", read_only=True)] + [
@@ -253,7 +239,8 @@ def get_volumes_for_program(
         ]
     else:
         if os.path.getsize(resources.code_path) > settings.MAX_PROGRAM_ARCHIVE_SIZE:
-            raise FileTooLargeError("Program file too large to pass as an inline zip")
+            msg = "Program file too large to pass as an inline zip"
+            raise FileTooLargeError(msg)
 
         code = resources.code_path.read_bytes() if resources.code_path else None
         volumes = [
@@ -268,7 +255,7 @@ def get_volumes_for_program(
 
 
 class AlephFirecrackerProgram(AlephFirecrackerExecutable[ProgramVmConfiguration]):
-    vm_configuration: Optional[ProgramVmConfiguration]
+    vm_configuration: ProgramVmConfiguration | None
     resources: AlephProgramResources
     is_instance = False
 
@@ -278,9 +265,9 @@ def __init__(
         vm_hash: ItemHash,
         resources: AlephProgramResources,
         enable_networking: bool = False,
-        enable_console: Optional[bool] = None,
+        enable_console: bool | None = None,
         hardware_resources: MachineResources = MachineResources(),
-        tap_interface: Optional[TapInterface] = None,
+        tap_interface: TapInterface | None = None,
     ):
         super().__init__(
             vm_id,
@@ -298,12 +285,8 @@ async def setup(self):
 
         self._firecracker_config = FirecrackerConfig(
             boot_source=BootSource(
-                kernel_image_path=Path(
-                    self.fvm.enable_kernel(self.resources.kernel_image_path)
-                ),
-                boot_args=BootSource.args(
-                    enable_console=self.enable_console, writable=False
-                ),
+                kernel_image_path=Path(self.fvm.enable_kernel(self.resources.kernel_image_path)),
+                boot_args=BootSource.args(enable_console=self.enable_console, writable=False),
             ),
             drives=[
                 Drive(
@@ -315,8 +298,7 @@ async def setup(self):
             ]
             + (
                 [self.fvm.enable_drive(self.resources.code_path)]
-                if hasattr(self.resources, "code_encoding")
-                and self.resources.code_encoding == Encoding.squashfs
+                if hasattr(self.resources, "code_encoding") and self.resources.code_encoding == Encoding.squashfs
                 else []
             )
             + [
@@ -328,11 +310,7 @@ async def setup(self):
                 mem_size_mib=self.hardware_resources.memory,
             ),
             vsock=Vsock(),
-            network_interfaces=[
-                NetworkInterface(
-                    iface_id="eth0", host_dev_name=self.tap_interface.device_name
-                )
-            ]
+            network_interfaces=[NetworkInterface(iface_id="eth0", host_dev_name=self.tap_interface.device_name)]
             if self.enable_networking
             else [],
         )
@@ -344,25 +322,21 @@ async def wait_for_init(self) -> None:
     async def configure(self) -> None:
         """Configure the VM by sending configuration info to it's init"""
 
-        code: Optional[bytes]
-        volumes: List[Volume]
+        code: bytes | None
+        volumes: list[Volume]
 
-        code, volumes = get_volumes_for_program(
-            resources=self.resources, drives=self.fvm.drives
-        )
+        code, volumes = get_volumes_for_program(resources=self.resources, drives=self.fvm.drives)
         interface: Interface = Interface.from_entrypoint(self.resources.code_entrypoint)
-        input_data: Optional[bytes] = read_input_data(self.resources.data_path)
+        input_data: bytes | None = read_input_data(self.resources.data_path)
 
-        await self._setup_configuration(
-            code=code, input_data=input_data, interface=interface, volumes=volumes
-        )
+        await self._setup_configuration(code=code, input_data=input_data, interface=interface, volumes=volumes)
 
     async def _setup_configuration(
         self,
-        code: Optional[bytes],
-        input_data: Optional[bytes],
+        code: bytes | None,
+        input_data: bytes | None,
         interface: Interface,
-        volumes: List[Volume],
+        volumes: list[Volume],
     ):
         """Set up the VM configuration. The program mode uses a VSOCK connection to the custom init of the virtual
         machine to send this configuration. Other modes may use Cloud-init, ..."""
@@ -378,12 +352,13 @@ async def _setup_configuration(
         ipv6_gateway = self.get_vm_ipv6_gateway()
 
         if not settings.DNS_NAMESERVERS:
-            raise ValueError("Invalid configuration: DNS nameservers missing")
+            msg = "Invalid configuration: DNS nameservers missing"
+            raise ValueError(msg)
 
         runtime_config = self.fvm.runtime_config
         assert runtime_config
 
-        authorized_keys: Optional[List[str]]
+        authorized_keys: list[str] | None
         if settings.USE_DEVELOPER_SSH_KEYS:
             authorized_keys = settings.DEVELOPER_SSH_KEYS
         else:
@@ -421,16 +396,15 @@ async def _setup_configuration(
 
     async def run_code(
         self,
-        scope: Optional[dict] = None,
+        scope: dict | None = None,
     ):
         if not self.fvm:
-            raise ValueError("MicroVM must be created first")
+            msg = "MicroVM must be created first"
+            raise ValueError(msg)
         logger.debug("running code")
         scope = scope or {}
 
-        async def communicate(
-            reader_: StreamReader, writer_: StreamWriter, scope_: dict
-        ) -> bytes:
+        async def communicate(reader_: StreamReader, writer_: StreamWriter, scope_: dict) -> bytes:
             payload = RunCodePayload(scope=scope_)
 
             writer_.write(b"CONNECT 52\n" + payload.as_msgpack())
@@ -445,11 +419,10 @@ async def communicate(
             return response
 
         try:
-            reader, writer = await asyncio.open_unix_connection(
-                path=self.fvm.vsock_path
-            )
+            reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
         except ConnectionRefusedError:
-            raise VmInitNotConnected("MicroVM may have crashed")
+            msg = "MicroVM may have crashed"
+            raise VmInitNotConnected(msg)
         try:
             return await asyncio.wait_for(
                 communicate(reader, writer, scope),
diff --git a/src/aleph/vm/guest_api/__init__.py b/src/aleph/vm/guest_api/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/guest_api/__main__.py b/src/aleph/vm/guest_api/__main__.py
similarity index 91%
rename from guest_api/__main__.py
rename to src/aleph/vm/guest_api/__main__.py
index 1692139a3..9bd1ce3c2 100644
--- a/guest_api/__main__.py
+++ b/src/aleph/vm/guest_api/__main__.py
@@ -39,9 +39,7 @@ async def proxy(request: web.Request):
     async with aiohttp.ClientSession() as session:
         async with session.request(method=request.method, url=url) as response:
             data = await response.read()
-            return web.Response(
-                body=data, status=response.status, content_type=response.content_type
-            )
+            return web.Response(body=data, status=response.status, content_type=response.content_type)
 
 
 async def repost(request: web.Request):
@@ -65,9 +63,7 @@ async def repost(request: web.Request):
     async with aiohttp.ClientSession() as session:
         async with session.post(url=url, json=new_data) as response:
             data = await response.read()
-            return web.Response(
-                body=data, status=response.status, content_type=response.content_type
-            )
+            return web.Response(body=data, status=response.status, content_type=response.content_type)
 
 
 # async def decrypt_secret(request: web.Request):
@@ -82,9 +78,7 @@ async def properties(request: web.Request):
     async with aiohttp.ClientSession() as session:
         async with session.get(url=url) as response:
             data = await response.read()
-            return web.Response(
-                body=data, status=response.status, content_type=response.content_type
-            )
+            return web.Response(body=data, status=response.status, content_type=response.content_type)
 
 
 async def sign(request: web.Request):
@@ -132,9 +126,7 @@ async def put_in_cache(request: web.Request):
     value: bytes = await request.read()
 
     redis: aioredis.Redis = await get_redis()
-    return web.json_response(
-        await redis.set(f"{prefix}:{key}", value, expire=CACHE_EXPIRES_AFTER)
-    )
+    return web.json_response(await redis.set(f"{prefix}:{key}", value, expire=CACHE_EXPIRES_AFTER))
 
 
 async def delete_from_cache(request: web.Request):
@@ -187,9 +179,7 @@ def run_guest_api(
     app.router.add_route(method="GET", path="/cache/", handler=list_keys_from_cache)
     app.router.add_route(method="GET", path="/cache/{key:.*}", handler=get_from_cache)
     app.router.add_route(method="PUT", path="/cache/{key:.*}", handler=put_in_cache)
-    app.router.add_route(
-        method="DELETE", path="/cache/{key:.*}", handler=delete_from_cache
-    )
+    app.router.add_route(method="DELETE", path="/cache/{key:.*}", handler=delete_from_cache)
 
     app.router.add_route(method="GET", path="/{tail:.*}", handler=proxy)
     app.router.add_route(method="HEAD", path="/{tail:.*}", handler=proxy)
diff --git a/src/aleph/vm/hypervisors/__init__.py b/src/aleph/vm/hypervisors/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/aleph/vm/hypervisors/firecracker/__init__.py b/src/aleph/vm/hypervisors/firecracker/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/firecracker/config.py b/src/aleph/vm/hypervisors/firecracker/config.py
similarity index 81%
rename from firecracker/config.py
rename to src/aleph/vm/hypervisors/firecracker/config.py
index 6db70cb71..67f5a71b7 100644
--- a/firecracker/config.py
+++ b/src/aleph/vm/hypervisors/firecracker/config.py
@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import List, Optional
+from typing import Optional
 
 from pydantic import BaseModel, PositiveInt
 
@@ -8,10 +8,7 @@
 
 class BootSource(BaseModel):
     kernel_image_path: Path = Path("vmlinux.bin")
-    boot_args: str = (
-        "console=ttyS0 reboot=k panic=1 pci=off "
-        "ro noapic nomodules random.trust_cpu=on"
-    )
+    boot_args: str = "console=ttyS0 reboot=k panic=1 pci=off ro noapic nomodules random.trust_cpu=on"
 
     @staticmethod
     def args(enable_console: bool = True, writable: bool = False):
@@ -53,11 +50,13 @@ class NetworkInterface(BaseModel):
 
 class FirecrackerConfig(BaseModel):
     boot_source: BootSource
-    drives: List[Drive]
+    drives: list[Drive]
     machine_config: MachineConfig
     vsock: Optional[Vsock]
-    network_interfaces: Optional[List[NetworkInterface]]
+    network_interfaces: Optional[list[NetworkInterface]]
 
     class Config:
         allow_population_by_field_name = True
-        alias_generator = lambda x: x.replace("_", "-")
+
+        def alias_generator(x):
+            return x.replace("_", "-")
diff --git a/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
similarity index 89%
rename from firecracker/microvm.py
rename to src/aleph/vm/hypervisors/firecracker/microvm.py
index 4e827d1e7..c0022bb14 100644
--- a/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -11,7 +11,7 @@
 from pathlib import Path
 from pwd import getpwnam
 from tempfile import NamedTemporaryFile
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Optional
 
 import msgpack
 
@@ -46,9 +46,7 @@ def system(command):
 async def setfacl():
     user = getuid()
     cmd = f"sudo setfacl -m u:{user}:rw /dev/kvm"
-    proc = await asyncio.create_subprocess_shell(
-        cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-    )
+    proc = await asyncio.create_subprocess_shell(cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
     stdout, stderr = await proc.communicate()
 
     if proc.returncode == 0:
@@ -76,9 +74,9 @@ class MicroVM:
     proc: Optional[asyncio.subprocess.Process] = None
     stdout_task: Optional[Task] = None
     stderr_task: Optional[Task] = None
-    log_queues: List[asyncio.Queue]
+    log_queues: list[asyncio.Queue]
     config_file_path: Optional[Path] = None
-    drives: List[Drive]
+    drives: list[Drive]
     init_timeout: float
     mounted_rootfs: Optional[Path] = None
     _unix_socket: Optional[Server] = None
@@ -121,7 +119,7 @@ def __init__(
         self.drives = []
         self.init_timeout = init_timeout
         self.runtime_config = None
-        self.log_queues: List[asyncio.Queue] = []
+        self.log_queues: list[asyncio.Queue] = []
 
     def to_dict(self):
         return {
@@ -156,18 +154,14 @@ async def start(self, config: FirecrackerConfig) -> asyncio.subprocess.Process:
         else:
             return await self.start_firecracker(config)
 
-    async def start_firecracker(
-        self, config: FirecrackerConfig
-    ) -> asyncio.subprocess.Process:
+    async def start_firecracker(self, config: FirecrackerConfig) -> asyncio.subprocess.Process:
         if os.path.exists(VSOCK_PATH):
             os.remove(VSOCK_PATH)
         if os.path.exists(self.socket_path):
             os.remove(self.socket_path)
 
         with NamedTemporaryFile(delete=False) as config_file:
-            config_file.write(
-                config.json(by_alias=True, exclude_none=True, indent=4).encode()
-            )
+            config_file.write(config.json(by_alias=True, exclude_none=True, indent=4).encode())
             config_file.flush()
             os.chmod(config_file.name, 0o644)
             self.config_file_path = Path(config_file.name)
@@ -196,18 +190,15 @@ async def start_firecracker(
         )
         return self.proc
 
-    async def start_jailed_firecracker(
-        self, config: FirecrackerConfig
-    ) -> asyncio.subprocess.Process:
+    async def start_jailed_firecracker(self, config: FirecrackerConfig) -> asyncio.subprocess.Process:
         if not self.jailer_bin_path:
-            raise ValueError("Jailer binary path is missing")
+            msg = "Jailer binary path is missing"
+            raise ValueError(msg)
         uid = str(getpwnam("jailman").pw_uid)
         gid = str(getpwnam("jailman").pw_gid)
 
         with open(f"{self.jailer_path}/tmp/config.json", "wb") as config_file:
-            config_file.write(
-                config.json(by_alias=True, exclude_none=True, indent=4).encode()
-            )
+            config_file.write(config.json(by_alias=True, exclude_none=True, indent=4).encode())
             config_file.flush()
             os.chmod(config_file.name, 0o644)
             self.config_file_path = Path(config_file.name)
@@ -273,7 +264,8 @@ def enable_rootfs(self, path_on_host: Path) -> Path:
         elif path_on_host.is_block_device():
             return self.enable_device_mapper_rootfs(path_on_host)
         else:
-            raise ValueError(f"Not a file or a block device: {path_on_host}")
+            msg = f"Not a file or a block device: {path_on_host}"
+            raise ValueError(msg)
 
     def enable_file_rootfs(self, path_on_host: Path) -> Path:
         """Make a rootfs available to the VM.
@@ -343,7 +335,7 @@ async def print_logs(self):
         while True:
             stdout = await self.proc.stdout.readline()
             for queue in self.log_queues:
-                await queue.put(('stdout', stdout))
+                await queue.put(("stdout", stdout))
             if stdout:
                 print(stdout.decode().strip())
             else:
@@ -355,13 +347,13 @@ async def print_logs_stderr(self):
         while True:
             stderr = await self.proc.stderr.readline()
             for queue in self.log_queues:
-                await queue.put(('stderr', stderr))
+                await queue.put(("stderr", stderr))
             if stderr:
                 print(stderr.decode().strip())
             else:
                 await asyncio.sleep(0.001)
 
-    def start_printing_logs(self) -> Tuple[Task, Task]:
+    def start_printing_logs(self) -> tuple[Task, Task]:
         loop = asyncio.get_running_loop()
         self.stdout_task = loop.create_task(self.print_logs())
         self.stderr_task = loop.create_task(self.print_logs_stderr())
@@ -372,12 +364,10 @@ async def wait_for_init(self):
         logger.debug("Waiting for init...")
         queue = asyncio.Queue()
 
-        async def unix_client_connected(
-            reader: asyncio.StreamReader, _writer: asyncio.StreamWriter
-        ):
+        async def unix_client_connected(reader: asyncio.StreamReader, _writer: asyncio.StreamWriter):
             data = await reader.read(1_000_000)
             if data:
-                config_dict: Dict[str, Any] = msgpack.loads(data)
+                config_dict: dict[str, Any] = msgpack.loads(data)
                 runtime_config = RuntimeConfiguration(version=config_dict["version"])
             else:
                 # Older runtimes do not send a config. Use a default.
@@ -386,14 +376,10 @@ async def unix_client_connected(
             logger.debug("Runtime version: %s", runtime_config)
             await queue.put(runtime_config)
 
-        self._unix_socket = await asyncio.start_unix_server(
-            unix_client_connected, path=f"{self.vsock_path}_52"
-        )
+        self._unix_socket = await asyncio.start_unix_server(unix_client_connected, path=f"{self.vsock_path}_52")
         system(f"chown jailman:jailman {self.vsock_path}_52")
         try:
-            self.runtime_config = await asyncio.wait_for(
-                queue.get(), timeout=self.init_timeout
-            )
+            self.runtime_config = await asyncio.wait_for(queue.get(), timeout=self.init_timeout)
             logger.debug("...signal from init received")
         except asyncio.TimeoutError:
             logger.warning("Never received signal from init")
@@ -408,9 +394,7 @@ async def shutdown(self) -> None:
             ConnectionResetError,
             ConnectionRefusedError,
         ) as error:
-            logger.warning(
-                f"VM={self.vm_id} cannot receive shutdown signal: {error.args}"
-            )
+            logger.warning(f"VM={self.vm_id} cannot receive shutdown signal: {error.args}")
             return
 
         try:
@@ -431,9 +415,7 @@ async def shutdown(self) -> None:
             if msg2 != b"STOPZ\n":
                 logger.warning(f"Unexpected response from VM: {msg2[:20]!r}")
         except ConnectionResetError as error:
-            logger.warning(
-                f"ConnectionResetError in shutdown of {self.vm_id}: {error.args}"
-            )
+            logger.warning(f"ConnectionResetError in shutdown of {self.vm_id}: {error.args}")
 
     async def stop(self):
         if self.proc:
diff --git a/vm_supervisor/INSTANCES.md b/src/aleph/vm/orchestrator/INSTANCES.md
similarity index 100%
rename from vm_supervisor/INSTANCES.md
rename to src/aleph/vm/orchestrator/INSTANCES.md
diff --git a/vm_supervisor/README.md b/src/aleph/vm/orchestrator/README.md
similarity index 95%
rename from vm_supervisor/README.md
rename to src/aleph/vm/orchestrator/README.md
index ab6fc699d..9b51fe032 100644
--- a/vm_supervisor/README.md
+++ b/src/aleph/vm/orchestrator/README.md
@@ -112,11 +112,11 @@ curl -fsSL -o ./target/vmlinux.bin https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g5
 Run the VM Supervisor with Python:
 ```shell
 export PYTHONPATH=$(pwd)
-python3 -m vm_supervisor
+python3 -m orchestrator
 ```
 or in debug mode:
 ```shell
-python3 -m vm_supervisor -vv --system-logs
+python3 -m orchestrator -vv --system-logs
 ```
 
 Test accessing the service on
@@ -128,12 +128,12 @@ The VM Supervisor can be configured using command-line arguments or using enviro
 
 List the available command-line arguments using:
 ```shell
-python3 -m vm_supervisor --help
+python3 -m orchestrator --help
 ```
 
 List available using environment variables using:
 ```shell
-python3 -m vm_supervisor --print-config --do-not-run
+python3 -m orchestrator --print-config --do-not-run
 ```
 
 Configuration environment variables can be stored in a file named `.env` in the local directory.
@@ -158,7 +158,7 @@ A runtime consist in the root filesystem used by a VM.
 Runtimes contain a customized init that allows the VM Supervisor to run
 functions within the MicroVM.
 
-Official Aleph runtimes are built using scripts located in [`../runtimes`](../runtimes), and are distributed on the Aleph network.
+Official Aleph runtimes are built using scripts located in [`../runtimes`](../../../../runtimes), and are distributed on the Aleph network.
 
 To build the default runtime locally:
 
diff --git a/vm_supervisor/__init__.py b/src/aleph/vm/orchestrator/__init__.py
similarity index 100%
rename from vm_supervisor/__init__.py
rename to src/aleph/vm/orchestrator/__init__.py
diff --git a/src/aleph/vm/orchestrator/__main__.py b/src/aleph/vm/orchestrator/__main__.py
new file mode 100644
index 000000000..a4912bc99
--- /dev/null
+++ b/src/aleph/vm/orchestrator/__main__.py
@@ -0,0 +1,4 @@
+from . import cli
+
+if __name__ == "__main__":
+    cli.main()
diff --git a/vm_supervisor/alembic.ini b/src/aleph/vm/orchestrator/alembic.ini
similarity index 98%
rename from vm_supervisor/alembic.ini
rename to src/aleph/vm/orchestrator/alembic.ini
index 1cf7e2b20..b472631ad 100644
--- a/vm_supervisor/alembic.ini
+++ b/src/aleph/vm/orchestrator/alembic.ini
@@ -2,7 +2,7 @@
 
 [alembic]
 # path to migration scripts
-;script_location = vm_supervisor/migrations
+;script_location = orchestrator/migrations
 script_location = migrations
 
 # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
diff --git a/vm_supervisor/__main__.py b/src/aleph/vm/orchestrator/cli.py
similarity index 88%
rename from vm_supervisor/__main__.py
rename to src/aleph/vm/orchestrator/cli.py
index 5342ebeac..85a952521 100644
--- a/vm_supervisor/__main__.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -7,7 +7,7 @@
 import time
 from pathlib import Path
 from statistics import mean
-from typing import Callable, Dict, List, Tuple
+from typing import Callable
 
 from aiohttp.web import Request, Response
 
@@ -29,9 +29,7 @@
 
 
 def parse_args(args):
-    parser = argparse.ArgumentParser(
-        prog="vm_supervisor", description="Aleph.im VM Supervisor"
-    )
+    parser = argparse.ArgumentParser(prog="orchestrator", description="Aleph.im VM Supervisor")
     parser.add_argument(
         "--system-logs",
         action="store_true",
@@ -50,9 +48,7 @@ def parse_args(args):
         dest="use_jailer",
         default=settings.USE_JAILER,
     )
-    parser.add_argument(
-        "--jailer", action="store_true", dest="use_jailer", default=settings.USE_JAILER
-    )
+    parser.add_argument("--jailer", action="store_true", dest="use_jailer", default=settings.USE_JAILER)
     parser.add_argument(
         "--prealloc",
         action="store",
@@ -169,9 +165,9 @@ async def benchmark(runs: int):
     FakeRequest: Request
 
     class FakeRequest:  # type: ignore[no-redef]
-        headers: Dict[str, str]
-        raw_headers: List[Tuple[bytes, bytes]]
-        match_info: Dict
+        headers: dict[str, str]
+        raw_headers: list[tuple[bytes, bytes]]
+        match_info: dict
         method: str
         query_string: str
         read: Callable
@@ -182,9 +178,7 @@ class FakeRequest:  # type: ignore[no-redef]
     fake_request.query_string = ""
 
     fake_request.headers = {"host": "127.0.0.1", "content-type": "application/json"}
-    fake_request.raw_headers = [
-        (name.encode(), value.encode()) for name, value in fake_request.headers.items()
-    ]
+    fake_request.raw_headers = [(name.encode(), value.encode()) for name, value in fake_request.headers.items()]
 
     async def fake_read() -> bytes:
         return b""
@@ -193,7 +187,7 @@ async def fake_read() -> bytes:
 
     logger.info("--- Start benchmark ---")
 
-    bench: List[float] = []
+    bench: list[float] = []
 
     # Does not make sense in benchmarks
     settings.WATCH_FOR_MESSAGES = False
@@ -213,27 +207,20 @@ async def fake_read() -> bytes:
         "/cache/keys",
     ):
         fake_request.match_info["suffix"] = path
-        response: Response = await run_code_on_request(
-            vm_hash=ref, path=path, request=fake_request
-        )
+        response: Response = await run_code_on_request(vm_hash=ref, path=path, request=fake_request)
         assert response.status == 200
 
     # Disable VM timeout to exit benchmark properly
     settings.REUSE_TIMEOUT = 0 if runs == 1 else 0.1
     path = "/"
-    for run in range(runs):
+    for _run in range(runs):
         t0 = time.time()
         fake_request.match_info["suffix"] = path
-        response2: Response = await run_code_on_request(
-            vm_hash=ref, path=path, request=fake_request
-        )
+        response2: Response = await run_code_on_request(vm_hash=ref, path=path, request=fake_request)
         assert response2.status == 200
         bench.append(time.time() - t0)
 
-    logger.info(
-        f"BENCHMARK: n={len(bench)} avg={mean(bench):03f} "
-        f"min={min(bench):03f} max={max(bench):03f}"
-    )
+    logger.info(f"BENCHMARK: n={len(bench)} avg={mean(bench):03f} min={min(bench):03f} max={max(bench):03f}")
     logger.info(bench)
 
     event = None
@@ -252,13 +239,11 @@ async def start_instance(item_hash: ItemHash) -> None:
     await start_persistent_vm(item_hash, dummy_pubsub)
 
 
-async def run_instances(instances: List[ItemHash]) -> None:
+async def run_instances(instances: list[ItemHash]) -> None:
     """Run instances from a list of message identifiers."""
     logger.info(f"Instances to run: {instances}")
 
-    await asyncio.gather(
-        *[start_instance(item_hash=instance_id) for instance_id in instances]
-    )
+    await asyncio.gather(*[start_instance(item_hash=instance_id) for instance_id in instances])
     await asyncio.Event().wait()  # wait forever
 
 
@@ -357,7 +342,3 @@ def main():
         sys.exit(0)
     else:
         supervisor.run()
-
-
-if __name__ == "__main__":
-    main()
diff --git a/vm_supervisor/conf.py b/src/aleph/vm/orchestrator/conf.py
similarity index 81%
rename from vm_supervisor/conf.py
rename to src/aleph/vm/orchestrator/conf.py
index 6bf5487a5..650d55427 100644
--- a/vm_supervisor/conf.py
+++ b/src/aleph/vm/orchestrator/conf.py
@@ -2,11 +2,12 @@
 import logging
 import os
 import re
+from collections.abc import Iterable
 from enum import Enum
 from os.path import abspath, exists, isdir, isfile, join
 from pathlib import Path
 from subprocess import CalledProcessError, check_output
-from typing import Any, Dict, Iterable, List, Literal, NewType, Optional, Union
+from typing import Any, Literal, NewType, Optional, Union
 
 from pydantic import BaseSettings, Field
 
@@ -34,7 +35,7 @@ class SnapshotCompressionAlgorithm(str, Enum):
 
 
 def etc_resolv_conf_dns_servers():
-    with open("/etc/resolv.conf", "r") as resolv_file:
+    with open("/etc/resolv.conf") as resolv_file:
         for line in resolv_file.readlines():
             ip = re.findall(r"^nameserver\s+([\w.]+)$", line)
             if ip:
@@ -72,7 +73,7 @@ def resolvectl_dns_servers_ipv4(interface: str) -> Iterable[str]:
 
 def get_default_interface() -> Optional[str]:
     """Returns the default network interface"""
-    with open("/proc/net/route", "r") as f:
+    with open("/proc/net/route") as f:
         for line in f.readlines():
             parts = line.strip().split()
             if parts[1] == "00000000":  # Indicates default route
@@ -80,7 +81,7 @@ def get_default_interface() -> Optional[str]:
     return None
 
 
-def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> List[str]:
+def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> list[str]:
     # The match syntax is not yet available as of Python 3.9
     # match dns_resolver:
     if dns_resolver == DnsResolver.detect:
@@ -92,7 +93,8 @@ def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> List[st
             if Path("/etc/resolv.conf").exists():
                 return list(etc_resolv_conf_dns_servers())
             else:
-                raise FileNotFoundError("No DNS resolver found")
+                msg = "No DNS resolver found"
+                raise FileNotFoundError(msg)
 
     elif dns_resolver == DnsResolver.resolv_conf:
         return list(etc_resolv_conf_dns_servers())
@@ -143,9 +145,7 @@ class Settings(BaseSettings):
         description="IPv6 address range assigned to the host. Example: 1111:2222:3333:4444::/64. "
         "Defaults to a local address range for compatibility with hosts not yet configured for IPv6.",
     )
-    IPV6_ALLOCATION_POLICY: IPv6AllocationPolicy = Field(
-        default=IPv6AllocationPolicy.static
-    )
+    IPV6_ALLOCATION_POLICY: IPv6AllocationPolicy = Field(default=IPv6AllocationPolicy.static)
     IPV6_SUBNET_PREFIX: int = Field(
         default=124,
         description="IPv6 subnet prefix for VMs. Made configurable for testing.",
@@ -161,7 +161,7 @@ class Settings(BaseSettings):
     )
 
     DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.detect
-    DNS_NAMESERVERS: Optional[List[str]] = None
+    DNS_NAMESERVERS: Optional[list[str]] = None
 
     FIRECRACKER_PATH = Path("/opt/firecracker/firecracker")
     JAILER_PATH = Path("/opt/firecracker/jailer")
@@ -197,29 +197,17 @@ class Settings(BaseSettings):
     )
 
     # hashlib.sha256(b"secret-token").hexdigest()
-    ALLOCATION_TOKEN_HASH = (
-        "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda"
-    )
+    ALLOCATION_TOKEN_HASH = "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda"
 
     # Tests on programs
 
     FAKE_DATA_PROGRAM: Optional[Path] = None
-    BENCHMARK_FAKE_DATA_PROGRAM = Path(
-        abspath(join(__file__, "../../examples/example_fastapi"))
-    )
+    BENCHMARK_FAKE_DATA_PROGRAM = Path(abspath(join(__file__, "../../examples/example_fastapi")))
 
-    FAKE_DATA_MESSAGE = Path(
-        abspath(join(__file__, "../../examples/program_message_from_aleph.json"))
-    )
-    FAKE_DATA_DATA: Optional[Path] = Path(
-        abspath(join(__file__, "../../examples/data/"))
-    )
-    FAKE_DATA_RUNTIME = Path(
-        abspath(join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs"))
-    )
-    FAKE_DATA_VOLUME: Optional[Path] = Path(
-        abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs"))
-    )
+    FAKE_DATA_MESSAGE = Path(abspath(join(__file__, "../../examples/program_message_from_aleph.json")))
+    FAKE_DATA_DATA: Optional[Path] = Path(abspath(join(__file__, "../../examples/data/")))
+    FAKE_DATA_RUNTIME = Path(abspath(join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs")))
+    FAKE_DATA_VOLUME: Optional[Path] = Path(abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs")))
 
     # Tests on instances
 
@@ -229,31 +217,25 @@ class Settings(BaseSettings):
     )
 
     USE_FAKE_INSTANCE_BASE = False
-    FAKE_INSTANCE_BASE = Path(
-        abspath(join(__file__, "../../runtimes/instance-debian-rootfs/rootfs.ext4"))
-    )
+    FAKE_INSTANCE_BASE = Path(abspath(join(__file__, "../../runtimes/instance-debian-rootfs/rootfs.ext4")))
     FAKE_INSTANCE_ID: str = Field(
         default="decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca",
         description="Identifier used for the 'fake instance' message defined in "
         "examples/instance_message_from_aleph.json",
     )
-    FAKE_INSTANCE_MESSAGE = Path(
-        abspath(join(__file__, "../../examples/instance_message_from_aleph.json"))
-    )
+    FAKE_INSTANCE_MESSAGE = Path(abspath(join(__file__, "../../examples/instance_message_from_aleph.json")))
 
-    CHECK_FASTAPI_VM_ID = (
-        "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af"
-    )
+    CHECK_FASTAPI_VM_ID = "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af"
 
     # Developer options
 
     SENTRY_DSN: Optional[str] = None
-    DEVELOPER_SSH_KEYS: Optional[List[str]] = []
+    DEVELOPER_SSH_KEYS: Optional[list[str]] = []
     # Using an object here forces the value to come from Python code and not from an environment variable.
     USE_DEVELOPER_SSH_KEYS: Union[Literal[False], object] = False
 
     # Fields
-    SENSITIVE_FIELDS: List[str] = Field(
+    SENSITIVE_FIELDS: list[str] = Field(
         default=["SENTRY_DSN"],
         description="Sensitive fields, redacted from `--print-settings`.",
     )
@@ -265,7 +247,8 @@ def update(self, **kwargs):
             if hasattr(self, key):
                 setattr(self, key, value)
             else:
-                raise ValueError(f"Unknown setting '{key}'")
+                msg = f"Unknown setting '{key}'"
+                raise ValueError(msg)
 
     def check(self):
         assert Path("/dev/kvm").exists(), "KVM not found on `/dev/kvm`."
@@ -273,9 +256,7 @@ def check(self):
         assert isfile(self.JAILER_PATH), f"File not found {self.JAILER_PATH}"
         assert isfile(self.LINUX_PATH), f"File not found {self.LINUX_PATH}"
         assert self.NETWORK_INTERFACE, "Network interface is not specified"
-        assert self.CONNECTOR_URL.startswith(
-            "http://"
-        ) or self.CONNECTOR_URL.startswith("https://")
+        assert self.CONNECTOR_URL.startswith("http://") or self.CONNECTOR_URL.startswith("https://")
         if self.ALLOW_VM_NETWORKING:
             assert exists(
                 f"/sys/class/net/{self.NETWORK_INTERFACE}"
@@ -287,17 +268,11 @@ def check(self):
             ), "The IPv4 address pool prefix must be shorter than an individual VM network prefix"
 
         if self.FAKE_DATA_PROGRAM:
-            assert isdir(
-                self.FAKE_DATA_PROGRAM
-            ), "Local fake program directory is missing"
+            assert isdir(self.FAKE_DATA_PROGRAM), "Local fake program directory is missing"
             assert isfile(self.FAKE_DATA_MESSAGE), "Local fake message is missing"
             assert isdir(self.FAKE_DATA_DATA), "Local fake data directory is missing"
-            assert isfile(
-                self.FAKE_DATA_RUNTIME
-            ), "Local runtime .squashfs build is missing"
-            assert isfile(
-                self.FAKE_DATA_VOLUME
-            ), "Local data volume .squashfs is missing"
+            assert isfile(self.FAKE_DATA_RUNTIME), "Local runtime .squashfs build is missing"
+            assert isfile(self.FAKE_DATA_VOLUME), "Local data volume .squashfs is missing"
 
     def setup(self):
         os.makedirs(self.MESSAGE_CACHE, exist_ok=True)
@@ -318,7 +293,7 @@ def setup(self):
             )
 
     def display(self) -> str:
-        attributes: Dict[str, Any] = {}
+        attributes: dict[str, Any] = {}
 
         for attr in self.__dict__.keys():
             if attr != attr.upper():
@@ -330,9 +305,7 @@ def display(self) -> str:
             else:
                 attributes[attr] = getattr(self, attr)
 
-        return "\n".join(
-            f"{attribute:<27} = {value}" for attribute, value in attributes.items()
-        )
+        return "\n".join(f"{attribute:<27} = {value}" for attribute, value in attributes.items())
 
     class Config:
         env_prefix = "ALEPH_VM_"
diff --git a/vm_supervisor/messages.py b/src/aleph/vm/orchestrator/messages.py
similarity index 81%
rename from vm_supervisor/messages.py
rename to src/aleph/vm/orchestrator/messages.py
index c5167a1d0..07c14f0fd 100644
--- a/vm_supervisor/messages.py
+++ b/src/aleph/vm/orchestrator/messages.py
@@ -1,6 +1,5 @@
 import asyncio
 import copy
-from typing import Tuple
 
 from aiohttp import ClientConnectorError, ClientResponseError
 from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable
@@ -29,9 +28,7 @@ async def get_latest_ref(item_hash: str) -> str:
         raise HTTPServiceUnavailable(reason="Aleph Connector unavailable")
     except ClientResponseError as error:
         if error.status == 404:
-            raise HTTPNotFound(
-                reason="Hash not found", text=f"Hash not found: {item_hash}"
-            )
+            raise HTTPNotFound(reason="Hash not found", text=f"Hash not found: {item_hash}")
         else:
             raise
 
@@ -55,25 +52,19 @@ async def update_message(message: ExecutableMessage):
             update_with_latest_ref(message.content.runtime),
             update_with_latest_ref(message.content.code),
             update_with_latest_ref(message.content.data),
-            *(
-                update_with_latest_ref(volume)
-                for volume in (message.content.volumes or [])
-            ),
+            *(update_with_latest_ref(volume) for volume in (message.content.volumes or [])),
         )
     else:
         assert message.type == MessageType.instance
         await asyncio.gather(
             update_with_latest_ref(message.content.rootfs.parent),
-            *(
-                update_with_latest_ref(volume)
-                for volume in (message.content.volumes or [])
-            ),
+            *(update_with_latest_ref(volume) for volume in (message.content.volumes or [])),
         )
 
 
 async def load_updated_message(
     ref: ItemHash,
-) -> Tuple[ExecutableMessage, ExecutableMessage]:
+) -> tuple[ExecutableMessage, ExecutableMessage]:
     original_message = await try_get_message(ref)
     message = copy.deepcopy(original_message)
     await update_message(message)
diff --git a/vm_supervisor/metrics.py b/src/aleph/vm/orchestrator/metrics.py
similarity index 88%
rename from vm_supervisor/metrics.py
rename to src/aleph/vm/orchestrator/metrics.py
index 8f54aeba3..b9aaf97fd 100644
--- a/vm_supervisor/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -1,13 +1,20 @@
 import logging
+from collections.abc import Iterable
 from pathlib import Path
-from typing import Any, Iterable
+from typing import Any
 from uuid import UUID
 
 from sqlalchemy import Column, DateTime, Float, Integer, String, create_engine
 from sqlalchemy.engine import Engine
-from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 
+try:
+    from sqlalchemy.orm import declarative_base
+except ImportError:
+    from sqlalchemy.ext.declarative import declarative_base
+
+
+
 from .conf import make_db_url, settings
 
 Session: sessionmaker
@@ -67,7 +74,7 @@ async def save_execution_data(execution_uuid: UUID, execution_data: str):
 
 async def save_record(record: ExecutionRecord):
     """Record the resource usage in database"""
-    session = Session()  # noqa: F821 undefined name 'Session'
+    session = Session()  # undefined name 'Session'
     try:
         session.add(record)
         session.commit()
@@ -77,7 +84,7 @@ async def save_record(record: ExecutionRecord):
 
 async def get_execution_records() -> Iterable[ExecutionRecord]:
     """Get the execution records from the database."""
-    session = Session()  # noqa: F821 undefined name 'Session'
+    session = Session()  # undefined name 'Session'
     try:
         return session.query(ExecutionRecord).all()
     finally:
diff --git a/src/aleph/vm/orchestrator/migrations/__init__.py b/src/aleph/vm/orchestrator/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/vm_supervisor/migrations/env.py b/src/aleph/vm/orchestrator/migrations/env.py
similarity index 94%
rename from vm_supervisor/migrations/env.py
rename to src/aleph/vm/orchestrator/migrations/env.py
index 71577f27e..c7fbe5004 100644
--- a/vm_supervisor/migrations/env.py
+++ b/src/aleph/vm/orchestrator/migrations/env.py
@@ -1,10 +1,10 @@
 from alembic import context
 from sqlalchemy import create_engine
 
-from vm_supervisor.conf import make_db_url
+from aleph.vm.orchestrator.conf import make_db_url
 
 # Auto-generate migrations
-from vm_supervisor.metrics import Base
+from aleph.vm.orchestrator.metrics import Base
 
 # # this is the Alembic Config object, which provides
 # # access to the values within the .ini file in use.
diff --git a/vm_supervisor/migrations/script.py.mako b/src/aleph/vm/orchestrator/migrations/script.py.mako
similarity index 100%
rename from vm_supervisor/migrations/script.py.mako
rename to src/aleph/vm/orchestrator/migrations/script.py.mako
diff --git a/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py b/src/aleph/vm/orchestrator/migrations/versions/0001_bbb12a12372e_execution_records.py
similarity index 97%
rename from vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py
rename to src/aleph/vm/orchestrator/migrations/versions/0001_bbb12a12372e_execution_records.py
index ddb1e55ff..2ee5a3efd 100644
--- a/vm_supervisor/migrations/versions/0001_bbb12a12372e_execution_records.py
+++ b/src/aleph/vm/orchestrator/migrations/versions/0001_bbb12a12372e_execution_records.py
@@ -12,7 +12,7 @@
 from sqlalchemy import create_engine
 from sqlalchemy.engine import reflection
 
-from vm_supervisor.conf import make_db_url
+from aleph.vm.orchestrator.conf import make_db_url
 
 revision = "bbb12a12372e"
 down_revision = None
diff --git a/src/aleph/vm/orchestrator/migrations/versions/__init__.py b/src/aleph/vm/orchestrator/migrations/versions/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/vm_supervisor/models.py b/src/aleph/vm/orchestrator/models.py
similarity index 88%
rename from vm_supervisor/models.py
rename to src/aleph/vm/orchestrator/models.py
index 626c20560..4a0cf2578 100644
--- a/vm_supervisor/models.py
+++ b/src/aleph/vm/orchestrator/models.py
@@ -5,7 +5,7 @@
 from asyncio import Task
 from dataclasses import dataclass
 from datetime import datetime
-from typing import TYPE_CHECKING, Dict, Optional, Union
+from typing import TYPE_CHECKING, Optional, Union
 
 from aleph_message.models import (
     ExecutableContent,
@@ -14,23 +14,24 @@
     ProgramContent,
 )
 
+from aleph.vm.controllers.firecracker.executable import AlephFirecrackerExecutable
+from aleph.vm.controllers.firecracker.instance import AlephInstanceResources
+from aleph.vm.controllers.firecracker.program import (
+    AlephFirecrackerProgram,
+    AlephFirecrackerResources,
+    AlephProgramResources,
+)
+
 from .conf import settings
 from .metrics import ExecutionRecord, save_execution_data, save_record
 from .network.interfaces import TapInterface
+from .pubsub import PubSub
+from .utils import create_task_log_exceptions, dumps_for_json
+from .vm import AlephFirecrackerInstance
 
 if TYPE_CHECKING:
     from .snapshot_manager import SnapshotManager
 
-from .pubsub import PubSub
-from .utils import create_task_log_exceptions, dumps_for_json
-from .vm import AlephFirecrackerInstance
-from .vm.firecracker.executable import AlephFirecrackerExecutable
-from .vm.firecracker.instance import AlephInstanceResources
-from .vm.firecracker.program import (
-    AlephFirecrackerProgram,
-    AlephFirecrackerResources,
-    AlephProgramResources,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -112,7 +113,7 @@ def __init__(
         self.stop_pending_lock = asyncio.Lock()
         self.snapshot_manager = snapshot_manager
 
-    def to_dict(self) -> Dict:
+    def to_dict(self) -> dict:
         return {
             "is_running": self.is_running,
             **self.__dict__,
@@ -129,16 +130,16 @@ async def prepare(self):
         elif self.is_instance:
             resources = AlephInstanceResources(self.message, namespace=self.vm_hash)
         else:
-            raise ValueError("Unknown executable message type")
+            msg = "Unknown executable message type"
+            raise ValueError(msg)
         await resources.download_all()
         self.times.prepared_at = datetime.now()
         self.resources = resources
 
-    async def create(
-        self, vm_id: int, tap_interface: Optional[TapInterface] = None
-    ) -> AlephFirecrackerExecutable:
+    async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None) -> AlephFirecrackerExecutable:
         if not self.resources:
-            raise ValueError("Execution resources must be configured first")
+            msg = "Execution resources must be configured first"
+            raise ValueError(msg)
         self.times.starting_at = datetime.now()
 
         vm: Union[AlephFirecrackerProgram, AlephFirecrackerInstance]
@@ -187,9 +188,7 @@ def stop_after_timeout(self, timeout: float = 5.0) -> Optional[Task]:
         if sys.version_info.major >= 3 and sys.version_info.minor >= 8:
             # Task can be named
             vm_id: str = str(self.vm.vm_id if self.vm else None)
-            self.expire_task = create_task_log_exceptions(
-                self.expire(timeout), name=f"expire {vm_id}"
-            )
+            self.expire_task = create_task_log_exceptions(self.expire(timeout), name=f"expire {vm_id}")
         else:
             self.expire_task = create_task_log_exceptions(self.expire(timeout))
         return self.expire_task
@@ -237,29 +236,19 @@ async def stop(self):
 
     def start_watching_for_updates(self, pubsub: PubSub):
         if not self.update_task:
-            self.update_task = create_task_log_exceptions(
-                self.watch_for_updates(pubsub=pubsub)
-            )
+            self.update_task = create_task_log_exceptions(self.watch_for_updates(pubsub=pubsub))
 
     async def watch_for_updates(self, pubsub: PubSub):
         if self.is_instance:
             await pubsub.msubscribe(
-                *(
-                    volume.ref
-                    for volume in (self.original.volumes or [])
-                    if hasattr(volume, "ref")
-                ),
+                *(volume.ref for volume in (self.original.volumes or []) if hasattr(volume, "ref")),
             )
         else:
             await pubsub.msubscribe(
                 self.original.code.ref,
                 self.original.runtime.ref,
                 self.original.data.ref if self.original.data else None,
-                *(
-                    volume.ref
-                    for volume in (self.original.volumes or [])
-                    if hasattr(volume, "ref")
-                ),
+                *(volume.ref for volume in (self.original.volumes or []) if hasattr(volume, "ref")),
             )
         logger.debug("Update received, stopping VM...")
         await self.stop()
@@ -275,9 +264,7 @@ async def all_runs_complete(self):
 
     async def record_usage(self):
         if settings.EXECUTION_LOG_ENABLED:
-            await save_execution_data(
-                execution_uuid=self.uuid, execution_data=self.to_json()
-            )
+            await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json())
         pid_info = self.vm.to_dict()
         # Handle cases when the process cannot be accessed
         if pid_info and pid_info.get("process"):
@@ -324,10 +311,12 @@ async def record_usage(self):
 
     async def run_code(self, scope: Optional[dict] = None) -> bytes:
         if not self.vm:
-            raise ValueError("The VM has not been created yet")
+            msg = "The VM has not been created yet"
+            raise ValueError(msg)
 
         if not self.is_program:
-            raise ValueError("Code can ony be run on programs")
+            msg = "Code can ony be run on programs"
+            raise ValueError(msg)
         assert isinstance(self.vm, AlephFirecrackerProgram)
 
         self.concurrent_runs += 1
diff --git a/src/aleph/vm/orchestrator/network/__init__.py b/src/aleph/vm/orchestrator/network/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/vm_supervisor/network/firewall.py b/src/aleph/vm/orchestrator/network/firewall.py
similarity index 91%
rename from vm_supervisor/network/firewall.py
rename to src/aleph/vm/orchestrator/network/firewall.py
index 276b8c6dd..073dc9ec6 100644
--- a/vm_supervisor/network/firewall.py
+++ b/src/aleph/vm/orchestrator/network/firewall.py
@@ -1,17 +1,17 @@
 import json
 import logging
 from functools import lru_cache
-from typing import Dict, List
 
 from nftables import Nftables
 
-from ..conf import settings
+from aleph.vm.orchestrator.conf import settings
+
 from .interfaces import TapInterface
 
 logger = logging.getLogger(__name__)
 
 
-@lru_cache()
+@lru_cache
 def get_customized_nftables() -> Nftables:
     nft = Nftables()
     nft.set_json_output(True)
@@ -22,7 +22,7 @@ def get_customized_nftables() -> Nftables:
     return nft
 
 
-def execute_json_nft_commands(commands: List[Dict]) -> int:
+def execute_json_nft_commands(commands: list[dict]) -> int:
     """Executes a list of nftables commands, and returns the exit status"""
     nft = get_customized_nftables()
     commands_dict = {"nftables": commands}
@@ -40,7 +40,7 @@ def execute_json_nft_commands(commands: List[Dict]) -> int:
     return return_code
 
 
-def get_existing_nftables_ruleset() -> Dict:
+def get_existing_nftables_ruleset() -> dict:
     """Retrieves the full nftables ruleset and returns it"""
     nft = get_customized_nftables()
     return_code, output, error = nft.cmd("list ruleset")
@@ -52,7 +52,7 @@ def get_existing_nftables_ruleset() -> Dict:
     return nft_ruleset
 
 
-def get_base_chains_for_hook(hook: str, family: str = "ip") -> List:
+def get_base_chains_for_hook(hook: str, family: str = "ip") -> list:
     """Looks through the nftables ruleset and creates a list of
     all chains that are base chains for the specified hook"""
     nft_ruleset = get_existing_nftables_ruleset()
@@ -99,8 +99,8 @@ def check_if_table_exists(family: str, table: str) -> bool:
 def initialize_nftables() -> None:
     """Creates basic chains and rules in the nftables ruleset to build on further.
     Additionally, stores some information in the class for later use."""
-    commands: List[Dict] = []
-    base_chains: Dict[str, Dict[str, str]] = {
+    commands: list[dict] = []
+    base_chains: dict[str, dict[str, str]] = {
         "postrouting": {},
         "forward": {},
     }
@@ -125,9 +125,8 @@ def initialize_nftables() -> None:
             commands.append({"add": new_chain})
             chains.append(new_chain)
         elif len(chains) > 1:
-            raise NotImplementedError(
-                f"Multiple base chains for an nftables basechain are not supported: {hook}"
-            )
+            msg = f"Multiple base chains for an nftables basechain are not supported: {hook}"
+            raise NotImplementedError(msg)
         base_chains[hook] = chains.pop()["chain"]
 
     add_chain(
@@ -142,13 +141,7 @@ def initialize_nftables() -> None:
                     "family": "ip",
                     "table": base_chains["postrouting"]["table"],
                     "chain": base_chains["postrouting"]["name"],
-                    "expr": [
-                        {
-                            "jump": {
-                                "target": f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-nat"
-                            }
-                        }
-                    ],
+                    "expr": [{"jump": {"target": f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-nat"}}],
                 }
             }
         }
@@ -166,13 +159,7 @@ def initialize_nftables() -> None:
                     "family": "ip",
                     "table": base_chains["forward"]["table"],
                     "chain": base_chains["forward"]["name"],
-                    "expr": [
-                        {
-                            "jump": {
-                                "target": f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-filter"
-                            }
-                        }
-                    ],
+                    "expr": [{"jump": {"target": f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-filter"}}],
                 }
             }
         }
@@ -200,7 +187,6 @@ def initialize_nftables() -> None:
     )
 
     execute_json_nft_commands(commands)
-    return
 
 
 def teardown_nftables() -> None:
@@ -208,7 +194,6 @@ def teardown_nftables() -> None:
     logger.debug("Tearing down nftables setup")
     remove_chain(f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-nat")
     remove_chain(f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-filter")
-    return
 
 
 def add_chain(family: str, table: str, name: str) -> int:
@@ -255,11 +240,7 @@ def remove_chain(name: str) -> int:
                     }
                 }
             )
-        elif (
-            isinstance(entry, dict)
-            and "chain" in entry
-            and entry["chain"]["name"] == name
-        ):
+        elif isinstance(entry, dict) and "chain" in entry and entry["chain"]["name"] == name:
             remove_chain_commands.append(
                 {
                     "delete": {
diff --git a/vm_supervisor/network/hostnetwork.py b/src/aleph/vm/orchestrator/network/hostnetwork.py
similarity index 83%
rename from vm_supervisor/network/hostnetwork.py
rename to src/aleph/vm/orchestrator/network/hostnetwork.py
index bba6ae362..520ae7e2c 100644
--- a/vm_supervisor/network/hostnetwork.py
+++ b/src/aleph/vm/orchestrator/network/hostnetwork.py
@@ -5,9 +5,9 @@
 
 from aleph_message.models import ItemHash
 
-from vm_supervisor.conf import IPv6AllocationPolicy
+from aleph.vm.orchestrator.conf import IPv6AllocationPolicy
+from aleph.vm.orchestrator.vm.vm_type import VmType
 
-from ..vm.vm_type import VmType
 from .firewall import initialize_nftables, setup_nftables_for_vm, teardown_nftables
 from .interfaces import TapInterface
 from .ipaddresses import IPv4NetworkWithInterfaces
@@ -31,9 +31,7 @@ def get_ipv6_forwarding_state() -> int:
 
 
 class IPv6Allocator(Protocol):
-    def allocate_vm_ipv6_subnet(
-        self, vm_id: int, vm_hash: ItemHash, vm_type: VmType
-    ) -> IPv6Network:
+    def allocate_vm_ipv6_subnet(self, vm_id: int, vm_hash: ItemHash, vm_type: VmType) -> IPv6Network:
         ...
 
 
@@ -56,18 +54,16 @@ class StaticIPv6Allocator(IPv6Allocator):
 
     def __init__(self, ipv6_range: IPv6Network, subnet_prefix: int):
         if ipv6_range.prefixlen != 64:
-            raise ValueError(
-                "The static IP address allocation scheme requires a /64 subnet"
-            )
+            msg = "The static IP address allocation scheme requires a /64 subnet"
+            raise ValueError(msg)
         if subnet_prefix < 124:
-            raise ValueError("The IPv6 subnet prefix cannot be larger than /124.")
+            msg = "The IPv6 subnet prefix cannot be larger than /124."
+            raise ValueError(msg)
 
         self.ipv6_range = ipv6_range
         self.subnet_prefix = subnet_prefix
 
-    def allocate_vm_ipv6_subnet(
-        self, vm_id: int, vm_hash: ItemHash, vm_type: VmType
-    ) -> IPv6Network:
+    def allocate_vm_ipv6_subnet(self, vm_id: int, vm_hash: ItemHash, vm_type: VmType) -> IPv6Network:
         ipv6_elems = self.ipv6_range.exploded.split(":")[:4]
         ipv6_elems += [self.VM_TYPE_PREFIX[vm_type]]
 
@@ -95,9 +91,7 @@ def __init__(self, ipv6_range: IPv6Network, subnet_prefix: int):
         # Assume the first subnet is reserved for the host
         _ = next(self.subnets_generator)
 
-    def allocate_vm_ipv6_subnet(
-        self, vm_id: int, vm_hash: ItemHash, vm_type: VmType
-    ) -> IPv6Network:
+    def allocate_vm_ipv6_subnet(self, vm_id: int, vm_hash: ItemHash, vm_type: VmType) -> IPv6Network:
         return next(self.subnets_generator)
 
 
@@ -105,21 +99,15 @@ def make_ipv6_allocator(
     allocation_policy: IPv6AllocationPolicy, address_pool: str, subnet_prefix: int
 ) -> IPv6Allocator:
     if allocation_policy == IPv6AllocationPolicy.static:
-        return StaticIPv6Allocator(
-            ipv6_range=IPv6Network(address_pool), subnet_prefix=subnet_prefix
-        )
+        return StaticIPv6Allocator(ipv6_range=IPv6Network(address_pool), subnet_prefix=subnet_prefix)
 
-    return DynamicIPv6Allocator(
-        ipv6_range=IPv6Network(address_pool), subnet_prefix=subnet_prefix
-    )
+    return DynamicIPv6Allocator(ipv6_range=IPv6Network(address_pool), subnet_prefix=subnet_prefix)
 
 
 class Network:
     ipv4_forward_state_before_setup: Optional[int] = None
     ipv6_forward_state_before_setup: Optional[int] = None
-    ipv4_address_pool: IPv4NetworkWithInterfaces = IPv4NetworkWithInterfaces(
-        "172.16.0.0/12"
-    )
+    ipv4_address_pool: IPv4NetworkWithInterfaces = IPv4NetworkWithInterfaces("172.16.0.0/12")
     ipv6_address_pool: IPv6Network
     network_size: int
     external_interface: str
@@ -139,9 +127,7 @@ def __init__(
         """Sets up the Network class with some information it needs so future function calls work as expected"""
         self.ipv4_address_pool = IPv4NetworkWithInterfaces(vm_ipv4_address_pool_range)
         if not self.ipv4_address_pool.is_private:
-            logger.warning(
-                f"Using a network range that is not private: {self.ipv4_address_pool}"
-            )
+            logger.warning(f"Using a network range that is not private: {self.ipv4_address_pool}")
         self.ipv6_allocator = ipv6_allocator
 
         self.network_size = vm_network_size
@@ -174,9 +160,7 @@ def reset_ipv4_forwarding_state(self) -> None:
             return
 
         if self.ipv4_forward_state_before_setup != get_ipv4_forwarding_state():
-            Path("/proc/sys/net/ipv4/ip_forward").write_text(
-                str(self.ipv4_forward_state_before_setup)
-            )
+            Path("/proc/sys/net/ipv4/ip_forward").write_text(str(self.ipv4_forward_state_before_setup))
 
     def enable_ipv6_forwarding(self) -> None:
         """Saves the host IPv6 forwarding state, and if it was disabled, enables it"""
@@ -192,18 +176,14 @@ def reset_ipv6_forwarding_state(self) -> None:
             return
 
         if self.ipv6_forward_state_before_setup != get_ipv6_forwarding_state():
-            Path("/proc/sys/net/ipv6/conf/all/forwarding").write_text(
-                str(self.ipv6_forward_state_before_setup)
-            )
+            Path("/proc/sys/net/ipv6/conf/all/forwarding").write_text(str(self.ipv6_forward_state_before_setup))
 
     def teardown(self) -> None:
         teardown_nftables()
         self.reset_ipv4_forwarding_state()
         self.reset_ipv6_forwarding_state()
 
-    async def create_tap(
-        self, vm_id: int, vm_hash: ItemHash, vm_type: VmType
-    ) -> TapInterface:
+    async def create_tap(self, vm_id: int, vm_hash: ItemHash, vm_type: VmType) -> TapInterface:
         """Create TAP interface to be used by VM"""
         interface = TapInterface(
             f"vmtap{vm_id}",
diff --git a/vm_supervisor/network/interfaces.py b/src/aleph/vm/orchestrator/network/interfaces.py
similarity index 100%
rename from vm_supervisor/network/interfaces.py
rename to src/aleph/vm/orchestrator/network/interfaces.py
diff --git a/vm_supervisor/network/ipaddresses.py b/src/aleph/vm/orchestrator/network/ipaddresses.py
similarity index 78%
rename from vm_supervisor/network/ipaddresses.py
rename to src/aleph/vm/orchestrator/network/ipaddresses.py
index 8a7c038b0..aef2a431f 100644
--- a/vm_supervisor/network/ipaddresses.py
+++ b/src/aleph/vm/orchestrator/network/ipaddresses.py
@@ -1,5 +1,5 @@
+from collections.abc import Iterator
 from ipaddress import IPv4Interface, IPv4Network
-from typing import Iterator
 
 
 class IPv4NetworkWithInterfaces(IPv4Network):
@@ -14,10 +14,12 @@ def __getitem__(self, n) -> IPv4Interface:
         broadcast = int(self.broadcast_address)
         if n >= 0:
             if network + n > broadcast:
-                raise IndexError("address out of range")
+                msg = "address out of range"
+                raise IndexError(msg)
             return IPv4Interface((network + n, self.prefixlen))
         else:
             n += 1
             if broadcast + n < network:
-                raise IndexError("address out of range")
+                msg = "address out of range"
+                raise IndexError(msg)
             return IPv4Interface((broadcast + n, self.prefixlen))
diff --git a/vm_supervisor/network/ndp_proxy.py b/src/aleph/vm/orchestrator/network/ndp_proxy.py
similarity index 93%
rename from vm_supervisor/network/ndp_proxy.py
rename to src/aleph/vm/orchestrator/network/ndp_proxy.py
index 577114059..d9f75ab2a 100644
--- a/vm_supervisor/network/ndp_proxy.py
+++ b/src/aleph/vm/orchestrator/network/ndp_proxy.py
@@ -13,9 +13,8 @@
 from dataclasses import dataclass
 from ipaddress import IPv6Network
 from pathlib import Path
-from typing import Dict
 
-from vm_supervisor.utils import run_in_subprocess
+from aleph.vm.orchestrator.utils import run_in_subprocess
 
 logger = logging.getLogger(__name__)
 
@@ -28,7 +27,7 @@ class NdpRule:
 class NdpProxy:
     def __init__(self, host_network_interface: str):
         self.host_network_interface = host_network_interface
-        self.interface_address_range_mapping: Dict[str, IPv6Network] = {}
+        self.interface_address_range_mapping: dict[str, IPv6Network] = {}
 
     @staticmethod
     async def _restart_ndppd():
diff --git a/vm_supervisor/pool.py b/src/aleph/vm/orchestrator/pool.py
similarity index 87%
rename from vm_supervisor/pool.py
rename to src/aleph/vm/orchestrator/pool.py
index fe232d105..a3cccfeb1 100644
--- a/vm_supervisor/pool.py
+++ b/src/aleph/vm/orchestrator/pool.py
@@ -1,14 +1,14 @@
 import asyncio
 import logging
-from typing import Dict, Iterable, Optional
+from collections.abc import Iterable
+from typing import Optional
 
 from aleph_message.models import ExecutableMessage, ItemHash
 from aleph_message.models.execution.instance import InstanceContent
 
-from vm_supervisor.network.hostnetwork import Network, make_ipv6_allocator
-
 from .conf import settings
 from .models import ExecutableContent, VmExecution
+from .network.hostnetwork import Network, make_ipv6_allocator
 from .snapshot_manager import SnapshotManager
 from .vm.vm_type import VmType
 
@@ -25,8 +25,8 @@ class VmPool:
     """
 
     counter: int  # Used to provide distinct ids to network interfaces
-    executions: Dict[ItemHash, VmExecution]
-    message_cache: Dict[str, ExecutableMessage] = {}
+    executions: dict[ItemHash, VmExecution]
+    message_cache: dict[str, ExecutableMessage] = {}
     network: Optional[Network]
     snapshot_manager: SnapshotManager
 
@@ -101,16 +101,13 @@ def get_unique_vm_id(self) -> int:
             #
             # We therefore recycle vm_id values from executions that are not running
             # anymore.
-            currently_used_vm_ids = set(
-                execution.vm_id
-                for execution in self.executions.values()
-                if execution.is_running
-            )
+            currently_used_vm_ids = {execution.vm_id for execution in self.executions.values() if execution.is_running}
             for i in range(settings.START_ID_INDEX, 255**2):
                 if i not in currently_used_vm_ids:
                     return i
             else:
-                raise ValueError("No available value for vm_id.")
+                msg = "No available value for vm_id."
+                raise ValueError(msg)
 
     async def get_running_vm(self, vm_hash: ItemHash) -> Optional[VmExecution]:
         """Return a running VM or None. Disables the VM expiration task."""
@@ -137,16 +134,14 @@ async def stop(self):
         """Stop all VMs in the pool."""
 
         # Stop executions in parallel:
-        await asyncio.gather(
-            *(execution.stop() for vm_hash, execution in self.executions.items())
-        )
+        await asyncio.gather(*(execution.stop() for vm_hash, execution in self.executions.items()))
 
     def get_persistent_executions(self) -> Iterable[VmExecution]:
-        for vm_hash, execution in self.executions.items():
+        for _vm_hash, execution in self.executions.items():
             if execution.persistent and execution.is_running:
                 yield execution
 
     def get_instance_executions(self) -> Iterable[VmExecution]:
-        for vm_hash, execution in self.executions.items():
+        for _vm_hash, execution in self.executions.items():
             if execution.is_instance and execution.is_running:
                 yield execution
diff --git a/vm_supervisor/pubsub.py b/src/aleph/vm/orchestrator/pubsub.py
similarity index 86%
rename from vm_supervisor/pubsub.py
rename to src/aleph/vm/orchestrator/pubsub.py
index ae9a40318..6dbe380d0 100644
--- a/vm_supervisor/pubsub.py
+++ b/src/aleph/vm/orchestrator/pubsub.py
@@ -6,17 +6,17 @@
 import asyncio
 import logging
 import sys
-from typing import Dict, Hashable, Set
+from collections.abc import Hashable
 
 logger = logging.getLogger(__name__)
 
 
 class PubSub:
     if sys.version_info >= (3, 9):
-        subscribers: Dict[Hashable, Set[asyncio.Queue[Set]]]
+        subscribers: dict[Hashable, set[asyncio.Queue[set]]]
     else:
         # Support for Python 3.8 (Ubuntu 20.04)
-        subscribers: Dict[Hashable, Set[asyncio.Queue]]
+        subscribers: dict[Hashable, set[asyncio.Queue]]
 
     def __init__(self):
         self.subscribers = {}
@@ -32,8 +32,6 @@ async def subscribe(self, key):
         if not self.subscribers.get(key):
             self.subscribers.pop(key)
 
-        return
-
     async def msubscribe(self, *keys):
         """Subscribe to multiple keys"""
         keys = tuple(key for key in keys if key is not None)
@@ -55,8 +53,7 @@ async def msubscribe(self, *keys):
                 # Remove keys with no remaining queue (empty set remaining)
                 if self.subscribers.get(key) == set():
                     self.subscribers.pop(key)
-        return
 
     async def publish(self, key, value):
-        for queue in self.subscribers.get(key, tuple()):
+        for queue in self.subscribers.get(key, ()):
             await queue.put(value)
diff --git a/vm_supervisor/reactor.py b/src/aleph/vm/orchestrator/reactor.py
similarity index 88%
rename from vm_supervisor/reactor.py
rename to src/aleph/vm/orchestrator/reactor.py
index 4c0bbe284..e5c96a583 100644
--- a/vm_supervisor/reactor.py
+++ b/src/aleph/vm/orchestrator/reactor.py
@@ -1,5 +1,5 @@
 import logging
-from typing import Coroutine, List
+from collections.abc import Coroutine
 
 from aleph_message.models import AlephMessage
 from aleph_message.models.execution.environment import Subscription
@@ -22,7 +22,8 @@ def is_equal_or_includes(value, compare_to) -> bool:
                 return False
         return True
     else:
-        raise ValueError("Unsupported value")
+        msg = "Unsupported value"
+        raise ValueError(msg)
 
 
 def subscription_matches(subscription: Subscription, message: AlephMessage) -> bool:
@@ -37,20 +38,19 @@ def subscription_matches(subscription: Subscription, message: AlephMessage) -> b
 
 class Reactor:
     pubsub: PubSub
-    listeners: List[AlephMessage]
+    listeners: list[AlephMessage]
 
     def __init__(self, pubsub: PubSub):
         self.pubsub = pubsub
         self.listeners = []
 
     async def trigger(self, message: AlephMessage):
-        coroutines: List[Coroutine] = []
+        coroutines: list[Coroutine] = []
 
         for listener in self.listeners:
             if not listener.content.on.message:
                 logger.warning(
-                    "Program with no subscription was registered in reactor listeners: "
-                    f"{listener.item_hash}"
+                    "Program with no subscription was registered in reactor listeners: " f"{listener.item_hash}"
                 )
                 continue
 
@@ -71,6 +71,5 @@ def register(self, message: AlephMessage):
             self.listeners.append(message)
         else:
             logger.debug(
-                "Program with no subscription cannot be registered in reactor listeners: "
-                f"{message.item_hash}"
+                "Program with no subscription cannot be registered in reactor listeners: " f"{message.item_hash}"
             )
diff --git a/vm_supervisor/resources.py b/src/aleph/vm/orchestrator/resources.py
similarity index 88%
rename from vm_supervisor/resources.py
rename to src/aleph/vm/orchestrator/resources.py
index fb2740891..e1640dc3c 100644
--- a/vm_supervisor/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -1,7 +1,7 @@
 import math
 from datetime import datetime, timezone
 from functools import lru_cache
-from typing import Optional, Set, Tuple
+from typing import Optional
 
 import cpuinfo
 import psutil
@@ -22,7 +22,7 @@ class LoadAverage(BaseModel):
     load15: float
 
     @classmethod
-    def from_psutil(cls, psutil_loadavg: Tuple[float, float, float]):
+    def from_psutil(cls, psutil_loadavg: tuple[float, float, float]):
         return cls(
             load1=psutil_loadavg[0],
             load5=psutil_loadavg[1],
@@ -105,10 +105,8 @@ async def about_system_usage(request: web.Request):
             available_kB=math.floor(psutil.virtual_memory().available / 1000),
         ),
         disk=DiskUsage(
-            total_kB=psutil.disk_usage(str(settings.PERSISTENT_VOLUMES_DIR)).total
-            // 1000,
-            available_kB=psutil.disk_usage(str(settings.PERSISTENT_VOLUMES_DIR)).free
-            // 1000,
+            total_kB=psutil.disk_usage(str(settings.PERSISTENT_VOLUMES_DIR)).total // 1000,
+            available_kB=psutil.disk_usage(str(settings.PERSISTENT_VOLUMES_DIR)).free // 1000,
         ),
         period=UsagePeriod(
             start_timestamp=period_start,
@@ -122,7 +120,7 @@ async def about_system_usage(request: web.Request):
 
 
 class Allocation(BaseModel):
-    persistent_vms: Set[str] = Field(default_factory=set)
-    instances: Set[str] = Field(default_factory=set)
-    on_demand_vms: Optional[Set[str]] = None
-    jobs: Optional[Set[str]] = None
+    persistent_vms: set[str] = Field(default_factory=set)
+    instances: set[str] = Field(default_factory=set)
+    on_demand_vms: Optional[set[str]] = None
+    jobs: Optional[set[str]] = None
diff --git a/vm_supervisor/run.py b/src/aleph/vm/orchestrator/run.py
similarity index 89%
rename from vm_supervisor/run.py
rename to src/aleph/vm/orchestrator/run.py
index 4d51fc238..110d81300 100644
--- a/vm_supervisor/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -1,6 +1,6 @@
 import asyncio
 import logging
-from typing import Any, Dict, Optional
+from typing import Any, Optional
 
 import msgpack
 from aiohttp import web
@@ -9,7 +9,12 @@
 from msgpack import UnpackValueError
 from multidict import CIMultiDict
 
-from firecracker.microvm import MicroVMFailedInit
+from aleph.vm.controllers.firecracker.program import (
+    FileTooLargeError,
+    ResourceDownloadError,
+    VmSetupError,
+)
+from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInit
 
 from .conf import settings
 from .messages import load_updated_message
@@ -17,18 +22,13 @@
 from .pool import VmPool
 from .pubsub import PubSub
 from .utils import HostNotFoundError
-from .vm.firecracker.program import (
-    FileTooLargeError,
-    ResourceDownloadError,
-    VmSetupError,
-)
 
 logger = logging.getLogger(__name__)
 
 pool = VmPool()
 
 
-async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]:
+async def build_asgi_scope(path: str, request: web.Request) -> dict[str, Any]:
     # ASGI mandates lowercase header names
     headers = tuple((name.lower(), value) for name, value in request.raw_headers)
     return {
@@ -41,7 +41,7 @@ async def build_asgi_scope(path: str, request: web.Request) -> Dict[str, Any]:
     }
 
 
-async def build_event_scope(event) -> Dict[str, Any]:
+async def build_event_scope(event) -> dict[str, Any]:
     return {
         "type": "aleph.message",
         "body": event,
@@ -52,9 +52,7 @@ async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
     message, original_message = await load_updated_message(vm_hash)
     pool.message_cache[vm_hash] = message
 
-    logger.debug(
-        f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}"
-    )
+    logger.debug(f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}")
 
     try:
         execution = await pool.create_a_vm(
@@ -82,7 +80,8 @@ async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
         raise HTTPInternalServerError(reason="Host did not respond to ping")
 
     if not execution.vm:
-        raise ValueError("The VM has not been created")
+        msg = "The VM has not been created"
+        raise ValueError(msg)
 
     return execution
 
@@ -110,9 +109,7 @@ async def create_vm_execution_or_raise_http_error(vm_hash: ItemHash) -> VmExecut
         raise HTTPInternalServerError(reason="Host did not respond to ping")
 
 
-async def run_code_on_request(
-    vm_hash: ItemHash, path: str, request: web.Request
-) -> web.Response:
+async def run_code_on_request(vm_hash: ItemHash, path: str, request: web.Request) -> web.Response:
     """
     Execute the code corresponding to the 'code id' in the path.
     """
@@ -124,7 +121,7 @@ async def run_code_on_request(
 
     logger.debug(f"Using vm={execution.vm_id}")
 
-    scope: Dict = await build_asgi_scope(path, request)
+    scope: dict = await build_asgi_scope(path, request)
 
     try:
         await execution.becomes_ready()
@@ -146,9 +143,7 @@ async def run_code_on_request(
 
     except asyncio.TimeoutError:
         logger.warning(f"VM{execution.vm_id} did not respond within `resource.seconds`")
-        return web.HTTPGatewayTimeout(
-            body="Program did not respond within `resource.seconds`"
-        )
+        return web.HTTPGatewayTimeout(body="Program did not respond within `resource.seconds`")
     except UnpackValueError as error:
         logger.exception(error)
         return web.Response(status=502, reason="Invalid response from VM")
@@ -166,10 +161,7 @@ async def run_code_on_request(
             # This fills the logs with noisy stack traces, so we ignore this specific error.
             ignored_error = 'raise CustomError("Whoops")'
 
-            if (
-                settings.IGNORE_TRACEBACK_FROM_DIAGNOSTICS
-                and ignored_error in result["traceback"]
-            ):
+            if settings.IGNORE_TRACEBACK_FROM_DIAGNOSTICS and ignored_error in result["traceback"]:
                 logger.debug('Ignored traceback from CustomError("Whoops")')
             else:
                 logger.warning(result["traceback"])
@@ -182,12 +174,7 @@ async def run_code_on_request(
             )
 
         # HTTP Headers require specific data structure
-        headers = CIMultiDict(
-            [
-                (key.decode().lower(), value.decode())
-                for key, value in result["headers"]["headers"]
-            ]
-        )
+        headers = CIMultiDict([(key.decode().lower(), value.decode()) for key, value in result["headers"]["headers"]])
         if "content-length" not in headers:
             headers["Content-Length".lower()] = str(len(result["body"]["body"]))
         for header in ["Content-Encoding", "Transfer-Encoding", "Vary"]:
@@ -231,7 +218,7 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub):
 
     logger.debug(f"Using vm={execution.vm_id}")
 
-    scope: Dict = await build_event_scope(event)
+    scope: dict = await build_event_scope(event)
 
     try:
         await execution.becomes_ready()
diff --git a/vm_supervisor/snapshot_manager.py b/src/aleph/vm/orchestrator/snapshot_manager.py
similarity index 77%
rename from vm_supervisor/snapshot_manager.py
rename to src/aleph/vm/orchestrator/snapshot_manager.py
index 75a865552..6ad3a89a0 100644
--- a/vm_supervisor/snapshot_manager.py
+++ b/src/aleph/vm/orchestrator/snapshot_manager.py
@@ -2,7 +2,7 @@
 import logging
 import threading
 from time import sleep
-from typing import Dict, Optional
+from typing import Optional
 
 from aleph_message.models import ItemHash
 from schedule import Job, Scheduler
@@ -31,12 +31,11 @@ async def do_execution_snapshot(execution: VmExecution) -> CompressedDiskVolumeS
         snapshot = await execution.vm.create_snapshot()
         await snapshot.upload()
 
-        logger.debug(
-            f"New snapshots for VM {execution.vm_hash} created in {snapshot.path}"
-        )
+        logger.debug(f"New snapshots for VM {execution.vm_hash} created in {snapshot.path}")
         return snapshot
     except ValueError:
-        raise ValueError("Something failed taking an snapshot")
+        msg = "Something failed taking an snapshot"
+        raise ValueError(msg)
 
 
 def infinite_run_scheduler_jobs(scheduler: Scheduler) -> None:
@@ -65,12 +64,8 @@ def __init__(
         self._scheduler = scheduler
 
     async def start(self) -> None:
-        logger.debug(
-            f"Starting snapshots for VM {self.vm_hash} every {self.frequency} minutes"
-        )
-        job = self._scheduler.every(self.frequency).minutes.do(
-            run_threaded_snapshot, self.execution
-        )
+        logger.debug(f"Starting snapshots for VM {self.vm_hash} every {self.frequency} minutes")
+        job = self._scheduler.every(self.frequency).minutes.do(run_threaded_snapshot, self.execution)
         self._job = job
 
     async def stop(self) -> None:
@@ -83,7 +78,7 @@ class SnapshotManager:
     Manage VM snapshots.
     """
 
-    executions: Dict[ItemHash, SnapshotExecution]
+    executions: dict[ItemHash, SnapshotExecution]
     _scheduler: Scheduler
 
     def __init__(self):
@@ -99,11 +94,10 @@ def run_snapshots(self) -> None:
         )
         job_thread.start()
 
-    async def start_for(
-        self, execution: VmExecution, frequency: Optional[int] = None
-    ) -> None:
+    async def start_for(self, execution: VmExecution, frequency: Optional[int] = None) -> None:
         if not execution.is_instance:
-            raise NotImplementedError("Snapshots are not implemented for programs.")
+            msg = "Snapshots are not implemented for programs."
+            raise NotImplementedError(msg)
 
         if not frequency:
             frequency = settings.SNAPSHOT_FREQUENCY
@@ -128,6 +122,4 @@ async def stop_for(self, vm_hash: ItemHash) -> None:
         await snapshot_execution.stop()
 
     async def stop_all(self) -> None:
-        await asyncio.gather(
-            *(self.stop_for(vm_hash) for vm_hash, execution in self.executions)
-        )
+        await asyncio.gather(*(self.stop_for(vm_hash) for vm_hash, execution in self.executions))
diff --git a/vm_supervisor/snapshots.py b/src/aleph/vm/orchestrator/snapshots.py
similarity index 85%
rename from vm_supervisor/snapshots.py
rename to src/aleph/vm/orchestrator/snapshots.py
index 43d7f3b1a..699a151e1 100644
--- a/vm_supervisor/snapshots.py
+++ b/src/aleph/vm/orchestrator/snapshots.py
@@ -43,13 +43,9 @@ def delete(self) -> None:
 
         self.path.unlink(missing_ok=True)
 
-    async def compress(
-        self, algorithm: SnapshotCompressionAlgorithm
-    ) -> CompressedDiskVolumeSnapshot:
+    async def compress(self, algorithm: SnapshotCompressionAlgorithm) -> CompressedDiskVolumeSnapshot:
         compressed_snapshot = await compress_volume_snapshot(self.path, algorithm)
-        compressed = CompressedDiskVolumeSnapshot(
-            path=compressed_snapshot, algorithm=algorithm
-        )
+        compressed = CompressedDiskVolumeSnapshot(path=compressed_snapshot, algorithm=algorithm)
         self.compressed = compressed
         return compressed
 
diff --git a/vm_supervisor/status.py b/src/aleph/vm/orchestrator/status.py
similarity index 83%
rename from vm_supervisor/status.py
rename to src/aleph/vm/orchestrator/status.py
index 439cde7fb..bfdd95472 100644
--- a/vm_supervisor/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -3,7 +3,7 @@
 in a deployed supervisor.
 """
 import logging
-from typing import Any, Dict, List
+from typing import Any
 
 from aiohttp import ClientResponseError, ClientSession
 
@@ -23,7 +23,7 @@ async def get_json_from_vm(session: ClientSession, suffix: str) -> Any:
 
 async def check_index(session: ClientSession) -> bool:
     try:
-        result: Dict = await get_json_from_vm(session, "/")
+        result: dict = await get_json_from_vm(session, "/")
         assert result["Example"] == "example_fastapi"
         return True
     except ClientResponseError:
@@ -32,7 +32,7 @@ async def check_index(session: ClientSession) -> bool:
 
 async def check_lifespan(session: ClientSession) -> bool:
     try:
-        result: Dict = await get_json_from_vm(session, "/lifespan")
+        result: dict = await get_json_from_vm(session, "/lifespan")
         return result["Lifespan"] is True
     except ClientResponseError:
         return False
@@ -40,7 +40,7 @@ async def check_lifespan(session: ClientSession) -> bool:
 
 async def check_environ(session: ClientSession) -> bool:
     try:
-        result: Dict = await get_json_from_vm(session, "/environ")
+        result: dict = await get_json_from_vm(session, "/environ")
         assert "ALEPH_API_HOST" in result
         assert "ALEPH_API_UNIX_SOCKET" in result
         assert "ALEPH_REMOTE_CRYPTO_HOST" in result
@@ -53,7 +53,7 @@ async def check_environ(session: ClientSession) -> bool:
 
 async def check_messages(session: ClientSession) -> bool:
     try:
-        result: Dict = await get_json_from_vm(session, "/messages")
+        result: dict = await get_json_from_vm(session, "/messages")
         assert "Messages" in result
         assert "messages" in result["Messages"]
         assert "item_hash" in result["Messages"]["messages"][0]
@@ -64,7 +64,7 @@ async def check_messages(session: ClientSession) -> bool:
 
 async def check_dns(session: ClientSession) -> bool:
     try:
-        result: Dict = await get_json_from_vm(session, "/dns")
+        result: dict = await get_json_from_vm(session, "/dns")
         assert result["ipv4"]
         assert result["ipv6"]
         return True
@@ -74,7 +74,7 @@ async def check_dns(session: ClientSession) -> bool:
 
 async def check_ipv4(session: ClientSession) -> bool:
     try:
-        result: Dict = await get_json_from_vm(session, "/ip/4")
+        result: dict = await get_json_from_vm(session, "/ip/4")
         assert result["result"] is True
         return True
     except ClientResponseError:
@@ -83,7 +83,7 @@ async def check_ipv4(session: ClientSession) -> bool:
 
 async def check_ipv6(session: ClientSession) -> bool:
     try:
-        result: Dict = await get_json_from_vm(session, "/ip/6")
+        result: dict = await get_json_from_vm(session, "/ip/6")
         assert result["result"] is True
         assert "headers" in result
         return True
@@ -93,7 +93,7 @@ async def check_ipv6(session: ClientSession) -> bool:
 
 async def check_internet(session: ClientSession) -> bool:
     try:
-        result: Dict = await get_json_from_vm(session, "/internet")
+        result: dict = await get_json_from_vm(session, "/internet")
         assert result["result"] == 200
         assert "Server" in result["headers"]
         return True
@@ -107,7 +107,7 @@ async def check_cache(session: ClientSession) -> bool:
         assert result1 is True
         result2: int = await get_json_from_vm(session, "/cache/get/a")
         assert result2 == "42"
-        keys: List[str] = await get_json_from_vm(session, "/cache/keys")
+        keys: list[str] = await get_json_from_vm(session, "/cache/keys")
         print("KEYS", keys)
         assert "a" in keys
         return True
@@ -117,9 +117,9 @@ async def check_cache(session: ClientSession) -> bool:
 
 async def check_persistent_storage(session: ClientSession) -> bool:
     try:
-        result: Dict = await get_json_from_vm(session, "/state/increment")
+        result: dict = await get_json_from_vm(session, "/state/increment")
         counter = result["counter"]
-        result_2: Dict = await get_json_from_vm(session, "/state/increment")
+        result_2: dict = await get_json_from_vm(session, "/state/increment")
         counter_2 = result_2["counter"]
         # Use >= to handle potential concurrency
         assert counter_2 >= counter + 1
@@ -145,7 +145,7 @@ async def check_crash_and_restart(session: ClientSession) -> bool:
 
     # Try loading the index page. A new execution should be created.
     try:
-        result: Dict = await get_json_from_vm(session, "/")
+        result: dict = await get_json_from_vm(session, "/")
         assert result["Example"] == "example_fastapi"
         return True
 
diff --git a/vm_supervisor/storage.py b/src/aleph/vm/orchestrator/storage.py
similarity index 89%
rename from vm_supervisor/storage.py
rename to src/aleph/vm/orchestrator/storage.py
index edc7ab5a7..eeb6d3666 100644
--- a/vm_supervisor/storage.py
+++ b/src/aleph/vm/orchestrator/storage.py
@@ -44,7 +44,8 @@ class NotEnoughDiskSpace(OSError):
 async def chown_to_jailman(path: Path) -> None:
     """Changes ownership of the target when running firecracker inside jailer isolation."""
     if not path.exists():
-        raise FileNotFoundError("No such file to change ownership from", path)
+        msg = "No such file to change ownership from"
+        raise FileNotFoundError(msg, path)
     if settings.USE_JAILER:
         await run_in_subprocess(["chown", "jailman:jailman", str(path)])
 
@@ -70,7 +71,7 @@ async def download_file(url: str, local_path: Path) -> None:
                     cache_file.write(chunk)
                     counter += 1
                     if not (counter % 20):
-                        sys.stdout.write(".")
+                        sys.stdout.write("")
                         sys.stdout.flush()
 
             sys.stdout.write("\n")
@@ -108,7 +109,7 @@ async def get_message(ref: str) -> Union[ProgramMessage, InstanceMessage]:
         url = f"{settings.CONNECTOR_URL}/download/message/{ref}"
         await download_file(url, cache_path)
 
-    with open(cache_path, "r") as cache_file:
+    with open(cache_path) as cache_file:
         msg = json.load(cache_file)
 
         if cache_path in (settings.FAKE_DATA_MESSAGE, settings.FAKE_INSTANCE_MESSAGE):
@@ -116,9 +117,7 @@ async def get_message(ref: str) -> Union[ProgramMessage, InstanceMessage]:
             msg = fix_message_validation(msg)
 
         result = parse_message(message_dict=msg)
-        assert isinstance(result, ProgramMessage) or isinstance(
-            result, InstanceMessage
-        ), "Parsed message is not executable"
+        assert isinstance(result, (InstanceMessage, ProgramMessage)), "Parsed message is not executable"
         return result
 
 
@@ -126,15 +125,11 @@ async def get_code_path(ref: str) -> Path:
     if settings.FAKE_DATA_PROGRAM:
         archive_path = Path(settings.FAKE_DATA_PROGRAM)
 
-        encoding: Encoding = (
-            await get_message(ref="fake-message")
-        ).content.code.encoding
+        encoding: Encoding = (await get_message(ref="fake-message")).content.code.encoding
         if encoding == Encoding.squashfs:
             squashfs_path = Path(archive_path.name + ".squashfs")
             squashfs_path.unlink(missing_ok=True)
-            await run_in_subprocess(
-                ["mksquashfs", str(archive_path), str(squashfs_path)]
-            )
+            await run_in_subprocess(["mksquashfs", str(archive_path), str(squashfs_path)])
             logger.debug(f"Squashfs generated on {squashfs_path}")
             return squashfs_path
         elif encoding == Encoding.zip:
@@ -143,7 +138,8 @@ async def get_code_path(ref: str) -> Path:
             logger.debug(f"Zip generated on {zip_path}")
             return zip_path
         else:
-            raise ValueError(f"Unsupported encoding: {encoding}")
+            msg = f"Unsupported encoding: {encoding}"
+            raise ValueError(msg)
 
     cache_path = Path(settings.CODE_CACHE) / ref
     url = f"{settings.CONNECTOR_URL}/download/code/{ref}"
@@ -200,9 +196,7 @@ async def create_ext4(path: Path, size_mib: int) -> bool:
     return True
 
 
-async def create_volume_file(
-    volume: Union[PersistentVolume, RootfsVolume], namespace: str
-) -> Path:
+async def create_volume_file(volume: Union[PersistentVolume, RootfsVolume], namespace: str) -> Path:
     volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
     # Assume that the main filesystem format is BTRFS
     path = settings.PERSISTENT_VOLUMES_DIR / namespace / f"{volume_name}.btrfs"
@@ -246,9 +240,7 @@ async def resize_and_tune_file_system(device_path: Path, mount_path: Path) -> No
     await run_in_subprocess(["umount", str(mount_path)])
 
 
-async def create_devmapper(
-    volume: Union[PersistentVolume, RootfsVolume], namespace: str
-) -> Path:
+async def create_devmapper(volume: Union[PersistentVolume, RootfsVolume], namespace: str) -> Path:
     """It creates a /dev/mapper/DEVICE inside the VM, that is an extended mapped device of the volume specified.
     We follow the steps described here: https://community.aleph.im/t/deploying-mutable-vm-instances-on-aleph/56/2
     """
@@ -277,9 +269,7 @@ async def create_devmapper(
     extended_block_size: int = await get_block_size(volume_path)
 
     mapped_volume_name_base = f"{namespace}_base"
-    path_mapped_volume_name_base = (
-        Path(DEVICE_MAPPER_DIRECTORY) / mapped_volume_name_base
-    )
+    path_mapped_volume_name_base = Path(DEVICE_MAPPER_DIRECTORY) / mapped_volume_name_base
     if not path_mapped_volume_name_base.is_block_device():
         # Creates the base rootfs block device with the entire rootfs size using the image block device as source
         base_table_command = (
@@ -291,7 +281,9 @@ async def create_devmapper(
     extended_loop_device = await create_loopback_device(volume_path)
 
     # Creates the final rootfs block device that is a snapshot of the base block device
-    snapshot_table_command = f"0 {extended_block_size} snapshot {path_mapped_volume_name_base} {extended_loop_device} P 8"
+    snapshot_table_command = (
+        f"0 {extended_block_size} snapshot {path_mapped_volume_name_base} {extended_loop_device} P 8"
+    )
     await create_mapped_device(mapped_volume_name, snapshot_table_command)
 
     mount_path = Path(f"/mnt/{mapped_volume_name}")
@@ -318,25 +310,24 @@ async def get_volume_path(volume: MachineVolume, namespace: str) -> Path:
     if isinstance(volume, ImmutableVolume):
         ref = volume.ref
         return await get_existing_file(ref)
-    elif isinstance(volume, PersistentVolume) or isinstance(volume, RootfsVolume):
+    elif isinstance(volume, (PersistentVolume, RootfsVolume)):
         volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
         if volume.persistence != VolumePersistence.host:
-            raise NotImplementedError("Only 'host' persistence is supported")
+            msg = "Only 'host' persistence is supported"
+            raise NotImplementedError(msg)
         if not re.match(r"^[\w\-_/]+$", volume_name):
-            raise ValueError(f"Invalid value for volume name: {volume_name}")
+            msg = f"Invalid value for volume name: {volume_name}"
+            raise ValueError(msg)
         (Path(settings.PERSISTENT_VOLUMES_DIR) / namespace).mkdir(exist_ok=True)
         if volume.parent:
             return await create_devmapper(volume, namespace)
         else:
-            volume_path = (
-                Path(settings.PERSISTENT_VOLUMES_DIR)
-                / namespace
-                / f"{volume_name}.ext4"
-            )
+            volume_path = Path(settings.PERSISTENT_VOLUMES_DIR) / namespace / f"{volume_name}.ext4"
             await create_ext4(volume_path, volume.size_mib)
             return volume_path
     else:
-        raise NotImplementedError("Only immutable volumes are supported")
+        msg = "Only immutable volumes are supported"
+        raise NotImplementedError(msg)
 
 
 async def create_volume_snapshot(path: Path) -> Path:
diff --git a/vm_supervisor/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
similarity index 95%
rename from vm_supervisor/supervisor.py
rename to src/aleph/vm/orchestrator/supervisor.py
index 959e306e6..1b68e8ab8 100644
--- a/vm_supervisor/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -6,8 +6,9 @@
 evolve in the future.
 """
 import logging
+from collections.abc import Awaitable
 from secrets import token_urlsafe
-from typing import Awaitable, Callable
+from typing import Callable
 
 from aiohttp import web
 
@@ -82,9 +83,7 @@ def run():
     # Require a random token to access /about APIs
     secret_token = token_urlsafe(nbytes=32)
     app["secret_token"] = secret_token
-    print(
-        f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}"
-    )
+    print(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
 
     engine = metrics.setup_engine()
     metrics.create_tables(engine)
diff --git a/vm_supervisor/tasks.py b/src/aleph/vm/orchestrator/tasks.py
similarity index 90%
rename from vm_supervisor/tasks.py
rename to src/aleph/vm/orchestrator/tasks.py
index 53b483d13..8adabcf3c 100644
--- a/vm_supervisor/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -3,7 +3,8 @@
 import logging
 import math
 import time
-from typing import AsyncIterable, TypeVar
+from collections.abc import AsyncIterable
+from typing import TypeVar
 
 import aiohttp
 import pydantic
@@ -22,9 +23,7 @@
 Value = TypeVar("Value")
 
 
-async def retry_generator(
-    generator: AsyncIterable[Value], max_seconds: int = 8
-) -> AsyncIterable[Value]:
+async def retry_generator(generator: AsyncIterable[Value], max_seconds: int = 8) -> AsyncIterable[Value]:
     retry_delay = 0.1
     while True:
         async for value in generator:
@@ -55,9 +54,7 @@ async def subscribe_via_ws(url) -> AsyncIterable[AlephMessage]:
                     if "item_type" not in data:
                         assert "content" not in data
                         assert "confirmation" in data
-                        logger.info(
-                            f"Ignoring confirmation message '{data['item_hash']}'"
-                        )
+                        logger.info(f"Ignoring confirmation message '{data['item_hash']}'")
                         continue
 
                     try:
@@ -87,9 +84,7 @@ async def subscribe_via_ws(url) -> AsyncIterable[AlephMessage]:
 async def watch_for_messages(dispatcher: PubSub, reactor: Reactor):
     """Watch for new Aleph messages"""
     logger.debug("watch_for_messages()")
-    url = URL(f"{settings.API_SERVER}/api/ws0/messages").with_query(
-        {"startDate": math.floor(time.time())}
-    )
+    url = URL(f"{settings.API_SERVER}/api/ws0/messages").with_query({"startDate": math.floor(time.time())})
 
     async for message in retry_generator(subscribe_via_ws(url)):
         # Dispatch update to running VMs
@@ -120,9 +115,7 @@ async def start_watch_for_messages_task(app: web.Application):
 
     app["pubsub"] = pubsub
     app["reactor"] = reactor
-    app["messages_listener"] = create_task_log_exceptions(
-        watch_for_messages(pubsub, reactor)
-    )
+    app["messages_listener"] = create_task_log_exceptions(watch_for_messages(pubsub, reactor))
 
 
 async def stop_watch_for_messages_task(app: web.Application):
diff --git a/vm_supervisor/utils.py b/src/aleph/vm/orchestrator/utils.py
similarity index 80%
rename from vm_supervisor/utils.py
rename to src/aleph/vm/orchestrator/utils.py
index 3acd264f9..3abd7cccb 100644
--- a/vm_supervisor/utils.py
+++ b/src/aleph/vm/orchestrator/utils.py
@@ -5,9 +5,10 @@
 import logging
 import subprocess
 from base64 import b16encode, b32decode
+from collections.abc import Coroutine
 from dataclasses import asdict as dataclass_as_dict
 from dataclasses import is_dataclass
-from typing import Any, Coroutine, Dict, List, Optional
+from typing import Any, Optional
 
 import aiodns
 import msgpack
@@ -18,14 +19,16 @@
 class MsgpackSerializable:
     def __post_init__(self, *args, **kwargs):
         if not is_dataclass(self):
-            raise TypeError(f"Decorated class must be a dataclass: {self}")
+            msg = f"Decorated class must be a dataclass: {self}"
+            raise TypeError(msg)
         super().__init_subclass__(*args, **kwargs)
 
     def as_msgpack(self) -> bytes:
         if is_dataclass(self):
             return msgpack.dumps(dataclasses.asdict(self), use_bin_type=True)  # type: ignore
         else:
-            raise TypeError(f"Decorated class must be a dataclass: {self}")
+            msg = f"Decorated class must be a dataclass: {self}"
+            raise TypeError(msg)
 
 
 def b32_to_b16(hash: str) -> bytes:
@@ -71,9 +74,7 @@ def create_task_log_exceptions(coro: Coroutine, *, name=None):
     return asyncio.create_task(run_and_log_exception(coro), name=name)
 
 
-async def run_in_subprocess(
-    command: List[str], check: bool = True, stdin_input: Optional[bytes] = None
-) -> bytes:
+async def run_in_subprocess(command: list[str], check: bool = True, stdin_input: Optional[bytes] = None) -> bytes:
     """Run the specified command in a subprocess, returns the stdout of the process."""
     logger.debug(f"command: {' '.join(command)}")
 
@@ -92,19 +93,15 @@ async def run_in_subprocess(
             f"    command = {command}\n"
             f"    stdout = {stderr!r}"
         )
-        raise subprocess.CalledProcessError(
-            process.returncode, str(command), stderr.decode()
-        )
+        raise subprocess.CalledProcessError(process.returncode, str(command), stderr.decode())
 
     return stdout
 
 
-def fix_message_validation(message: Dict) -> Dict:
+def fix_message_validation(message: dict) -> dict:
     """Patch a fake message program to pass validation."""
     message["item_content"] = json.dumps(message["content"])
-    message["item_hash"] = hashlib.sha256(
-        message["item_content"].encode("utf-8")
-    ).hexdigest()
+    message["item_hash"] = hashlib.sha256(message["item_content"].encode("utf-8")).hexdigest()
     return message
 
 
@@ -118,8 +115,6 @@ async def ping(host: str, packets: int, timeout: float):
     """
 
     try:
-        await run_in_subprocess(
-            ["ping", "-c", str(packets), "-W", str(timeout), host], check=True
-        )
+        await run_in_subprocess(["ping", "-c", str(packets), "-W", str(timeout), host], check=True)
     except subprocess.CalledProcessError as err:
         raise HostNotFoundError() from err
diff --git a/vm_supervisor/version.py b/src/aleph/vm/orchestrator/version.py
similarity index 100%
rename from vm_supervisor/version.py
rename to src/aleph/vm/orchestrator/version.py
diff --git a/vm_supervisor/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
similarity index 84%
rename from vm_supervisor/views/__init__.py
rename to src/aleph/vm/orchestrator/views/__init__.py
index a60327d95..e6ce2f874 100644
--- a/vm_supervisor/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -1,9 +1,10 @@
 import binascii
 import logging
+from collections.abc import Awaitable
 from hashlib import sha256
 from pathlib import Path
 from string import Template
-from typing import Awaitable, Dict, Optional
+from typing import Optional
 
 import aiodns
 import aiohttp
@@ -13,23 +14,26 @@
 from aleph_message.models import ItemHash
 from pydantic import ValidationError
 
-from firecracker.microvm import MicroVMFailedInit
-from packaging.version import InvalidVersion, Version
-from vm_supervisor import status
-from vm_supervisor.conf import settings
-from vm_supervisor.metrics import get_execution_records
-from vm_supervisor.pubsub import PubSub
-from vm_supervisor.resources import Allocation
-from vm_supervisor.run import pool, run_code_on_request, start_persistent_vm
-from vm_supervisor.utils import (
+from aleph.vm.controllers.firecracker.executable import (
+    ResourceDownloadError,
+    VmSetupError,
+)
+from aleph.vm.controllers.firecracker.program import FileTooLargeError
+from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInit
+from aleph.vm.orchestrator import status
+from aleph.vm.orchestrator.conf import settings
+from aleph.vm.orchestrator.metrics import get_execution_records
+from aleph.vm.orchestrator.pubsub import PubSub
+from aleph.vm.orchestrator.resources import Allocation
+from aleph.vm.orchestrator.run import pool, run_code_on_request, start_persistent_vm
+from aleph.vm.orchestrator.utils import (
     HostNotFoundError,
     b32_to_b16,
     dumps_for_json,
     get_ref_from_dns,
 )
-from vm_supervisor.version import __version__
-from vm_supervisor.vm.firecracker.executable import ResourceDownloadError, VmSetupError
-from vm_supervisor.vm.firecracker.program import FileTooLargeError
+from aleph.vm.orchestrator.version import __version__
+from packaging.version import InvalidVersion, Version
 
 logger = logging.getLogger(__name__)
 
@@ -57,11 +61,7 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
     we expect the hash to be encoded in base32 instead of hexadecimal. Padding is added
     automatically.
     """
-    if (
-        request.host.split(":")[0] == settings.DOMAIN_NAME
-        and request.method == "GET"
-        and request.path == "/"
-    ):
+    if request.host.split(":")[0] == settings.DOMAIN_NAME and request.method == "GET" and request.path == "/":
         # Serve the index page
         return await index(request=request)
 
@@ -70,20 +70,14 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
 
     message_ref_base32 = request.host.split(".")[0]
     if settings.FAKE_DATA_PROGRAM:
-        message_ref = ItemHash(
-            "cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe"
-        )
+        message_ref = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")
     else:
         try:
             message_ref = ItemHash(b32_to_b16(message_ref_base32).decode())
-            logger.debug(
-                f"Using base32 message id from hostname to obtain '{message_ref}"
-            )
+            logger.debug(f"Using base32 message id from hostname to obtain '{message_ref}")
         except binascii.Error:
             try:
-                message_ref = ItemHash(
-                    await get_ref_from_dns(domain=f"_aleph-id.{request.host}")
-                )
+                message_ref = ItemHash(await get_ref_from_dns(domain=f"_aleph-id.{request.host}"))
                 logger.debug(f"Using DNS TXT record to obtain '{message_ref}'")
             except aiodns.error.DNSError:
                 raise HTTPNotFound(reason="Invalid message reference")
@@ -110,7 +104,7 @@ async def about_login(request: web.Request) -> web.Response:
 async def about_executions(request: web.Request) -> web.Response:
     authenticate_request(request)
     return web.json_response(
-        [{key: value for key, value in pool.executions.items()}],
+        [dict(pool.executions.items())],
         dumps=dumps_for_json,
     )
 
@@ -189,9 +183,7 @@ async def status_check_version(request: web.Request):
         raise web.HTTPServiceUnavailable(text=error.args[0])
 
     if current >= reference:
-        return web.Response(
-            status=200, text=f"Up-to-date: version {current} >= {reference}"
-        )
+        return web.Response(status=200, text=f"Up-to-date: version {current} >= {reference}")
     else:
         return web.HTTPForbidden(text=f"Outdated: version {current} < {reference}")
 
@@ -215,9 +207,7 @@ async def update_allocations(request: web.Request):
         data = await request.json()
         allocation = Allocation.parse_obj(data)
     except ValidationError as error:
-        return web.json_response(
-            data=error.json(), status=web.HTTPBadRequest.status_code
-        )
+        return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code)
 
     pubsub: PubSub = request.app["pubsub"]
 
@@ -242,7 +232,7 @@ async def update_allocations(request: web.Request):
         HostNotFoundError,
     )
 
-    scheduling_errors: Dict[ItemHash, Exception] = {}
+    scheduling_errors: dict[ItemHash, Exception] = {}
 
     # Schedule the start of persistent VMs:
     for vm_hash in allocation.persistent_vms:
@@ -286,9 +276,7 @@ async def update_allocations(request: web.Request):
             "success": not failing,
             "successful": list(successful),
             "failing": list(failing),
-            "errors": {
-                vm_hash: repr(error) for vm_hash, error in scheduling_errors.items()
-            },
+            "errors": {vm_hash: repr(error) for vm_hash, error in scheduling_errors.items()},
         },
         status=status_code,
     )
diff --git a/vm_supervisor/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
similarity index 95%
rename from vm_supervisor/views/operator.py
rename to src/aleph/vm/orchestrator/views/operator.py
index 6bbd4ec9d..f25b20667 100644
--- a/vm_supervisor/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -2,8 +2,9 @@
 import functools
 import json
 import logging
+from collections.abc import Awaitable
 from datetime import datetime, timedelta
-from typing import Awaitable, Callable
+from typing import Callable
 
 import aiohttp.web_exceptions
 from aiohttp import web
@@ -14,8 +15,8 @@
 from eth_account.messages import encode_defunct
 from jwskate import Jwk
 
-from ..models import VmExecution
-from ..run import pool
+from aleph.vm.orchestrator.models import VmExecution
+from aleph.vm.orchestrator.run import pool
 
 logger = logging.getLogger(__name__)
 
@@ -96,9 +97,7 @@ async def authenticate_jwk(request: web.Request):
         raise web.HTTPUnauthorized(reason="Signature could not verified")
 
 
-def require_jwk_authentication(
-    handler: Callable[[web.Request], Awaitable[web.StreamResponse]]
-):
+def require_jwk_authentication(handler: Callable[[web.Request], Awaitable[web.StreamResponse]]):
     @functools.wraps(handler)
     async def wrapper(request):
         try:
@@ -177,9 +176,7 @@ async def operate_expire(request: web.Request):
     await execution.expire(timeout=timeout)
     execution.persistent = False
 
-    return web.Response(
-        status=200, body=f"Expiring VM with ref {vm_hash} in {timeout} seconds"
-    )
+    return web.Response(status=200, body=f"Expiring VM with ref {vm_hash} in {timeout} seconds")
 
 
 @require_jwk_authentication
diff --git a/vm_supervisor/views/templates/index.html b/src/aleph/vm/orchestrator/views/templates/index.html
similarity index 100%
rename from vm_supervisor/views/templates/index.html
rename to src/aleph/vm/orchestrator/views/templates/index.html
diff --git a/src/aleph/vm/orchestrator/vm/__init__.py b/src/aleph/vm/orchestrator/vm/__init__.py
new file mode 100644
index 000000000..7d85867cc
--- /dev/null
+++ b/src/aleph/vm/orchestrator/vm/__init__.py
@@ -0,0 +1,9 @@
+from aleph.vm.controllers.firecracker import (
+    AlephFirecrackerInstance,
+    AlephFirecrackerProgram,
+)
+
+__all__ = (
+    "AlephFirecrackerProgram",
+    "AlephFirecrackerInstance",
+)
diff --git a/vm_supervisor/vm/vm_type.py b/src/aleph/vm/orchestrator/vm/vm_type.py
similarity index 84%
rename from vm_supervisor/vm/vm_type.py
rename to src/aleph/vm/orchestrator/vm/vm_type.py
index 7e568862b..eb0b5e42a 100644
--- a/vm_supervisor/vm/vm_type.py
+++ b/src/aleph/vm/orchestrator/vm/vm_type.py
@@ -18,4 +18,5 @@ def from_message_content(content: ExecutableContent) -> "VmType":
                 return VmType.persistent_program
             return VmType.microvm
 
-        raise TypeError(f"Unexpected message content type: {type(content)}")
+        msg = f"Unexpected message content type: {type(content)}"
+        raise TypeError(msg)
diff --git a/tests/supervisor/test_ipv6_allocator.py b/tests/supervisor/test_ipv6_allocator.py
index ff354ed86..c9cf264cc 100644
--- a/tests/supervisor/test_ipv6_allocator.py
+++ b/tests/supervisor/test_ipv6_allocator.py
@@ -1,6 +1,7 @@
 import os
 
-from vm_supervisor.vm.vm_type import VmType
+from aleph.vm.orchestrator.network.hostnetwork import StaticIPv6Allocator
+from aleph.vm.orchestrator.vm.vm_type import VmType
 
 # Avoid failures linked to settings when initializing the global VmPool object
 os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
@@ -9,18 +10,12 @@
 
 from aleph_message.models import ItemHash
 
-from vm_supervisor.network.hostnetwork import StaticIPv6Allocator
-
 
 def test_static_ipv6_allocator():
-    allocator = StaticIPv6Allocator(
-        ipv6_range=IPv6Network("1111:2222:3333:4444::/64"), subnet_prefix=124
-    )
+    allocator = StaticIPv6Allocator(ipv6_range=IPv6Network("1111:2222:3333:4444::/64"), subnet_prefix=124)
     ip_subnet = allocator.allocate_vm_ipv6_subnet(
         vm_id=3,
-        vm_hash=ItemHash(
-            "8920215b2e961a4d4c59a8ceb2803af53f91530ff53d6704273ab4d380bc6446"
-        ),
+        vm_hash=ItemHash("8920215b2e961a4d4c59a8ceb2803af53f91530ff53d6704273ab4d380bc6446"),
         vm_type=VmType.microvm,
     )
     assert ip_subnet == IPv6Network("1111:2222:3333:4444:0001:8920:215b:2e90/124")
diff --git a/tests/supervisor/test_jwk.py b/tests/supervisor/test_jwk.py
index e1db545bb..4f075d5f7 100644
--- a/tests/supervisor/test_jwk.py
+++ b/tests/supervisor/test_jwk.py
@@ -2,6 +2,8 @@
 
 from aiohttp import web
 
+from aleph.vm.orchestrator.views.operator import authenticate_jwk
+
 # Avoid failures linked to settings when initializing the global VmPool object
 os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
 
@@ -9,14 +11,10 @@
 
 import pytest
 
-from vm_supervisor.views.operator import authenticate_jwk
-
 
 @pytest.fixture
 def valid_jwk_headers(mocker):
-    mocker.patch(
-        "vm_supervisor.views.operator.is_token_still_valid", lambda timestamp: True
-    )
+    mocker.patch("aleph.vm.orchestrator.views.operator.is_token_still_valid", lambda timestamp: True)
     return {
         "X-SignedPubKey": '{"payload":"7b227075626b6579223a7b22616c67223a224553323536222c22637276223a22502d323536222c22657874223a747275652c226b65795f6f7073223a5b22766572696679225d2c226b7479223a224543222c2278223a224b65763844614d7356454673365a6b4679525a4272796344564138566a334f656e49756f34743561374634222c2279223a2279597343556d715978654767673643743736794f47525873545867446444795234644f5639514c6f6b6477227d2c22616c67223a224543445341222c22646f6d61696e223a226c6f63616c686f7374222c2261646472657373223a22307833343932346566393435623933316431653932393337353535366636396365326537666535646363222c2265787069726573223a313638393337353132342e3532317d","signature":"0x58e1498a6c4f88ac1982e7147ff49405ffe1b9633e048bb74cf741abb05ce0b63bb406f3079f641ae89f597654ecd2a704d37ffbf86a28e462140033cc0eedcb1c"}',
         "X-SignedOperation": '{"time":"2023-07-14T22:14:14.132Z","signature":"96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0add"}',
@@ -44,9 +42,7 @@ async def test_invalid_signature(valid_jwk_headers: Dict[str, Any], mocker):
 
 @pytest.mark.asyncio
 async def test_expired_token(valid_jwk_headers: Dict[str, Any], mocker):
-    mocker.patch(
-        "vm_supervisor.views.operator.is_token_still_valid", lambda timestamp: False
-    )
+    mocker.patch("aleph.vm.orchestrator.views.operator.is_token_still_valid", lambda timestamp: False)
     request = mocker.AsyncMock()
     request.headers = valid_jwk_headers
 
diff --git a/tests/supervisor/test_resolvectl_dns_servers.py b/tests/supervisor/test_resolvectl_dns_servers.py
index 0889ba23c..dc45fa31b 100644
--- a/tests/supervisor/test_resolvectl_dns_servers.py
+++ b/tests/supervisor/test_resolvectl_dns_servers.py
@@ -1,13 +1,17 @@
 # Avoid failures linked to nftables when initializing the global VmPool object
 import os
-os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
 
-from vm_supervisor.conf import resolvectl_dns_servers, resolvectl_dns_servers_ipv4
+from aleph.vm.orchestrator.conf import (
+    resolvectl_dns_servers,
+    resolvectl_dns_servers_ipv4,
+)
+
+os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
 
 
 def test_resolvectl(mocker):
     with mocker.patch(
-        "vm_supervisor.conf.check_output",
+        "aleph.vm.orchestrator.conf.check_output",
         return_value="Link 2 (eth0): 109.88.203.3 62.197.111.140\n",
     ):
         servers = {"109.88.203.3", "62.197.111.140"}
@@ -21,7 +25,7 @@ def test_resolvectl(mocker):
 
 def test_resolvectl_ipv6(mocker):
     with mocker.patch(
-        "vm_supervisor.conf.check_output",
+        "aleph.vm.orchestrator.conf.check_output",
         return_value="Link 2 (eth0): 109.88.203.3 62.197.111.140 2a02:2788:fff0:7::3\n        2a02:2788:fff0:5::140\n",
     ):
         ipv4_servers = {"109.88.203.3", "62.197.111.140"}
diff --git a/tutorials/TESTING.md b/tutorials/TESTING.md
index a222fd042..71086da78 100644
--- a/tutorials/TESTING.md
+++ b/tutorials/TESTING.md
@@ -46,7 +46,7 @@ bash ./docker/run_vm_supervisor.sh
 
 Within the container, run the supervisor with fake data:
 ```shell
-python3 -m vm_supervisor --print-settings --very-verbose --system-logs --fake-data-program ./examples/example_fastapi
+python3 -m orchestrator --print-settings --very-verbose --system-logs --fake-data-program ./examples/example_fastapi
 ```
 
 > ℹ️ The command is in your .bash_history, press key up to skip typing it. 
@@ -55,11 +55,11 @@ python3 -m vm_supervisor --print-settings --very-verbose --system-logs --fake-da
 
 ### 2.a. Install the system requirements
 
-See [../vm_supervisor/README.md](../vm_supervisor/README.md) to install the system requirements.
+See [../vm_supervisor/README.md](../src/aleph/vm/orchestrator/README.md) to install the system requirements.
 
 ### 2.b. Run the supervisor with fake data:
 
 ```shell
-python3 -m vm_supervisor --print-settings --very-verbose --system-logs --fake-data-program ./examples/example_fastapi
+python3 -m orchestrator --print-settings --very-verbose --system-logs --fake-data-program ./examples/example_fastapi
 ```
 
diff --git a/vm_connector/README.md b/vm_connector/README.md
index ad86e1a2c..40397d97b 100644
--- a/vm_connector/README.md
+++ b/vm_connector/README.md
@@ -2,7 +2,7 @@
 
 Service to schedule the execution of Aleph VM functions 
 for the [Aleph.im](https://aleph.im/) project and assist 
-[VM Supervisors](../vm_supervisor) with operations related 
+[VM Supervisors](../src/aleph/vm/orchestrator) with operations related 
 to the Aleph network.
 
 ## 1. Supported platforms
diff --git a/vm_connector/conf.py b/vm_connector/conf.py
index 58b5862a3..d2ee465fc 100644
--- a/vm_connector/conf.py
+++ b/vm_connector/conf.py
@@ -24,8 +24,7 @@ def update(self, **kwargs):
 
     def display(self) -> str:
         return "\n".join(
-            f"{annotation:<17} = {getattr(self, annotation)}"
-            for annotation, value in self.__annotations__.items()
+            f"{annotation:<17} = {getattr(self, annotation)}" for annotation, value in self.__annotations__.items()
         )
 
     class Config:
diff --git a/vm_connector/main.py b/vm_connector/main.py
index 324d3101b..02662d923 100644
--- a/vm_connector/main.py
+++ b/vm_connector/main.py
@@ -1,14 +1,14 @@
 import json
 import logging
-from typing import Optional, Dict, Union
+from typing import Dict, Optional, Union
 
 import aiohttp
 from aleph_client.asynchronous import create_post
 from aleph_client.chains.common import get_fallback_private_key
 from aleph_client.chains.ethereum import ETHAccount
 from aleph_client.types import StorageEnum
-from fastapi import FastAPI, Request, HTTPException
-from fastapi.responses import StreamingResponse, Response
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import Response, StreamingResponse
 from pydantic import BaseModel
 
 from .conf import settings
@@ -27,8 +27,7 @@ def read_root():
 async def get_latest_message_amend(ref: str, sender: str) -> Optional[Dict]:
     async with aiohttp.ClientSession() as session:
         url = (
-            f"{settings.API_SERVER}/api/v0/messages.json?msgType=STORE&sort_order=-1"
-            f"&refs={ref}&addresses={sender}"
+            f"{settings.API_SERVER}/api/v0/messages.json?msgType=STORE&sort_order=-1" f"&refs={ref}&addresses={sender}"
         )
         resp = await session.get(url)
         resp.raise_for_status()
diff --git a/vm_supervisor/vm/__init__.py b/vm_supervisor/vm/__init__.py
deleted file mode 100644
index 5cab3b7b3..000000000
--- a/vm_supervisor/vm/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from .firecracker import AlephFirecrackerInstance, AlephFirecrackerProgram
-
-__all__ = (
-    "AlephFirecrackerProgram",
-    "AlephFirecrackerInstance",
-)

From b092b612fb4ab0271f86d3ae31c92440c5b50e79 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 11 Oct 2023 14:38:36 +0200
Subject: [PATCH 509/990] Fix: pyproject.toml was required by "pip"

This adds a `pyproject.toml` file for pip, with dependencies
and commands to test the software.
---
 .github/workflows/code-quality.yml        |  26 +--
 .github/workflows/test-build-examples.yml |   4 +-
 pyproject.toml                            | 208 ++++++++++++++++++++++
 3 files changed, 215 insertions(+), 23 deletions(-)
 create mode 100644 pyproject.toml

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
index 738a6ed27..d1a25d9d1 100644
--- a/.github/workflows/code-quality.yml
+++ b/.github/workflows/code-quality.yml
@@ -21,31 +21,15 @@ jobs:
 
       - name: Install required Python packages
         run: |
-          python3 -m pip install mypy pytest black isort flake8
+          python3 -m pip install hatch
 
-      - name: Test with Black
+      - name: Test style wth ruff, black and isoort
         run: |
-          black --check ./vm_supervisor
-          black --check ./runtimes/aleph-debian-11-python/init1.py
-          black --check ./examples/example_fastapi/
+          hatch run lint:style
 
-      - name: Test with isort
+      - name: Test typing with Mypy
         run: |
-          isort --check-only --profile=black ./vm_supervisor
-          isort --check-only --profile=black ./runtimes/aleph-debian-11-python/init1.py
-          isort --check-only --profile=black ./examples/example_fastapi/
-
-      - name: Test with MyPy
-        run: |
-          mypy --ignore-missing-imports ./vm_supervisor
-          mypy --ignore-missing-imports ./runtimes/aleph-debian-11-python/init1.py
-          mypy --ignore-missing-imports ./examples/example_fastapi/
-
-      - name: Test with flake8
-        run: |
-          flake8 --extend-ignore E501 ./vm_supervisor
-          flake8 --extend-ignore E501,E402 ./runtimes/aleph-debian-11-python/init1.py
-          flake8 --extend-ignore E501,E402 ./examples/example_fastapi/
+          hatch run lint:typing
 
   code-quality-shell:
     runs-on: ubuntu-22.04
diff --git a/.github/workflows/test-build-examples.yml b/.github/workflows/test-build-examples.yml
index c37e74e8c..2cf702c89 100644
--- a/.github/workflows/test-build-examples.yml
+++ b/.github/workflows/test-build-examples.yml
@@ -25,10 +25,10 @@ jobs:
           sudo chown $(whoami) /opt/packages
 
       - run: |
-          pip3 install aleph-sdk-python
+          pip3 install hatch
 
       - run: |
-           hatch build
+          hatch build
 
       - run: |
           ls
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000..4367dac9f
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,208 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "aleph-vm"
+dynamic = ["version"]
+description = "Aleph.im VM execution engine"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+keywords = []
+authors = [
+  { name="Hugo Herter", email="git@hugoherter.com" },
+]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Environment :: Console",
+  "Framework :: aiohttp",
+  "Intended Audience :: Information Technology",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: POSIX :: Linux",
+  "Programming Language :: Python :: 3",
+  "Topic :: System :: Distributed Computing",
+]
+dependencies = [
+  "pydantic[dotenv]~=1.10.13",
+  "aiohttp~=3.8.6",
+  "aiodns~=3.1.0",
+  "alembic~=1.7.6",
+  "setproctitle~=1.3.3",
+  "pyyaml~=6.0.1",
+  "aleph-message~=0.4.0",
+  "jwskate~=0.8.0",
+  "eth-account~=0.9.0",
+  "sentry-sdk~=1.31.0",
+  "aioredis~=1.3.1",
+  "psutil~=5.9.5",
+  "py-cpuinfo~=9.0.0",
+  "schedule~=1.2.1",
+  "nftables @ git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py",
+  "msgpack~=1.0.7",
+  "packaging~=23.2",
+  "jsonschema==4.19.1",
+]
+
+[project.urls]
+Documentation = "https://docs.aleph.im/nodes/compute/"
+Issues = "https://github.com/aleph-im/aleph-vm/issues"
+Source = "https://github.com/aleph-im/aleph-vm"
+Discussions = "https://community.aleph.im/"
+
+[project.scripts]
+aleph-vm = "aleph.vm.orchestrator.cli:main"
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/aleph"]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.envs.default]
+platforms = ["linux"]
+dependencies = [
+#  "git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py",
+]
+
+[tool.hatch.envs.default.scripts]
+orchestrator = "aleph-vm orchestrator run {args:--help}"
+config = "aleph-vm orchestrator config {args:--help}"
+check = "aleph-vm controller run {args:--help}"
+
+[tool.hatch.envs.testing]
+dependencies = [
+  "coverage[toml]~=7.3.2",
+  "pytest~=7.4.2",
+  "pytest-mock~=3.11.1",
+  "pytest-asyncio~=0.21.1 ",
+]
+[tool.hatch.envs.testing.scripts]
+test = "pytest {args:tests}"
+test-cov = "coverage run -m pytest {args:tests}"
+cov-report = [
+  "- coverage combine",
+  "coverage report",
+]
+cov = [
+  "test-cov",
+  "cov-report",
+]
+
+[[tool.hatch.envs.all.matrix]]
+python = ["3.10", "3.11", "3.12"]
+
+[tool.hatch.envs.lint]
+detached = true
+dependencies = [
+  "black>=23.9.0",
+  "mypy>=1.6.0",
+  "ruff>=0.0.292",
+  "isort>=5.12.0",
+]
+[tool.hatch.envs.lint.scripts]
+typing = "mypy --install-types --non-interactive --ignore-missing-imports --explicit-package-bases {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}"
+style = [
+#  "ruff {args:.}",
+  "black --check --diff {args:.}",
+  "isort --check-only --profile black {args:.}",
+]
+fmt = [
+  "black {args:.}",
+#  "ruff --fix {args:.}",
+  "isort --profile black {args:.}",
+  "style",
+]
+all = [
+  "style",
+  "typing",
+]
+
+[tool.pytest.ini_options]
+pythonpath = [
+  "src"
+]
+
+[tool.black]
+target-version = ["py39"]
+line-length = 120
+#skip-string-normalization = true
+
+[tool.ruff]
+target-version = "py39"
+line-length = 120
+select = [
+  "A",
+  "ARG",
+  "B",
+  "C",
+  "DTZ",
+  "E",
+  "EM",
+  "F",
+  "FBT",
+  "I",
+  "ICN",
+  "ISC",
+  "N",
+  "PLC",
+  "PLE",
+  "PLR",
+  "PLW",
+  "Q",
+  "RUF",
+  "S",
+  "T",
+  "TID",
+  "UP",
+  "W",
+  "YTT",
+]
+ignore = [
+#  # Allow non-abstract empty methods in abstract base classes
+#  "B027",
+#  # Allow boolean positional values in function calls, like `dict.get(... True)`
+#  "FBT003",
+#  # Ignore checks for possible passwords
+#  "S105", "S106", "S107",
+#  # Ignore complexity
+#  "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
+  # Allow the use of assert statements
+  "S101"
+]
+#unfixable = [
+#  # Don't touch unused imports
+#  "F401",
+#]
+
+[tool.ruff.isort]
+known-first-party = ["aleph.vm"]
+
+#[tool.ruff.flake8-tidy-imports]
+#ban-relative-imports = "all"
+
+[tool.ruff.per-file-ignores]
+# Tests can use magic values, assertions, and relative imports
+"tests/**/*" = ["PLR2004", "S101", "TID252"]
+
+[tool.coverage.run]
+source_pkgs = ["aleph.vm", "tests"]
+branch = true
+parallel = true
+omit = [
+  "src/aleph/vm/__about__.py",
+]
+
+[tool.coverage.paths]
+aleph_vm = ["src/aleph/vm", "*/aleph-vm/src/aleph/vm"]
+tests = ["tests", "*/aleph-vm/tests"]
+
+[tool.coverage.report]
+exclude_lines = [
+  "no cov",
+  "if __name__ == .__main__.:",
+  "if TYPE_CHECKING:",
+]

From 3e8b3493c4b04394c49acaee868a076b8849a565 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 11 Oct 2023 16:42:01 +0200
Subject: [PATCH 510/990] Fix: Missing commands crashed during execution

This checks for the presence of these commands when starting the service, and recommends a way to install them when available.
---
 src/aleph/vm/orchestrator/conf.py  | 5 +++++
 src/aleph/vm/orchestrator/utils.py | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/src/aleph/vm/orchestrator/conf.py b/src/aleph/vm/orchestrator/conf.py
index 650d55427..8d17c2ed0 100644
--- a/src/aleph/vm/orchestrator/conf.py
+++ b/src/aleph/vm/orchestrator/conf.py
@@ -11,6 +11,8 @@
 
 from pydantic import BaseSettings, Field
 
+from aleph.vm.orchestrator.utils import is_command_available
+
 logger = logging.getLogger(__name__)
 
 Url = NewType("Url", str)
@@ -274,6 +276,9 @@ def check(self):
             assert isfile(self.FAKE_DATA_RUNTIME), "Local runtime .squashfs build is missing"
             assert isfile(self.FAKE_DATA_VOLUME), "Local data volume .squashfs is missing"
 
+        assert is_command_available("setfacl"), "Command `setfacl` not found, run `apt install acl`"
+        assert is_command_available("ndppd"), "Command `ndppd` not found, run `apt install ndppd`"
+
     def setup(self):
         os.makedirs(self.MESSAGE_CACHE, exist_ok=True)
         os.makedirs(self.CODE_CACHE, exist_ok=True)
diff --git a/src/aleph/vm/orchestrator/utils.py b/src/aleph/vm/orchestrator/utils.py
index 3abd7cccb..72554a03a 100644
--- a/src/aleph/vm/orchestrator/utils.py
+++ b/src/aleph/vm/orchestrator/utils.py
@@ -98,6 +98,14 @@ async def run_in_subprocess(command: list[str], check: bool = True, stdin_input:
     return stdout
 
 
+def is_command_available(command):
+    try:
+        subprocess.check_output(["which", command], stderr=subprocess.STDOUT)
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+
 def fix_message_validation(message: dict) -> dict:
     """Patch a fake message program to pass validation."""
     message["item_content"] = json.dumps(message["content"])

From 3ea89feaa8b0e690f72acf306418c65b1ee57705 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 11 Oct 2023 16:43:34 +0200
Subject: [PATCH 511/990] Fix: Error "Context has already been set"

Solution: Handle that scenario and dismiss this specific error.
---
 src/aleph/vm/controllers/firecracker/executable.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index 67fede347..4b7798d46 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -29,7 +29,15 @@
     psutil = None  # type: ignore [assignment]
 
 logger = logging.getLogger(__name__)
-set_start_method("spawn")
+
+try:
+    set_start_method("spawn")
+except RuntimeError as error:
+    if error.args == ("context has already been set",):
+        logger.info("Start method has already been set")
+        pass
+    else:
+        raise error
 
 
 class ResourceDownloadError(ClientResponseError):

From 0489076efc7f1314340ef1503f61e7e323b18a7f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 11 Oct 2023 17:12:03 +0200
Subject: [PATCH 512/990] Fix: Deprecation warnings when running `pytest`

---
 src/aleph/vm/orchestrator/metrics.py            |  1 -
 tests/supervisor/test_resolvectl_dns_servers.py | 10 +++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index b9aaf97fd..b0af25e3b 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -14,7 +14,6 @@
     from sqlalchemy.ext.declarative import declarative_base
 
 
-
 from .conf import make_db_url, settings
 
 Session: sessionmaker
diff --git a/tests/supervisor/test_resolvectl_dns_servers.py b/tests/supervisor/test_resolvectl_dns_servers.py
index dc45fa31b..362cab70b 100644
--- a/tests/supervisor/test_resolvectl_dns_servers.py
+++ b/tests/supervisor/test_resolvectl_dns_servers.py
@@ -1,6 +1,6 @@
 # Avoid failures linked to nftables when initializing the global VmPool object
 import os
-
+from unittest import mock
 from aleph.vm.orchestrator.conf import (
     resolvectl_dns_servers,
     resolvectl_dns_servers_ipv4,
@@ -9,8 +9,8 @@
 os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
 
 
-def test_resolvectl(mocker):
-    with mocker.patch(
+def test_resolvectl():
+    with mock.patch(
         "aleph.vm.orchestrator.conf.check_output",
         return_value="Link 2 (eth0): 109.88.203.3 62.197.111.140\n",
     ):
@@ -23,8 +23,8 @@ def test_resolvectl(mocker):
         assert dns_servers_ipv4 == servers
 
 
-def test_resolvectl_ipv6(mocker):
-    with mocker.patch(
+def test_resolvectl_ipv6():
+    with mock.patch(
         "aleph.vm.orchestrator.conf.check_output",
         return_value="Link 2 (eth0): 109.88.203.3 62.197.111.140 2a02:2788:fff0:7::3\n        2a02:2788:fff0:5::140\n",
     ):

From 36bcd28b03de66647a8eb37696b17b871146e520 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 13 Oct 2023 13:12:09 +0200
Subject: [PATCH 513/990] Cleanup: Use black

---
 src/aleph/vm/orchestrator/views/__init__.py     | 4 +---
 tests/supervisor/test_resolvectl_dns_servers.py | 1 +
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index e6ce2f874..dbd96ffeb 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -140,9 +140,7 @@ async def index(request: web.Request):
 
 
 async def status_check_fastapi(request: web.Request):
-    retro_compatibility: bool = (
-        request.rel_url.query.get("retro-compatibility", "false") == "true"
-    )
+    retro_compatibility: bool = request.rel_url.query.get("retro-compatibility", "false") == "true"
 
     async with aiohttp.ClientSession() as session:
         result = {
diff --git a/tests/supervisor/test_resolvectl_dns_servers.py b/tests/supervisor/test_resolvectl_dns_servers.py
index 362cab70b..8d66e60fd 100644
--- a/tests/supervisor/test_resolvectl_dns_servers.py
+++ b/tests/supervisor/test_resolvectl_dns_servers.py
@@ -1,6 +1,7 @@
 # Avoid failures linked to nftables when initializing the global VmPool object
 import os
 from unittest import mock
+
 from aleph.vm.orchestrator.conf import (
     resolvectl_dns_servers,
     resolvectl_dns_servers_ipv4,

From 45a727d2b1c9b265eb5d7c4d9b393f0bedaf9fe9 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 17 Oct 2023 14:59:38 +0200
Subject: [PATCH 514/990] Fix: CI could not download Kubo

Use the opportunity to upgrade Kubo 21 -> 23
---
 packaging/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index abc851e2a..77bca7d20 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -43,7 +43,7 @@ vmlinux:
 
 download-ipfs-kubo: target-dir build-dir
 	mkdir -p ./target/kubo
-	curl -fsSL https://dist.ipfs.tech/kubo/v0.21.0/kubo_v0.21.0_linux-amd64.tar.gz | tar -xz --directory ./target/kubo
+	curl -fsSL https://github.com/ipfs/kubo/releases/download/v0.23.0/kubo_v0.23.0_linux-amd64.tar.gz | tar -xz --directory ./target/kubo
 
 version:
 	python3 ./version_from_git.py --inplace deb aleph-vm/DEBIAN/control

From 9e1d46d6d1038c92fbf892c3f5eec14c02439dbf Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 19 Oct 2023 16:29:29 +0200
Subject: [PATCH 515/990] Fix: Circular imports between controllers and
 orchestrator

Solution: Move code around
---
 src/aleph/vm/{orchestrator => }/conf.py       |  2 +-
 .../vm/controllers/firecracker/executable.py  | 10 +++++-----
 .../vm/controllers/firecracker/instance.py    | 19 ++++++++-----------
 .../vm/controllers/firecracker/program.py     |  8 ++++----
 .../firecracker}/snapshot_manager.py          |  5 +++--
 .../firecracker}/snapshots.py                 |  4 ++--
 .../firecracker/storage.py}                   |  0
 src/aleph/vm/network/__init__.py              |  0
 .../vm/{orchestrator => }/network/firewall.py |  2 +-
 .../{orchestrator => }/network/hostnetwork.py |  2 +-
 .../{orchestrator => }/network/interfaces.py  |  0
 .../{orchestrator => }/network/ipaddresses.py |  0
 .../{orchestrator => }/network/ndp_proxy.py   |  2 +-
 src/aleph/vm/orchestrator/__init__.py         |  6 ------
 src/aleph/vm/orchestrator/__main__.py         |  4 ++--
 src/aleph/vm/orchestrator/cli.py              |  2 +-
 src/aleph/vm/orchestrator/messages.py         |  2 +-
 src/aleph/vm/orchestrator/metrics.py          |  2 +-
 src/aleph/vm/orchestrator/migrations/env.py   |  2 +-
 .../0001_bbb12a12372e_execution_records.py    |  2 +-
 src/aleph/vm/orchestrator/models.py           |  8 ++++----
 src/aleph/vm/orchestrator/pool.py             |  7 ++++---
 src/aleph/vm/orchestrator/reactor.py          |  3 ++-
 src/aleph/vm/orchestrator/resources.py        |  2 +-
 src/aleph/vm/orchestrator/run.py              |  4 ++--
 src/aleph/vm/orchestrator/status.py           |  2 +-
 src/aleph/vm/orchestrator/supervisor.py       |  8 ++++----
 src/aleph/vm/orchestrator/tasks.py            |  5 +++--
 src/aleph/vm/orchestrator/views/__init__.py   |  6 +++---
 src/aleph/vm/{orchestrator => }/storage.py    | 16 +++-------------
 src/aleph/vm/{orchestrator => }/utils.py      | 10 ++++++++++
 tests/supervisor/test_ipv6_allocator.py       |  2 +-
 32 files changed, 71 insertions(+), 76 deletions(-)
 rename src/aleph/vm/{orchestrator => }/conf.py (99%)
 rename src/aleph/vm/{orchestrator => controllers/firecracker}/snapshot_manager.py (97%)
 rename src/aleph/vm/{orchestrator => controllers/firecracker}/snapshots.py (91%)
 rename src/aleph/vm/{orchestrator/network/__init__.py => controllers/firecracker/storage.py} (100%)
 create mode 100644 src/aleph/vm/network/__init__.py
 rename src/aleph/vm/{orchestrator => }/network/firewall.py (99%)
 rename src/aleph/vm/{orchestrator => }/network/hostnetwork.py (99%)
 rename src/aleph/vm/{orchestrator => }/network/interfaces.py (100%)
 rename src/aleph/vm/{orchestrator => }/network/ipaddresses.py (100%)
 rename src/aleph/vm/{orchestrator => }/network/ndp_proxy.py (97%)
 rename src/aleph/vm/{orchestrator => }/storage.py (97%)
 rename src/aleph/vm/{orchestrator => }/utils.py (94%)

diff --git a/src/aleph/vm/orchestrator/conf.py b/src/aleph/vm/conf.py
similarity index 99%
rename from src/aleph/vm/orchestrator/conf.py
rename to src/aleph/vm/conf.py
index 8d17c2ed0..a397c4e8e 100644
--- a/src/aleph/vm/orchestrator/conf.py
+++ b/src/aleph/vm/conf.py
@@ -11,7 +11,7 @@
 
 from pydantic import BaseSettings, Field
 
-from aleph.vm.orchestrator.utils import is_command_available
+from aleph.vm.utils import is_command_available
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index 4b7798d46..0b751434f 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -15,13 +15,13 @@
 from aleph_message.models import ExecutableContent, ItemHash
 from aleph_message.models.execution.environment import MachineResources
 
+from aleph.vm.conf import settings
+from aleph.vm.controllers.firecracker.snapshots import CompressedDiskVolumeSnapshot
 from aleph.vm.guest_api.__main__ import run_guest_api
 from aleph.vm.hypervisors.firecracker.microvm import FirecrackerConfig, MicroVM
-from aleph.vm.orchestrator.conf import settings
-from aleph.vm.orchestrator.network.firewall import teardown_nftables_for_vm
-from aleph.vm.orchestrator.network.interfaces import TapInterface
-from aleph.vm.orchestrator.snapshots import CompressedDiskVolumeSnapshot
-from aleph.vm.orchestrator.storage import get_volume_path
+from aleph.vm.network.firewall import teardown_nftables_for_vm
+from aleph.vm.network.interfaces import TapInterface
+from aleph.vm.storage import get_volume_path
 
 try:
     import psutil  # type: ignore [no-redef]
diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index 7f18d0585..179385127 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -10,6 +10,7 @@
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
 
+from aleph.vm.conf import settings
 from aleph.vm.hypervisors.firecracker.config import (
     BootSource,
     Drive,
@@ -19,26 +20,22 @@
     Vsock,
 )
 from aleph.vm.hypervisors.firecracker.microvm import setfacl
-from aleph.vm.orchestrator.conf import settings
-from aleph.vm.orchestrator.network.interfaces import TapInterface
-from aleph.vm.orchestrator.snapshots import (
-    CompressedDiskVolumeSnapshot,
-    DiskVolume,
-    DiskVolumeSnapshot,
-)
-from aleph.vm.orchestrator.storage import (
+from aleph.vm.network.interfaces import TapInterface
+from aleph.vm.utils import (
+    HostNotFoundError,
     NotEnoughDiskSpace,
     check_disk_space,
-    create_devmapper,
-    create_volume_file,
+    ping,
+    run_in_subprocess,
 )
-from aleph.vm.orchestrator.utils import HostNotFoundError, ping, run_in_subprocess
 
+from ...storage import create_devmapper, create_volume_file
 from .executable import (
     AlephFirecrackerExecutable,
     AlephFirecrackerResources,
     BaseConfiguration,
 )
+from .snapshots import CompressedDiskVolumeSnapshot, DiskVolume, DiskVolumeSnapshot
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/aleph/vm/controllers/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
index a1a9aeb37..b8590ca98 100644
--- a/src/aleph/vm/controllers/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -15,6 +15,7 @@
 from aleph_message.models.execution.base import Encoding
 from aleph_message.models.execution.environment import MachineResources
 
+from aleph.vm.conf import settings
 from aleph.vm.hypervisors.firecracker.config import (
     BootSource,
     Drive,
@@ -24,10 +25,9 @@
     Vsock,
 )
 from aleph.vm.hypervisors.firecracker.microvm import RuntimeConfiguration, setfacl
-from aleph.vm.orchestrator.conf import settings
-from aleph.vm.orchestrator.network.interfaces import TapInterface
-from aleph.vm.orchestrator.storage import get_code_path, get_data_path, get_runtime_path
-from aleph.vm.orchestrator.utils import MsgpackSerializable
+from aleph.vm.network.interfaces import TapInterface
+from aleph.vm.storage import get_code_path, get_data_path, get_runtime_path
+from aleph.vm.utils import MsgpackSerializable
 
 from .executable import (
     AlephFirecrackerExecutable,
diff --git a/src/aleph/vm/orchestrator/snapshot_manager.py b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
similarity index 97%
rename from src/aleph/vm/orchestrator/snapshot_manager.py
rename to src/aleph/vm/controllers/firecracker/snapshot_manager.py
index 6ad3a89a0..28d1e5e2e 100644
--- a/src/aleph/vm/orchestrator/snapshot_manager.py
+++ b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
@@ -7,8 +7,9 @@
 from aleph_message.models import ItemHash
 from schedule import Job, Scheduler
 
-from .conf import settings
-from .models import VmExecution
+from aleph.vm.conf import settings
+from aleph.vm.orchestrator.models import VmExecution
+
 from .snapshots import CompressedDiskVolumeSnapshot
 
 logger = logging.getLogger(__name__)
diff --git a/src/aleph/vm/orchestrator/snapshots.py b/src/aleph/vm/controllers/firecracker/snapshots.py
similarity index 91%
rename from src/aleph/vm/orchestrator/snapshots.py
rename to src/aleph/vm/controllers/firecracker/snapshots.py
index 699a151e1..f1bb24436 100644
--- a/src/aleph/vm/orchestrator/snapshots.py
+++ b/src/aleph/vm/controllers/firecracker/snapshots.py
@@ -4,8 +4,8 @@
 
 from aleph_message.models import ItemHash
 
-from .conf import SnapshotCompressionAlgorithm
-from .storage import compress_volume_snapshot, create_volume_snapshot
+from aleph.vm.conf import SnapshotCompressionAlgorithm
+from aleph.vm.storage import compress_volume_snapshot, create_volume_snapshot
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/aleph/vm/orchestrator/network/__init__.py b/src/aleph/vm/controllers/firecracker/storage.py
similarity index 100%
rename from src/aleph/vm/orchestrator/network/__init__.py
rename to src/aleph/vm/controllers/firecracker/storage.py
diff --git a/src/aleph/vm/network/__init__.py b/src/aleph/vm/network/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/aleph/vm/orchestrator/network/firewall.py b/src/aleph/vm/network/firewall.py
similarity index 99%
rename from src/aleph/vm/orchestrator/network/firewall.py
rename to src/aleph/vm/network/firewall.py
index 073dc9ec6..416f00d3c 100644
--- a/src/aleph/vm/orchestrator/network/firewall.py
+++ b/src/aleph/vm/network/firewall.py
@@ -4,7 +4,7 @@
 
 from nftables import Nftables
 
-from aleph.vm.orchestrator.conf import settings
+from aleph.vm.conf import settings
 
 from .interfaces import TapInterface
 
diff --git a/src/aleph/vm/orchestrator/network/hostnetwork.py b/src/aleph/vm/network/hostnetwork.py
similarity index 99%
rename from src/aleph/vm/orchestrator/network/hostnetwork.py
rename to src/aleph/vm/network/hostnetwork.py
index 520ae7e2c..7f21a0c57 100644
--- a/src/aleph/vm/orchestrator/network/hostnetwork.py
+++ b/src/aleph/vm/network/hostnetwork.py
@@ -5,7 +5,7 @@
 
 from aleph_message.models import ItemHash
 
-from aleph.vm.orchestrator.conf import IPv6AllocationPolicy
+from aleph.vm.conf import IPv6AllocationPolicy
 from aleph.vm.orchestrator.vm.vm_type import VmType
 
 from .firewall import initialize_nftables, setup_nftables_for_vm, teardown_nftables
diff --git a/src/aleph/vm/orchestrator/network/interfaces.py b/src/aleph/vm/network/interfaces.py
similarity index 100%
rename from src/aleph/vm/orchestrator/network/interfaces.py
rename to src/aleph/vm/network/interfaces.py
diff --git a/src/aleph/vm/orchestrator/network/ipaddresses.py b/src/aleph/vm/network/ipaddresses.py
similarity index 100%
rename from src/aleph/vm/orchestrator/network/ipaddresses.py
rename to src/aleph/vm/network/ipaddresses.py
diff --git a/src/aleph/vm/orchestrator/network/ndp_proxy.py b/src/aleph/vm/network/ndp_proxy.py
similarity index 97%
rename from src/aleph/vm/orchestrator/network/ndp_proxy.py
rename to src/aleph/vm/network/ndp_proxy.py
index d9f75ab2a..1cd6b29b4 100644
--- a/src/aleph/vm/orchestrator/network/ndp_proxy.py
+++ b/src/aleph/vm/network/ndp_proxy.py
@@ -14,7 +14,7 @@
 from ipaddress import IPv6Network
 from pathlib import Path
 
-from aleph.vm.orchestrator.utils import run_in_subprocess
+from aleph.vm.utils import run_in_subprocess
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/aleph/vm/orchestrator/__init__.py b/src/aleph/vm/orchestrator/__init__.py
index aa99dfdaf..61dd8436c 100644
--- a/src/aleph/vm/orchestrator/__init__.py
+++ b/src/aleph/vm/orchestrator/__init__.py
@@ -1,5 +1,4 @@
 from . import (
-    conf,
     messages,
     metrics,
     models,
@@ -9,10 +8,8 @@
     resources,
     run,
     status,
-    storage,
     supervisor,
     tasks,
-    utils,
     version,
     views,
     vm,
@@ -21,7 +18,6 @@
 __version__ = version.__version__
 
 __all__ = (
-    "conf",
     "messages",
     "metrics",
     "models",
@@ -31,10 +27,8 @@
     "resources",
     "run",
     "status",
-    "storage",
     "supervisor",
     "tasks",
-    "utils",
     "version",
     "views",
     "vm",
diff --git a/src/aleph/vm/orchestrator/__main__.py b/src/aleph/vm/orchestrator/__main__.py
index a4912bc99..9ae637f13 100644
--- a/src/aleph/vm/orchestrator/__main__.py
+++ b/src/aleph/vm/orchestrator/__main__.py
@@ -1,4 +1,4 @@
-from . import cli
+from .cli import main
 
 if __name__ == "__main__":
-    cli.main()
+    main()
diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index 85a952521..fcc6f2b23 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -20,8 +20,8 @@
 import alembic.config
 from aleph_message.models import ItemHash
 
+from ..conf import ALLOW_DEVELOPER_SSH_KEYS, make_db_url, settings
 from . import metrics, supervisor
-from .conf import ALLOW_DEVELOPER_SSH_KEYS, make_db_url, settings
 from .pubsub import PubSub
 from .run import run_code_on_event, run_code_on_request, start_persistent_vm
 
diff --git a/src/aleph/vm/orchestrator/messages.py b/src/aleph/vm/orchestrator/messages.py
index 07c14f0fd..a00e22310 100644
--- a/src/aleph/vm/orchestrator/messages.py
+++ b/src/aleph/vm/orchestrator/messages.py
@@ -5,7 +5,7 @@
 from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable
 from aleph_message.models import ExecutableMessage, ItemHash, MessageType
 
-from .storage import get_latest_amend, get_message
+from aleph.vm.storage import get_latest_amend, get_message
 
 
 async def try_get_message(ref: str) -> ExecutableMessage:
diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index b0af25e3b..125ff90b4 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -14,7 +14,7 @@
     from sqlalchemy.ext.declarative import declarative_base
 
 
-from .conf import make_db_url, settings
+from ..conf import make_db_url, settings
 
 Session: sessionmaker
 
diff --git a/src/aleph/vm/orchestrator/migrations/env.py b/src/aleph/vm/orchestrator/migrations/env.py
index c7fbe5004..2cf116bb6 100644
--- a/src/aleph/vm/orchestrator/migrations/env.py
+++ b/src/aleph/vm/orchestrator/migrations/env.py
@@ -1,7 +1,7 @@
 from alembic import context
 from sqlalchemy import create_engine
 
-from aleph.vm.orchestrator.conf import make_db_url
+from aleph.vm.conf import make_db_url
 
 # Auto-generate migrations
 from aleph.vm.orchestrator.metrics import Base
diff --git a/src/aleph/vm/orchestrator/migrations/versions/0001_bbb12a12372e_execution_records.py b/src/aleph/vm/orchestrator/migrations/versions/0001_bbb12a12372e_execution_records.py
index 2ee5a3efd..84e2011e1 100644
--- a/src/aleph/vm/orchestrator/migrations/versions/0001_bbb12a12372e_execution_records.py
+++ b/src/aleph/vm/orchestrator/migrations/versions/0001_bbb12a12372e_execution_records.py
@@ -12,7 +12,7 @@
 from sqlalchemy import create_engine
 from sqlalchemy.engine import reflection
 
-from aleph.vm.orchestrator.conf import make_db_url
+from aleph.vm.conf import make_db_url
 
 revision = "bbb12a12372e"
 down_revision = None
diff --git a/src/aleph/vm/orchestrator/models.py b/src/aleph/vm/orchestrator/models.py
index 4a0cf2578..958847a0c 100644
--- a/src/aleph/vm/orchestrator/models.py
+++ b/src/aleph/vm/orchestrator/models.py
@@ -21,16 +21,16 @@
     AlephFirecrackerResources,
     AlephProgramResources,
 )
+from aleph.vm.network.interfaces import TapInterface
+from aleph.vm.utils import create_task_log_exceptions, dumps_for_json
 
-from .conf import settings
+from ..conf import settings
 from .metrics import ExecutionRecord, save_execution_data, save_record
-from .network.interfaces import TapInterface
 from .pubsub import PubSub
-from .utils import create_task_log_exceptions, dumps_for_json
 from .vm import AlephFirecrackerInstance
 
 if TYPE_CHECKING:
-    from .snapshot_manager import SnapshotManager
+    from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
 
 
 logger = logging.getLogger(__name__)
diff --git a/src/aleph/vm/orchestrator/pool.py b/src/aleph/vm/orchestrator/pool.py
index a3cccfeb1..ba16f67f8 100644
--- a/src/aleph/vm/orchestrator/pool.py
+++ b/src/aleph/vm/orchestrator/pool.py
@@ -6,10 +6,11 @@
 from aleph_message.models import ExecutableMessage, ItemHash
 from aleph_message.models.execution.instance import InstanceContent
 
-from .conf import settings
+from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
+
+from ..conf import settings
+from ..network.hostnetwork import Network, make_ipv6_allocator
 from .models import ExecutableContent, VmExecution
-from .network.hostnetwork import Network, make_ipv6_allocator
-from .snapshot_manager import SnapshotManager
 from .vm.vm_type import VmType
 
 logger = logging.getLogger(__name__)
diff --git a/src/aleph/vm/orchestrator/reactor.py b/src/aleph/vm/orchestrator/reactor.py
index e5c96a583..7f3c72a0b 100644
--- a/src/aleph/vm/orchestrator/reactor.py
+++ b/src/aleph/vm/orchestrator/reactor.py
@@ -4,9 +4,10 @@
 from aleph_message.models import AlephMessage
 from aleph_message.models.execution.environment import Subscription
 
+from aleph.vm.utils import create_task_log_exceptions
+
 from .pubsub import PubSub
 from .run import run_code_on_event
-from .utils import create_task_log_exceptions
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index e1640dc3c..7f5fdbdf6 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -9,7 +9,7 @@
 from aleph_message.models.execution.environment import CpuProperties
 from pydantic import BaseModel, Field
 
-from .conf import settings
+from ..conf import settings
 
 
 class Period(BaseModel):
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index 110d81300..a92409d4c 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -15,13 +15,13 @@
     VmSetupError,
 )
 from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInit
+from aleph.vm.utils import HostNotFoundError
 
-from .conf import settings
+from ..conf import settings
 from .messages import load_updated_message
 from .models import VmExecution
 from .pool import VmPool
 from .pubsub import PubSub
-from .utils import HostNotFoundError
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py
index bfdd95472..4c4a332c6 100644
--- a/src/aleph/vm/orchestrator/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -7,7 +7,7 @@
 
 from aiohttp import ClientResponseError, ClientSession
 
-from .conf import settings
+from ..conf import settings
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 1b68e8ab8..16bb75a78 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -12,8 +12,8 @@
 
 from aiohttp import web
 
-from . import metrics
-from .conf import settings
+from ..conf import settings
+from .metrics import create_tables, setup_engine
 from .resources import about_system_usage
 from .run import pool
 from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task
@@ -85,8 +85,8 @@ def run():
     app["secret_token"] = secret_token
     print(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
 
-    engine = metrics.setup_engine()
-    metrics.create_tables(engine)
+    engine = setup_engine()
+    create_tables(engine)
 
     try:
         if settings.WATCH_FOR_MESSAGES:
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 8adabcf3c..49f787d2c 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -12,11 +12,12 @@
 from aleph_message.models import AlephMessage, ItemHash, ProgramMessage, parse_message
 from yarl import URL
 
-from .conf import settings
+from aleph.vm.utils import create_task_log_exceptions
+
+from ..conf import settings
 from .messages import load_updated_message
 from .pubsub import PubSub
 from .reactor import Reactor
-from .utils import create_task_log_exceptions
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index dbd96ffeb..f1ac5882b 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -14,6 +14,7 @@
 from aleph_message.models import ItemHash
 from pydantic import ValidationError
 
+from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.executable import (
     ResourceDownloadError,
     VmSetupError,
@@ -21,18 +22,17 @@
 from aleph.vm.controllers.firecracker.program import FileTooLargeError
 from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInit
 from aleph.vm.orchestrator import status
-from aleph.vm.orchestrator.conf import settings
 from aleph.vm.orchestrator.metrics import get_execution_records
 from aleph.vm.orchestrator.pubsub import PubSub
 from aleph.vm.orchestrator.resources import Allocation
 from aleph.vm.orchestrator.run import pool, run_code_on_request, start_persistent_vm
-from aleph.vm.orchestrator.utils import (
+from aleph.vm.orchestrator.version import __version__
+from aleph.vm.utils import (
     HostNotFoundError,
     b32_to_b16,
     dumps_for_json,
     get_ref_from_dns,
 )
-from aleph.vm.orchestrator.version import __version__
 from packaging.version import InvalidVersion, Version
 
 logger = logging.getLogger(__name__)
diff --git a/src/aleph/vm/orchestrator/storage.py b/src/aleph/vm/storage.py
similarity index 97%
rename from src/aleph/vm/orchestrator/storage.py
rename to src/aleph/vm/storage.py
index eeb6d3666..655bb2d82 100644
--- a/src/aleph/vm/orchestrator/storage.py
+++ b/src/aleph/vm/storage.py
@@ -10,7 +10,7 @@
 import sys
 from datetime import datetime
 from pathlib import Path
-from shutil import copy2, disk_usage, make_archive
+from shutil import copy2, make_archive
 from typing import Union
 
 import aiohttp
@@ -29,18 +29,13 @@
     VolumePersistence,
 )
 
-from .conf import SnapshotCompressionAlgorithm, settings
-from .utils import fix_message_validation, run_in_subprocess
+from aleph.vm.conf import SnapshotCompressionAlgorithm, settings
+from aleph.vm.utils import fix_message_validation, run_in_subprocess
 
 logger = logging.getLogger(__name__)
-
 DEVICE_MAPPER_DIRECTORY = "/dev/mapper"
 
 
-class NotEnoughDiskSpace(OSError):
-    pass
-
-
 async def chown_to_jailman(path: Path) -> None:
     """Changes ownership of the target when running firecracker inside jailer isolation."""
     if not path.exists():
@@ -353,8 +348,3 @@ async def compress_volume_snapshot(
     )
 
     return new_path
-
-
-def check_disk_space(bytes_to_use: int) -> bool:
-    host_disk_usage = disk_usage("/")
-    return host_disk_usage.free >= bytes_to_use
diff --git a/src/aleph/vm/orchestrator/utils.py b/src/aleph/vm/utils.py
similarity index 94%
rename from src/aleph/vm/orchestrator/utils.py
rename to src/aleph/vm/utils.py
index 72554a03a..f0858ea4e 100644
--- a/src/aleph/vm/orchestrator/utils.py
+++ b/src/aleph/vm/utils.py
@@ -8,6 +8,7 @@
 from collections.abc import Coroutine
 from dataclasses import asdict as dataclass_as_dict
 from dataclasses import is_dataclass
+from shutil import disk_usage
 from typing import Any, Optional
 
 import aiodns
@@ -126,3 +127,12 @@ async def ping(host: str, packets: int, timeout: float):
         await run_in_subprocess(["ping", "-c", str(packets), "-W", str(timeout), host], check=True)
     except subprocess.CalledProcessError as err:
         raise HostNotFoundError() from err
+
+
+def check_disk_space(bytes_to_use: int) -> bool:
+    host_disk_usage = disk_usage("/")
+    return host_disk_usage.free >= bytes_to_use
+
+
+class NotEnoughDiskSpace(OSError):
+    pass
diff --git a/tests/supervisor/test_ipv6_allocator.py b/tests/supervisor/test_ipv6_allocator.py
index c9cf264cc..bdf8d2a7b 100644
--- a/tests/supervisor/test_ipv6_allocator.py
+++ b/tests/supervisor/test_ipv6_allocator.py
@@ -1,6 +1,6 @@
 import os
 
-from aleph.vm.orchestrator.network.hostnetwork import StaticIPv6Allocator
+from aleph.vm.network.hostnetwork import StaticIPv6Allocator
 from aleph.vm.orchestrator.vm.vm_type import VmType
 
 # Avoid failures linked to settings when initializing the global VmPool object

From eb7da3bd1730f3d8b9bebf09a21fb9ffc35e1d4c Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 19 Oct 2023 16:48:49 +0200
Subject: [PATCH 516/990] Fix: Added default frequency ensuring is an int
 always.

---
 src/aleph/vm/controllers/firecracker/snapshot_manager.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/snapshot_manager.py b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
index 28d1e5e2e..8ab3a4d6c 100644
--- a/src/aleph/vm/controllers/firecracker/snapshot_manager.py
+++ b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
@@ -100,15 +100,14 @@ async def start_for(self, execution: VmExecution, frequency: Optional[int] = Non
             msg = "Snapshots are not implemented for programs."
             raise NotImplementedError(msg)
 
-        if not frequency:
-            frequency = settings.SNAPSHOT_FREQUENCY
+        default_frequency = frequency or settings.SNAPSHOT_FREQUENCY
 
         vm_hash = execution.vm_hash
         snapshot_execution = SnapshotExecution(
             scheduler=self._scheduler,
             vm_hash=vm_hash,
             execution=execution,
-            frequency=frequency,
+            frequency=default_frequency,
         )
         self.executions[vm_hash] = snapshot_execution
         await snapshot_execution.start()

From eac6b2b47cb643f4ab17cbb2f16dc2cc5e545cfb Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Mon, 23 Oct 2023 15:28:59 +0200
Subject: [PATCH 517/990] Fix: Fixed pyproject file to support Python 3.9.

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 4367dac9f..e225f6714 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,7 +7,7 @@ name = "aleph-vm"
 dynamic = ["version"]
 description = "Aleph.im VM execution engine"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.9"
 license = "MIT"
 keywords = []
 authors = [
@@ -93,7 +93,7 @@ cov = [
 ]
 
 [[tool.hatch.envs.all.matrix]]
-python = ["3.10", "3.11", "3.12"]
+python = ["3.9", "3.10", "3.11", "3.12"]
 
 [tool.hatch.envs.lint]
 detached = true

From 85089853c5bf35c1b0e9225cece119992228d2eb Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Mon, 23 Oct 2023 15:58:41 +0200
Subject: [PATCH 518/990] Problem: VMPool and SnapshotManager class are coupled
 to orchestrator path.

Solution: Refactor VMPool and SnapshotManager classes to decouple it.
---
 .../firecracker/snapshot_manager.py           | 32 +++++++++----------
 src/aleph/vm/{orchestrator => }/models.py     | 13 +++++---
 src/aleph/vm/orchestrator/__init__.py         |  4 ---
 src/aleph/vm/orchestrator/run.py              |  4 +--
 src/aleph/vm/orchestrator/views/operator.py   |  2 +-
 src/aleph/vm/{orchestrator => }/pool.py       |  8 ++---
 6 files changed, 31 insertions(+), 32 deletions(-)
 rename src/aleph/vm/{orchestrator => }/models.py (97%)
 rename src/aleph/vm/{orchestrator => }/pool.py (96%)

diff --git a/src/aleph/vm/controllers/firecracker/snapshot_manager.py b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
index 8ab3a4d6c..1b5eac64e 100644
--- a/src/aleph/vm/controllers/firecracker/snapshot_manager.py
+++ b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
@@ -8,31 +8,31 @@
 from schedule import Job, Scheduler
 
 from aleph.vm.conf import settings
-from aleph.vm.orchestrator.models import VmExecution
 
+from .executable import AlephFirecrackerExecutable
 from .snapshots import CompressedDiskVolumeSnapshot
 
 logger = logging.getLogger(__name__)
 
 
-def wrap_async_snapshot(execution):
-    asyncio.run(do_execution_snapshot(execution))
+def wrap_async_snapshot(vm):
+    asyncio.run(do_vm_snapshot(vm))
 
 
-def run_threaded_snapshot(execution):
-    job_thread = threading.Thread(target=wrap_async_snapshot, args=(execution,))
+def run_threaded_snapshot(vm):
+    job_thread = threading.Thread(target=wrap_async_snapshot, args=(vm,))
     job_thread.start()
 
 
-async def do_execution_snapshot(execution: VmExecution) -> CompressedDiskVolumeSnapshot:
+async def do_vm_snapshot(vm: AlephFirecrackerExecutable) -> CompressedDiskVolumeSnapshot:
     try:
-        logger.debug(f"Starting new snapshot for VM {execution.vm_hash}")
-        assert execution.vm, "VM execution not set"
+        logger.debug(f"Starting new snapshot for VM {vm.vm_hash}")
+        assert vm, "VM execution not set"
 
-        snapshot = await execution.vm.create_snapshot()
+        snapshot = await vm.create_snapshot()
         await snapshot.upload()
 
-        logger.debug(f"New snapshots for VM {execution.vm_hash} created in {snapshot.path}")
+        logger.debug(f"New snapshots for VM {vm.vm_hash} created in {snapshot.path}")
         return snapshot
     except ValueError:
         msg = "Something failed taking an snapshot"
@@ -47,7 +47,7 @@ def infinite_run_scheduler_jobs(scheduler: Scheduler) -> None:
 
 class SnapshotExecution:
     vm_hash: ItemHash
-    execution: VmExecution
+    execution: AlephFirecrackerExecutable
     frequency: int
     _scheduler: Scheduler
     _job: Job
@@ -56,7 +56,7 @@ def __init__(
         self,
         scheduler: Scheduler,
         vm_hash: ItemHash,
-        execution: VmExecution,
+        execution: AlephFirecrackerExecutable,
         frequency: int,
     ):
         self.vm_hash = vm_hash
@@ -95,18 +95,18 @@ def run_snapshots(self) -> None:
         )
         job_thread.start()
 
-    async def start_for(self, execution: VmExecution, frequency: Optional[int] = None) -> None:
-        if not execution.is_instance:
+    async def start_for(self, vm: AlephFirecrackerExecutable, frequency: Optional[int] = None) -> None:
+        if not vm.is_instance:
             msg = "Snapshots are not implemented for programs."
             raise NotImplementedError(msg)
 
         default_frequency = frequency or settings.SNAPSHOT_FREQUENCY
 
-        vm_hash = execution.vm_hash
+        vm_hash = vm.vm_hash
         snapshot_execution = SnapshotExecution(
             scheduler=self._scheduler,
             vm_hash=vm_hash,
-            execution=execution,
+            execution=vm,
             frequency=default_frequency,
         )
         self.executions[vm_hash] = snapshot_execution
diff --git a/src/aleph/vm/orchestrator/models.py b/src/aleph/vm/models.py
similarity index 97%
rename from src/aleph/vm/orchestrator/models.py
rename to src/aleph/vm/models.py
index 958847a0c..a025e10fd 100644
--- a/src/aleph/vm/orchestrator/models.py
+++ b/src/aleph/vm/models.py
@@ -14,6 +14,7 @@
     ProgramContent,
 )
 
+from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.executable import AlephFirecrackerExecutable
 from aleph.vm.controllers.firecracker.instance import AlephInstanceResources
 from aleph.vm.controllers.firecracker.program import (
@@ -22,13 +23,15 @@
     AlephProgramResources,
 )
 from aleph.vm.network.interfaces import TapInterface
+from aleph.vm.orchestrator.metrics import (
+    ExecutionRecord,
+    save_execution_data,
+    save_record,
+)
+from aleph.vm.orchestrator.pubsub import PubSub
+from aleph.vm.orchestrator.vm import AlephFirecrackerInstance
 from aleph.vm.utils import create_task_log_exceptions, dumps_for_json
 
-from ..conf import settings
-from .metrics import ExecutionRecord, save_execution_data, save_record
-from .pubsub import PubSub
-from .vm import AlephFirecrackerInstance
-
 if TYPE_CHECKING:
     from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
 
diff --git a/src/aleph/vm/orchestrator/__init__.py b/src/aleph/vm/orchestrator/__init__.py
index 61dd8436c..8f783cf65 100644
--- a/src/aleph/vm/orchestrator/__init__.py
+++ b/src/aleph/vm/orchestrator/__init__.py
@@ -1,8 +1,6 @@
 from . import (
     messages,
     metrics,
-    models,
-    pool,
     pubsub,
     reactor,
     resources,
@@ -20,8 +18,6 @@
 __all__ = (
     "messages",
     "metrics",
-    "models",
-    "pool",
     "pubsub",
     "reactor",
     "resources",
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index a92409d4c..37f28164a 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -18,9 +18,9 @@
 from aleph.vm.utils import HostNotFoundError
 
 from ..conf import settings
+from ..models import VmExecution
+from ..pool import VmPool
 from .messages import load_updated_message
-from .models import VmExecution
-from .pool import VmPool
 from .pubsub import PubSub
 
 logger = logging.getLogger(__name__)
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index f25b20667..c2e9b08fd 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -15,7 +15,7 @@
 from eth_account.messages import encode_defunct
 from jwskate import Jwk
 
-from aleph.vm.orchestrator.models import VmExecution
+from aleph.vm.models import VmExecution
 from aleph.vm.orchestrator.run import pool
 
 logger = logging.getLogger(__name__)
diff --git a/src/aleph/vm/orchestrator/pool.py b/src/aleph/vm/pool.py
similarity index 96%
rename from src/aleph/vm/orchestrator/pool.py
rename to src/aleph/vm/pool.py
index ba16f67f8..93346b3db 100644
--- a/src/aleph/vm/orchestrator/pool.py
+++ b/src/aleph/vm/pool.py
@@ -6,12 +6,12 @@
 from aleph_message.models import ExecutableMessage, ItemHash
 from aleph_message.models.execution.instance import InstanceContent
 
+from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
+from aleph.vm.network.hostnetwork import Network, make_ipv6_allocator
+from aleph.vm.orchestrator.vm.vm_type import VmType
 
-from ..conf import settings
-from ..network.hostnetwork import Network, make_ipv6_allocator
 from .models import ExecutableContent, VmExecution
-from .vm.vm_type import VmType
 
 logger = logging.getLogger(__name__)
 
@@ -79,7 +79,7 @@ async def create_a_vm(
 
         # Start VM snapshots automatically
         if isinstance(message, InstanceContent):
-            await self.snapshot_manager.start_for(execution=execution)
+            await self.snapshot_manager.start_for(vm=execution.vm)
 
         return execution
 

From 7bd2a8b2684eff439e9d5c381c49301c55929f74 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 23 Oct 2023 16:58:27 +0200
Subject: [PATCH 519/990] Doc fix typo in Readme.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e6226d8b0..f57d3c2d9 100644
--- a/README.md
+++ b/README.md
@@ -26,7 +26,7 @@ For development and testing, install Aleph-VM from source.
 
 1. Install the [VM-Connector](./vm_connector/README.md)
 2. Install the [VM-Supervisor](src/aleph/vm/orchestrator/README.md).
-3. Install and [configure a reverse-proxy such as [Caddy](./CONFIGURE_CADDY.md)
+3. Install and configure a reverse-proxy such as [Caddy](./CONFIGURE_CADDY.md)
 
 ## 3. Create and run an Aleph Program 
 

From 93af1e369b422565e9c5b434ac9104f6983d4f4f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 24 Oct 2023 11:53:03 +0200
Subject: [PATCH 520/990] Fix: Initializing VM Pool had side effects

Problem: Initializing the VM Pool had the side effect of setting up the network interfaces.

Solution: Move the setup of the networking in a separate method of the Network class, that has to be called explicitly.
---
 src/aleph/vm/network/hostnetwork.py     | 36 +++++++++++++++++++------
 src/aleph/vm/orchestrator/run.py        |  1 +
 src/aleph/vm/orchestrator/supervisor.py |  2 +-
 src/aleph/vm/pool.py                    | 10 +++++++
 4 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/aleph/vm/network/hostnetwork.py b/src/aleph/vm/network/hostnetwork.py
index 7f21a0c57..22f76acf7 100644
--- a/src/aleph/vm/network/hostnetwork.py
+++ b/src/aleph/vm/network/hostnetwork.py
@@ -107,10 +107,13 @@ def make_ipv6_allocator(
 class Network:
     ipv4_forward_state_before_setup: Optional[int] = None
     ipv6_forward_state_before_setup: Optional[int] = None
+    external_interface: str
+    ipv4_forwarding_enabled: bool
+    ipv6_forwarding_enabled: bool
+    use_ndp_proxy: bool
     ipv4_address_pool: IPv4NetworkWithInterfaces = IPv4NetworkWithInterfaces("172.16.0.0/12")
     ipv6_address_pool: IPv6Network
     network_size: int
-    external_interface: str
     ndp_proxy: Optional[NdpProxy] = None
 
     IPV6_SUBNET_PREFIX: int = 124
@@ -122,25 +125,42 @@ def __init__(
         external_interface: str,
         ipv6_allocator: IPv6Allocator,
         use_ndp_proxy: bool,
+        ipv4_forwarding_enabled: bool = True,
         ipv6_forwarding_enabled: bool = True,
     ) -> None:
-        """Sets up the Network class with some information it needs so future function calls work as expected"""
+        """Initialize the Network class with the relevant configuration."""
         self.ipv4_address_pool = IPv4NetworkWithInterfaces(vm_ipv4_address_pool_range)
-        if not self.ipv4_address_pool.is_private:
-            logger.warning(f"Using a network range that is not private: {self.ipv4_address_pool}")
         self.ipv6_allocator = ipv6_allocator
 
         self.network_size = vm_network_size
         self.external_interface = external_interface
+        self.ipv4_forwarding_enabled = ipv4_forwarding_enabled
+        self.ipv6_forwarding_enabled = ipv6_forwarding_enabled
+        self.use_ndp_proxy = use_ndp_proxy
 
-        self.enable_ipv4_forwarding()
-        if ipv6_forwarding_enabled:
+        if not self.ipv4_address_pool.is_private:
+            logger.warning(f"Using a network range that is not private: {self.ipv4_address_pool}")
+
+    def setup(self) -> None:
+        """Set up the network for use by the VMs"""
+        logger.debug("Enabling IPv4 forwarding")
+        if self.ipv4_forwarding_enabled:
+            self.enable_ipv4_forwarding()
+        else:
+            logger.warning("IPv4 forwarding is disabled, VMs will not have internet access on IPv4")
+        logger.debug("Enabling IPv6 forwarding")
+        if self.ipv6_forwarding_enabled:
             self.enable_ipv6_forwarding()
+        else:
+            logger.warning("IPv6 forwarding is disabled, VMs will not have internet access on IPv6")
 
-        if use_ndp_proxy:
-            self.ndp_proxy = NdpProxy(host_network_interface=external_interface)
+        logger.debug("Enabling NDP proxy")
+        if self.use_ndp_proxy:
+            self.ndp_proxy = NdpProxy(host_network_interface=self.external_interface)
 
+        logger.debug("Initializing nftables")
         initialize_nftables()
+        logger.debug("Network setup complete")
 
     def get_network_for_tap(self, vm_id: int) -> IPv4NetworkWithInterfaces:
         subnets = list(self.ipv4_address_pool.subnets(new_prefix=self.network_size))
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index 37f28164a..02fe112b0 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -26,6 +26,7 @@
 logger = logging.getLogger(__name__)
 
 pool = VmPool()
+pool.setup()
 
 
 async def build_asgi_scope(path: str, request: web.Request) -> dict[str, Any]:
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 16bb75a78..ca86d3dec 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -105,4 +105,4 @@ def run():
             raise
     finally:
         if settings.ALLOW_VM_NETWORKING:
-            pool.network.teardown()
+            pool.teardown()
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 93346b3db..cba891c7f 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -55,6 +55,16 @@ def __init__(self):
         logger.debug("Initializing SnapshotManager ...")
         self.snapshot_manager.run_snapshots()
 
+    def setup(self) -> None:
+        """Set up the VM pool and the network."""
+        if self.network:
+            self.network.setup()
+
+    def teardown(self) -> None:
+        """Stop the VM pool and the network properly."""
+        if self.network:
+            self.network.teardown()
+
     async def create_a_vm(
         self, vm_hash: ItemHash, message: ExecutableContent, original: ExecutableContent
     ) -> VmExecution:

From 90640099742c2600202cb59b978da36e93cb1878 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 25 Oct 2023 13:40:25 +0200
Subject: [PATCH 521/990] Fix: Dates did not use UTC (#449)

Problem: The timezone of the system was often used for dates. This can cause inconsistencies and should be avoided in favour of UTC.
---
 src/aleph/vm/models.py                      | 16 ++++++++--------
 src/aleph/vm/orchestrator/views/operator.py |  6 +++---
 src/aleph/vm/storage.py                     |  4 ++--
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index a025e10fd..9a92b6300 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -4,7 +4,7 @@
 import uuid
 from asyncio import Task
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Optional, Union
 
 from aleph_message.models import (
@@ -109,7 +109,7 @@ def __init__(
         self.vm_hash = vm_hash
         self.message = message
         self.original = original
-        self.times = VmExecutionTimes(defined_at=datetime.now())
+        self.times = VmExecutionTimes(defined_at=datetime.now(tz=timezone.utc))
         self.ready_event = asyncio.Event()
         self.concurrent_runs = 0
         self.runs_done_event = asyncio.Event()
@@ -127,7 +127,7 @@ def to_json(self, indent: Optional[int] = None) -> str:
 
     async def prepare(self):
         """Download VM required files"""
-        self.times.preparing_at = datetime.now()
+        self.times.preparing_at = datetime.now(tz=timezone.utc)
         if self.is_program:
             resources = AlephProgramResources(self.message, namespace=self.vm_hash)
         elif self.is_instance:
@@ -136,14 +136,14 @@ async def prepare(self):
             msg = "Unknown executable message type"
             raise ValueError(msg)
         await resources.download_all()
-        self.times.prepared_at = datetime.now()
+        self.times.prepared_at = datetime.now(tz=timezone.utc)
         self.resources = resources
 
     async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None) -> AlephFirecrackerExecutable:
         if not self.resources:
             msg = "Execution resources must be configured first"
             raise ValueError(msg)
-        self.times.starting_at = datetime.now()
+        self.times.starting_at = datetime.now(tz=timezone.utc)
 
         vm: Union[AlephFirecrackerProgram, AlephFirecrackerInstance]
         if self.is_program:
@@ -172,7 +172,7 @@ async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None)
             await vm.start()
             await vm.configure()
             await vm.start_guest_api()
-            self.times.started_at = datetime.now()
+            self.times.started_at = datetime.now(tz=timezone.utc)
             self.ready_event.set()
             return vm
         except Exception:
@@ -227,10 +227,10 @@ async def stop(self):
                 logger.debug(f"VM={self.vm.vm_id} already stopped")
                 return
             await self.all_runs_complete()
-            self.times.stopping_at = datetime.now()
+            self.times.stopping_at = datetime.now(tz=timezone.utc)
             await self.record_usage()
             await self.vm.teardown()
-            self.times.stopped_at = datetime.now()
+            self.times.stopped_at = datetime.now(tz=timezone.utc)
             self.cancel_expiration()
             self.cancel_update()
 
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index c2e9b08fd..7aeef84e6 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -3,7 +3,7 @@
 import json
 import logging
 from collections.abc import Awaitable
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from typing import Callable
 
 import aiohttp.web_exceptions
@@ -26,8 +26,8 @@ def is_token_still_valid(timestamp):
     Checks if a token has exprired based on its timestamp
     """
     timestamp = int(timestamp)
-    current_datetime = datetime.now()
-    target_datetime = datetime.fromtimestamp(timestamp)
+    current_datetime = datetime.now(tz=timezone.utc)
+    target_datetime = datetime.fromtimestamp(timestamp, tz=timezone.utc)
 
     return target_datetime > current_datetime
 
diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index 655bb2d82..9dd9ef380 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -8,7 +8,7 @@
 import logging
 import re
 import sys
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from shutil import copy2, make_archive
 from typing import Union
@@ -326,7 +326,7 @@ async def get_volume_path(volume: MachineVolume, namespace: str) -> Path:
 
 
 async def create_volume_snapshot(path: Path) -> Path:
-    new_path = Path(f"{path}.{datetime.today().strftime('%d%m%Y-%H%M%S')}.bak")
+    new_path = Path(f"{path}.{datetime.now(tz=timezone.utc).date().strftime('%d%m%Y-%H%M%S')}.bak")
     copy2(path, new_path)
     return new_path
 

From 5130793723e310216c2b9f9799f4aad5866f3880 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 25 Oct 2023 13:41:15 +0200
Subject: [PATCH 522/990] Attach loop singleton to aiohttp `app` singleton.
 (#448)

* Fix: Pool singleton caused issues in tests

Problem: When importing arbitrary parts of the software, the definition of the VmPool singleton was too "magic" and network setup automatically created issues.

Solution: Create the Pool singleton explicitly when starting the supervisor, and store it on the aiohttp `app` singleton.

* Fix: Don't create a dummy object when could be Optional
---
 src/aleph/vm/orchestrator/cli.py            | 15 ++++++----
 src/aleph/vm/orchestrator/reactor.py        |  7 +++--
 src/aleph/vm/orchestrator/run.py            | 27 ++++++++---------
 src/aleph/vm/orchestrator/supervisor.py     |  8 ++++-
 src/aleph/vm/orchestrator/tasks.py          |  4 ++-
 src/aleph/vm/orchestrator/views/__init__.py | 33 +++++++++++----------
 src/aleph/vm/orchestrator/views/operator.py | 21 ++++++++-----
 7 files changed, 67 insertions(+), 48 deletions(-)

diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index fcc6f2b23..190dcc7c1 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -21,6 +21,7 @@
 from aleph_message.models import ItemHash
 
 from ..conf import ALLOW_DEVELOPER_SSH_KEYS, make_db_url, settings
+from ..pool import VmPool
 from . import metrics, supervisor
 from .pubsub import PubSub
 from .run import run_code_on_event, run_code_on_request, start_persistent_vm
@@ -189,6 +190,9 @@ async def fake_read() -> bytes:
 
     bench: list[float] = []
 
+    pool = VmPool()
+    pool.setup()
+
     # Does not make sense in benchmarks
     settings.WATCH_FOR_MESSAGES = False
     settings.WATCH_FOR_UPDATES = False
@@ -207,7 +211,7 @@ async def fake_read() -> bytes:
         "/cache/keys",
     ):
         fake_request.match_info["suffix"] = path
-        response: Response = await run_code_on_request(vm_hash=ref, path=path, request=fake_request)
+        response: Response = await run_code_on_request(vm_hash=ref, path=path, pool=pool, request=fake_request)
         assert response.status == 200
 
     # Disable VM timeout to exit benchmark properly
@@ -216,7 +220,7 @@ async def fake_read() -> bytes:
     for _run in range(runs):
         t0 = time.time()
         fake_request.match_info["suffix"] = path
-        response2: Response = await run_code_on_request(vm_hash=ref, path=path, request=fake_request)
+        response2: Response = await run_code_on_request(vm_hash=ref, path=path, pool=pool, request=fake_request)
         assert response2.status == 200
         bench.append(time.time() - t0)
 
@@ -224,19 +228,20 @@ async def fake_read() -> bytes:
     logger.info(bench)
 
     event = None
-    result = await run_code_on_event(vm_hash=ref, event=event, pubsub=PubSub())
+    result = await run_code_on_event(vm_hash=ref, event=event, pubsub=PubSub(), pool=pool)
     print("Event result", result)
 
 
 async def start_instance(item_hash: ItemHash) -> None:
     """Run an instance from an InstanceMessage."""
+    pool = VmPool()
 
     # The main program uses a singleton pubsub instance in order to watch for updates.
     # We create another instance here since that singleton is not initialized yet.
     # Watching for updates on this instance will therefore not work.
-    dummy_pubsub = PubSub()
+    pubsub = None
 
-    await start_persistent_vm(item_hash, dummy_pubsub)
+    await start_persistent_vm(item_hash, pubsub, pool)
 
 
 async def run_instances(instances: list[ItemHash]) -> None:
diff --git a/src/aleph/vm/orchestrator/reactor.py b/src/aleph/vm/orchestrator/reactor.py
index 7f3c72a0b..4c2ec0002 100644
--- a/src/aleph/vm/orchestrator/reactor.py
+++ b/src/aleph/vm/orchestrator/reactor.py
@@ -6,6 +6,7 @@
 
 from aleph.vm.utils import create_task_log_exceptions
 
+from ..pool import VmPool
 from .pubsub import PubSub
 from .run import run_code_on_event
 
@@ -39,10 +40,12 @@ def subscription_matches(subscription: Subscription, message: AlephMessage) -> b
 
 class Reactor:
     pubsub: PubSub
+    pool: VmPool
     listeners: list[AlephMessage]
 
-    def __init__(self, pubsub: PubSub):
+    def __init__(self, pubsub: PubSub, pool: VmPool):
         self.pubsub = pubsub
+        self.pool = pool
         self.listeners = []
 
     async def trigger(self, message: AlephMessage):
@@ -60,7 +63,7 @@ async def trigger(self, message: AlephMessage):
                     vm_hash = listener.item_hash
                     event = message.json()
                     # Register the listener in the list of coroutines to run asynchronously:
-                    coroutines.append(run_code_on_event(vm_hash, event, self.pubsub))
+                    coroutines.append(run_code_on_event(vm_hash, event, self.pubsub, pool=self.pool))
                     break
 
         # Call all listeners asynchronously from the event loop:
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index 02fe112b0..d3f8a26ef 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -25,9 +25,6 @@
 
 logger = logging.getLogger(__name__)
 
-pool = VmPool()
-pool.setup()
-
 
 async def build_asgi_scope(path: str, request: web.Request) -> dict[str, Any]:
     # ASGI mandates lowercase header names
@@ -49,7 +46,7 @@ async def build_event_scope(event) -> dict[str, Any]:
     }
 
 
-async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
+async def create_vm_execution(vm_hash: ItemHash, pool: VmPool) -> VmExecution:
     message, original_message = await load_updated_message(vm_hash)
     pool.message_cache[vm_hash] = message
 
@@ -87,9 +84,9 @@ async def create_vm_execution(vm_hash: ItemHash) -> VmExecution:
     return execution
 
 
-async def create_vm_execution_or_raise_http_error(vm_hash: ItemHash) -> VmExecution:
+async def create_vm_execution_or_raise_http_error(vm_hash: ItemHash, pool: VmPool) -> VmExecution:
     try:
-        return await create_vm_execution(vm_hash=vm_hash)
+        return await create_vm_execution(vm_hash=vm_hash, pool=pool)
     except ResourceDownloadError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
@@ -110,7 +107,7 @@ async def create_vm_execution_or_raise_http_error(vm_hash: ItemHash) -> VmExecut
         raise HTTPInternalServerError(reason="Host did not respond to ping")
 
 
-async def run_code_on_request(vm_hash: ItemHash, path: str, request: web.Request) -> web.Response:
+async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, request: web.Request) -> web.Response:
     """
     Execute the code corresponding to the 'code id' in the path.
     """
@@ -118,7 +115,7 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, request: web.Request
     execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
-        execution = await create_vm_execution_or_raise_http_error(vm_hash=vm_hash)
+        execution = await create_vm_execution_or_raise_http_error(vm_hash=vm_hash, pool=pool)
 
     logger.debug(f"Using vm={execution.vm_id}")
 
@@ -207,7 +204,7 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, request: web.Request
             await execution.stop()
 
 
-async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub):
+async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub, pool: VmPool):
     """
     Execute code in response to an event.
     """
@@ -215,7 +212,7 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub):
     execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
-        execution = await create_vm_execution_or_raise_http_error(vm_hash=vm_hash)
+        execution = await create_vm_execution_or_raise_http_error(vm_hash=vm_hash, pool=pool)
 
     logger.debug(f"Using vm={execution.vm_id}")
 
@@ -252,17 +249,17 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub):
         if settings.REUSE_TIMEOUT > 0:
             if settings.WATCH_FOR_UPDATES:
                 execution.start_watching_for_updates(pubsub=pubsub)
-            execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT)
+            _ = execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT)
         else:
             await execution.stop()
 
 
-async def start_persistent_vm(vm_hash: ItemHash, pubsub: PubSub) -> VmExecution:
+async def start_persistent_vm(vm_hash: ItemHash, pubsub: Optional[PubSub], pool: VmPool) -> VmExecution:
     execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
         logger.info(f"Starting persistent virtual machine with id: {vm_hash}")
-        execution = await create_vm_execution(vm_hash=vm_hash)
+        execution = await create_vm_execution(vm_hash=vm_hash, pool=pool)
 
     # If the VM was already running in lambda mode, it should not expire
     # as long as it is also scheduled as long-running
@@ -271,13 +268,13 @@ async def start_persistent_vm(vm_hash: ItemHash, pubsub: PubSub) -> VmExecution:
 
     await execution.becomes_ready()
 
-    if settings.WATCH_FOR_UPDATES:
+    if pubsub and settings.WATCH_FOR_UPDATES:
         execution.start_watching_for_updates(pubsub=pubsub)
 
     return execution
 
 
-async def stop_persistent_vm(vm_hash: ItemHash) -> Optional[VmExecution]:
+async def stop_persistent_vm(vm_hash: ItemHash, pool: VmPool) -> Optional[VmExecution]:
     logger.info(f"Stopping persistent VM {vm_hash}")
     execution = await pool.get_running_vm(vm_hash)
 
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index ca86d3dec..067fc0ff0 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -13,9 +13,9 @@
 from aiohttp import web
 
 from ..conf import settings
+from ..pool import VmPool
 from .metrics import create_tables, setup_engine
 from .resources import about_system_usage
-from .run import pool
 from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task
 from .version import __version__
 from .views import (
@@ -70,19 +70,25 @@ async def server_version_middleware(
 
 
 async def stop_all_vms(app: web.Application):
+    pool: VmPool = app["vm_pool"]
     await pool.stop()
 
 
 def run():
     """Run the VM Supervisor."""
     settings.check()
+    pool = VmPool()
+    pool.setup()
 
     hostname = settings.DOMAIN_NAME
     protocol = "http" if hostname == "localhost" else "https"
 
     # Require a random token to access /about APIs
     secret_token = token_urlsafe(nbytes=32)
+    # Store app singletons. Note that app["pubsub"] will also be created.
     app["secret_token"] = secret_token
+    app["vm_pool"] = pool
+
     print(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
 
     engine = setup_engine()
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 49f787d2c..1a66c8d54 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -15,6 +15,7 @@
 from aleph.vm.utils import create_task_log_exceptions
 
 from ..conf import settings
+from ..pool import VmPool
 from .messages import load_updated_message
 from .pubsub import PubSub
 from .reactor import Reactor
@@ -103,7 +104,8 @@ async def watch_for_messages(dispatcher: PubSub, reactor: Reactor):
 async def start_watch_for_messages_task(app: web.Application):
     logger.debug("start_watch_for_messages_task()")
     pubsub = PubSub()
-    reactor = Reactor(pubsub)
+    pool: VmPool = app["vm_pool"]
+    reactor = Reactor(pubsub, pool)
 
     # Register an hardcoded initial program
     # TODO: Register all programs with subscriptions
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index f1ac5882b..994df3230 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -21,20 +21,17 @@
 )
 from aleph.vm.controllers.firecracker.program import FileTooLargeError
 from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInit
-from aleph.vm.orchestrator import status
-from aleph.vm.orchestrator.metrics import get_execution_records
-from aleph.vm.orchestrator.pubsub import PubSub
-from aleph.vm.orchestrator.resources import Allocation
-from aleph.vm.orchestrator.run import pool, run_code_on_request, start_persistent_vm
-from aleph.vm.orchestrator.version import __version__
-from aleph.vm.utils import (
-    HostNotFoundError,
-    b32_to_b16,
-    dumps_for_json,
-    get_ref_from_dns,
-)
 from packaging.version import InvalidVersion, Version
 
+from ...pool import VmPool
+from ...utils import HostNotFoundError, b32_to_b16, dumps_for_json, get_ref_from_dns
+from .. import status
+from ..metrics import get_execution_records
+from ..pubsub import PubSub
+from ..resources import Allocation
+from ..run import run_code_on_request, start_persistent_vm
+from ..version import __version__
+
 logger = logging.getLogger(__name__)
 
 
@@ -48,7 +45,8 @@ def run_code_from_path(request: web.Request) -> Awaitable[web.Response]:
     path = path if path.startswith("/") else f"/{path}"
 
     message_ref = ItemHash(request.match_info["ref"])
-    return run_code_on_request(message_ref, path, request)
+    pool: VmPool = request.app["vm_pool"]
+    return run_code_on_request(message_ref, path, pool, request)
 
 
 async def run_code_from_hostname(request: web.Request) -> web.Response:
@@ -82,7 +80,8 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
             except aiodns.error.DNSError:
                 raise HTTPNotFound(reason="Invalid message reference")
 
-    return await run_code_on_request(message_ref, path, request)
+    pool = request.app["vm_pool"]
+    return await run_code_on_request(message_ref, path, pool, request)
 
 
 def authenticate_request(request: web.Request) -> None:
@@ -103,6 +102,7 @@ async def about_login(request: web.Request) -> web.Response:
 
 async def about_executions(request: web.Request) -> web.Response:
     authenticate_request(request)
+    pool: VmPool = request.app["vm_pool"]
     return web.json_response(
         [dict(pool.executions.items())],
         dumps=dumps_for_json,
@@ -208,6 +208,7 @@ async def update_allocations(request: web.Request):
         return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code)
 
     pubsub: PubSub = request.app["pubsub"]
+    pool: VmPool = request.app["vm_pool"]
 
     # First free resources from persistent programs and instances that are not scheduled anymore.
     allocations = allocation.persistent_vms | allocation.instances
@@ -237,7 +238,7 @@ async def update_allocations(request: web.Request):
         try:
             logger.info(f"Starting long running VM '{vm_hash}'")
             vm_hash = ItemHash(vm_hash)
-            await start_persistent_vm(vm_hash, pubsub)
+            await start_persistent_vm(vm_hash, pubsub, pool)
         except vm_creation_exceptions as error:
             logger.exception(error)
             scheduling_errors[vm_hash] = error
@@ -247,7 +248,7 @@ async def update_allocations(request: web.Request):
         logger.info(f"Starting instance '{instance_hash}'")
         try:
             instance_hash = ItemHash(instance_hash)
-            await start_persistent_vm(instance_hash, pubsub)
+            await start_persistent_vm(instance_hash, pubsub, pool)
         except vm_creation_exceptions as error:
             logger.exception(error)
             scheduling_errors[instance_hash] = error
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 7aeef84e6..c4980afda 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -15,8 +15,8 @@
 from eth_account.messages import encode_defunct
 from jwskate import Jwk
 
-from aleph.vm.models import VmExecution
-from aleph.vm.orchestrator.run import pool
+from ...models import VmExecution
+from ...pool import VmPool
 
 logger = logging.getLogger(__name__)
 
@@ -121,7 +121,7 @@ def get_itemhash_or_400(match_info: UrlMappingMatchInfo) -> ItemHash:
         raise aiohttp.web_exceptions.HTTPBadRequest(body=f"Invalid ref: '{ref}'")
 
 
-def get_execution_or_404(ref: ItemHash) -> VmExecution:
+def get_execution_or_404(ref: ItemHash, pool: VmPool) -> VmExecution:
     """Return the execution corresponding to the ref or raise an HTTP 404 error."""
     execution = pool.executions.get(ref)
     if execution:
@@ -134,7 +134,8 @@ def get_execution_or_404(ref: ItemHash) -> VmExecution:
 async def stream_logs(request: web.Request):
     # TODO: Add user authentication
     vm_hash = get_itemhash_or_400(request.match_info)
-    execution = get_execution_or_404(vm_hash)
+    pool: VmPool = request.app["vm_pool"]
+    execution = get_execution_or_404(vm_hash, pool=pool)
 
     if execution.vm is None:
         raise web.HTTPBadRequest(body=f"VM {vm_hash} is not running")
@@ -170,7 +171,8 @@ async def operate_expire(request: web.Request):
     if not 0 < timeout < timedelta(days=10).total_seconds():
         return web.HTTPBadRequest(body="Invalid timeout duration")
 
-    execution = get_execution_or_404(vm_hash)
+    pool: VmPool = request.app["vm_pool"]
+    execution = get_execution_or_404(vm_hash, pool=pool)
 
     logger.info(f"Expiring in {timeout} seconds: {execution.vm_hash}")
     await execution.expire(timeout=timeout)
@@ -185,8 +187,9 @@ async def operate_stop(request: web.Request):
     # TODO: Add user authentication
     vm_hash = get_itemhash_or_400(request.match_info)
 
+    pool: VmPool = request.app["vm_pool"]
     logger.debug(f"Iterating through running executions... {pool.executions}")
-    execution = get_execution_or_404(vm_hash)
+    execution = get_execution_or_404(vm_hash, pool=pool)
 
     if execution.is_running:
         logger.info(f"Stopping {execution.vm_hash}")
@@ -203,7 +206,8 @@ async def operate_reboot(request: web.Request):
     Reboots the virtual machine, smoothly if possible.
     """
     vm_hash = get_itemhash_or_400(request.match_info)
-    execution = get_execution_or_404(vm_hash)
+    pool: VmPool = request.app["vm_pool"]
+    execution = get_execution_or_404(vm_hash, pool=pool)
 
     # TODO: implement this endpoint
     logger.info(f"Rebooting {execution.vm_hash}")
@@ -216,7 +220,8 @@ async def operate_erase(request: web.Request):
     Stop the virtual machine first if needed.
     """
     vm_hash = get_itemhash_or_400(request.match_info)
-    execution = get_execution_or_404(vm_hash)
+    pool: VmPool = request.app["vm_pool"]
+    execution = get_execution_or_404(vm_hash, pool=pool)
 
     logger.info(f"Erasing {execution.vm_hash}")
 

From 30419469bce469c2ccdda584c519c8fb19c60cc6 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 25 Oct 2023 17:15:20 +0200
Subject: [PATCH 523/990] Problem: CACHE_ROOT & EXECUTION_ROOT not respected
 (#452)

* Problem: CACHE_ROOT & EXECUTION_ROOT not respected

The value set by the user for CACHE_ROOT and EXECUTION_ROOT were not respected when calculating the default value of MESSAGE_CACHE, CODE_CACHE, EXECUTION_DATABASE, etc...

Solution:
This was caused by the default value being defined at class definition, this patch move the defaults computation into Settings.check() so we have the user passed settings.

Reproduce the problem:
1. Without setting MESSAGE_CACHE
2. set CACHE_ROOT (via environment variable ALEPH_VM_CACHE_ROOT)
3. Print the settings  e.g
   ALEPH_VM_CACHE_ROOT=/tmp/ aleph-vm --do-not-run --print-settings

What was happening:
MESSAGE_CACHE = /var/cache/aleph/vm/message

What is expected and is happening now:
MESSAGE_CACHE = /tmp/message

* move default calculation to __init__

to ensure they are present. Resolve mypy error
---
 src/aleph/vm/conf.py | 49 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 42 insertions(+), 7 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index a397c4e8e..08fa518af 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -10,6 +10,8 @@
 from typing import Any, Literal, NewType, Optional, Union
 
 from pydantic import BaseSettings, Field
+from pydantic.env_settings import DotenvType, env_file_sentinel
+from pydantic.typing import StrPath
 
 from aleph.vm.utils import is_command_available
 
@@ -173,17 +175,26 @@ class Settings(BaseSettings):
     CONNECTOR_URL = Url("http://localhost:4021")
 
     CACHE_ROOT = Path("/var/cache/aleph/vm")
-    MESSAGE_CACHE = CACHE_ROOT / "message"
-    CODE_CACHE = CACHE_ROOT / "code"
-    RUNTIME_CACHE = CACHE_ROOT / "runtime"
-    DATA_CACHE = CACHE_ROOT / "data"
+    MESSAGE_CACHE: Path = Field(
+        None,
+        description="Default to CACHE_ROOT/message",
+    )
+    CODE_CACHE: Path = Field(None, description="Default to CACHE_ROOT/code")
+    RUNTIME_CACHE: Path = Field(None, description="Default to CACHE_ROOT/runtime")
+    DATA_CACHE: Path = Field(None, description="Default to CACHE_ROOT/data")
 
     EXECUTION_ROOT = Path("/var/lib/aleph/vm")
-    EXECUTION_DATABASE = EXECUTION_ROOT / "executions.sqlite3"
+    EXECUTION_DATABASE: Path = Field(
+        None, description="Location of database file. Default to EXECUTION_ROOT/executions.sqlite3"
+    )
     EXECUTION_LOG_ENABLED = False
-    EXECUTION_LOG_DIRECTORY = EXECUTION_ROOT / "executions"
+    EXECUTION_LOG_DIRECTORY: Path = Field(
+        None, description="Location of executions log. Default to EXECUTION_ROOT/executions/"
+    )
 
-    PERSISTENT_VOLUMES_DIR = EXECUTION_ROOT / "volumes" / "persistent"
+    PERSISTENT_VOLUMES_DIR: Path = Field(
+        None, description="Persistent volumes location. Default to EXECUTION_ROOT/volumes/persistent/"
+    )
 
     MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000  # 10 MB
     MAX_DATA_ARCHIVE_SIZE = 10_000_000  # 10 MB
@@ -284,7 +295,9 @@ def setup(self):
         os.makedirs(self.CODE_CACHE, exist_ok=True)
         os.makedirs(self.RUNTIME_CACHE, exist_ok=True)
         os.makedirs(self.DATA_CACHE, exist_ok=True)
+
         os.makedirs(self.EXECUTION_ROOT, exist_ok=True)
+
         os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True)
         os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True)
 
@@ -312,6 +325,28 @@ def display(self) -> str:
 
         return "\n".join(f"{attribute:<27} = {value}" for attribute, value in attributes.items())
 
+    def __init__(
+        self,
+        _env_file: Optional[DotenvType] = env_file_sentinel,
+        _env_file_encoding: Optional[str] = None,
+        _env_nested_delimiter: Optional[str] = None,
+        _secrets_dir: Optional[StrPath] = None,
+        **values: Any,
+    ) -> None:
+        super().__init__(_env_file, _env_file_encoding, _env_nested_delimiter, _secrets_dir, **values)
+        if not self.MESSAGE_CACHE:
+            self.MESSAGE_CACHE = self.CACHE_ROOT / "message"
+        if not self.CODE_CACHE:
+            self.CODE_CACHE = self.CACHE_ROOT / "code"
+        if not self.RUNTIME_CACHE:
+            self.RUNTIME_CACHE = self.CACHE_ROOT / "runtime"
+        if not self.DATA_CACHE:
+            self.DATA_CACHE = self.CACHE_ROOT / "data"
+        if not self.PERSISTENT_VOLUMES_DIR:
+            self.PERSISTENT_VOLUMES_DIR = self.EXECUTION_ROOT / "volumes" / "persistent"
+        if not self.EXECUTION_LOG_DIRECTORY:
+            self.EXECUTION_LOG_DIRECTORY = self.EXECUTION_ROOT / "executions"
+
     class Config:
         env_prefix = "ALEPH_VM_"
         case_sensitive = False

From 80eabbf804aca41cfaf00f646c1e7f4e23b2ec1c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 31 Oct 2023 11:01:12 +0100
Subject: [PATCH 524/990] Fix configuration edge cases (#453)

* Fix crash when not using jailer

* Improve system() wrapper for debug

Now in case a command failed it tell us which function called it

* Problem: command warnings if no jailman user exist

even with USE_JAILER=False

Solution: This was caused by prepare_jailer() even thought it was not needed.
Made it's running conditional on use_jailer

* Problem: Crash when lack of perm for nftables

e.g: When running as not root and with network disable
this was caused by the json.loads() unable to parse the empty output

Solution: Return a default value

* Problem: Crash on VM clean up if no tap interface

Solution: made conditional

* Problem: Status endpoint was hanging for unhandled error

it was stuck and never returned anything

Solution: Had a catchall for the exception that were not handled before

* Problem: ndppd error even if USE_NDP_PROXY=0

Solution: Make check conditional

* fix isort

* Update src/aleph/vm/orchestrator/run.py

Co-authored-by: Hugo Herter 

---------

Co-authored-by: Hugo Herter 
---
 src/aleph/vm/conf.py                               |  3 ++-
 src/aleph/vm/controllers/firecracker/executable.py | 11 +++++++----
 src/aleph/vm/hypervisors/firecracker/microvm.py    | 13 +++++++++++--
 src/aleph/vm/network/firewall.py                   |  1 +
 src/aleph/vm/orchestrator/run.py                   |  4 ++++
 5 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 08fa518af..38eb92d9f 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -288,7 +288,8 @@ def check(self):
             assert isfile(self.FAKE_DATA_VOLUME), "Local data volume .squashfs is missing"
 
         assert is_command_available("setfacl"), "Command `setfacl` not found, run `apt install acl`"
-        assert is_command_available("ndppd"), "Command `ndppd` not found, run `apt install ndppd`"
+        if self.USE_NDP_PROXY:
+            assert is_command_available("ndppd"), "Command `ndppd` not found, run `apt install ndppd`"
 
     def setup(self):
         os.makedirs(self.MESSAGE_CACHE, exist_ok=True)
diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index 0b751434f..9ddd39bdb 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -21,7 +21,7 @@
 from aleph.vm.hypervisors.firecracker.microvm import FirecrackerConfig, MicroVM
 from aleph.vm.network.firewall import teardown_nftables_for_vm
 from aleph.vm.network.interfaces import TapInterface
-from aleph.vm.storage import get_volume_path
+from aleph.vm.storage import chown_to_jailman, get_volume_path
 
 try:
     import psutil  # type: ignore [no-redef]
@@ -250,9 +250,11 @@ async def start(self):
             logger.debug("setup done")
         except Exception:
             # Stop the VM and clear network interfaces in case any error prevented the start of the virtual machine.
+            logger.error("VM startup failed, cleaning up network")
             await self.fvm.teardown()
             teardown_nftables_for_vm(self.vm_id)
-            await self.tap_interface.delete()
+            if self.tap_interface:
+                await self.tap_interface.delete()
             raise
 
         if self.enable_console:
@@ -280,7 +282,7 @@ async def start_guest_api(self):
         self.guest_api_process.start()
         while not exists(vsock_path):
             await asyncio.sleep(0.01)
-        subprocess.run(f"chown jailman:jailman {vsock_path}", shell=True, check=True)
+        await chown_to_jailman(Path(vsock_path))
         logger.debug(f"started guest API for {self.vm_id}")
 
     async def stop_guest_api(self):
@@ -291,7 +293,8 @@ async def teardown(self):
         if self.fvm:
             await self.fvm.teardown()
             teardown_nftables_for_vm(self.vm_id)
-            await self.tap_interface.delete()
+            if self.tap_interface:
+                await self.tap_interface.delete()
         await self.stop_guest_api()
 
     async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index c0022bb14..878134836 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -4,6 +4,7 @@
 import os.path
 import shutil
 import string
+import traceback
 from asyncio import Task
 from asyncio.base_events import Server
 from dataclasses import dataclass
@@ -40,7 +41,12 @@ def default(self, obj):
 
 def system(command):
     logger.debug(f"shell {command}")
-    return os.system(command)
+    ret = os.system(command)
+    if ret != 0:
+        logger.warning(f"Failed shell `{command}`: return code {ret}")
+        # print trace so we know who called this
+        traceback.print_stack()
+    return ret
 
 
 async def setfacl():
@@ -130,6 +136,8 @@ def to_dict(self):
         }
 
     def prepare_jailer(self):
+        if not self.use_jailer:
+            return False
         system(f"rm -fr {self.jailer_path}")
 
         # system(f"rm -fr {self.jailer_path}/run/")
@@ -377,7 +385,8 @@ async def unix_client_connected(reader: asyncio.StreamReader, _writer: asyncio.S
             await queue.put(runtime_config)
 
         self._unix_socket = await asyncio.start_unix_server(unix_client_connected, path=f"{self.vsock_path}_52")
-        system(f"chown jailman:jailman {self.vsock_path}_52")
+        if self.use_jailer:
+            system(f"chown jailman:jailman {self.vsock_path}_52")
         try:
             self.runtime_config = await asyncio.wait_for(queue.get(), timeout=self.init_timeout)
             logger.debug("...signal from init received")
diff --git a/src/aleph/vm/network/firewall.py b/src/aleph/vm/network/firewall.py
index 416f00d3c..eb76597ea 100644
--- a/src/aleph/vm/network/firewall.py
+++ b/src/aleph/vm/network/firewall.py
@@ -47,6 +47,7 @@ def get_existing_nftables_ruleset() -> dict:
 
     if return_code != 0:
         logger.error(f"Unable to get nftables ruleset: {error}")
+        return {"nftables": []}
 
     nft_ruleset = json.loads(output)
     return nft_ruleset
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index d3f8a26ef..6651256fb 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -105,6 +105,10 @@ async def create_vm_execution_or_raise_http_error(vm_hash: ItemHash, pool: VmPoo
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
         raise HTTPInternalServerError(reason="Host did not respond to ping")
+    except Exception as error:
+        logger.exception(error)
+        pool.forget_vm(vm_hash=vm_hash)
+        raise HTTPInternalServerError(reason="unhandled error during initialisation") from error
 
 
 async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, request: web.Request) -> web.Response:

From a3d5b52addd93747e0886b13f9ba3d8d9a794722 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 31 Oct 2023 15:09:04 +0100
Subject: [PATCH 525/990] Fix: Sentry context did not include version info
 (#451)

Solution: Add `git` and `apt` version info to the Sentry context.
---
 packaging/Makefile                          |  2 +-
 src/aleph/vm/guest_api/__main__.py          |  9 +++++++++
 src/aleph/vm/orchestrator/__init__.py       |  7 +++----
 src/aleph/vm/orchestrator/cli.py            |  9 +++++++++
 src/aleph/vm/orchestrator/supervisor.py     |  3 ++-
 src/aleph/vm/orchestrator/views/__init__.py | 22 ++++++++++++---------
 src/aleph/vm/{orchestrator => }/version.py  |  0
 7 files changed, 37 insertions(+), 15 deletions(-)
 rename src/aleph/vm/{orchestrator => }/version.py (100%)

diff --git a/packaging/Makefile b/packaging/Makefile
index 77bca7d20..1808cf047 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -47,7 +47,7 @@ download-ipfs-kubo: target-dir build-dir
 
 version:
 	python3 ./version_from_git.py --inplace deb aleph-vm/DEBIAN/control
-	python3 ./version_from_git.py --inplace __version__ ../src/aleph/vm/orchestrator/version.py
+	python3 ./version_from_git.py --inplace __version__ ../src/aleph/vm/version.py
 
 build-dir:
 	mkdir -p target
diff --git a/src/aleph/vm/guest_api/__main__.py b/src/aleph/vm/guest_api/__main__.py
index 9bd1ce3c2..a15ffafd7 100644
--- a/src/aleph/vm/guest_api/__main__.py
+++ b/src/aleph/vm/guest_api/__main__.py
@@ -8,6 +8,8 @@
 from aiohttp import web
 from setproctitle import setproctitle
 
+from aleph.vm.version import get_version_from_apt, get_version_from_git
+
 try:
     import sentry_sdk
 except ImportError:
@@ -168,6 +170,13 @@ def run_guest_api(
             # We recommend adjusting this value in production.
             traces_sample_rate=1.0,
         )
+        sentry_sdk.set_context(
+            "version",
+            {
+                "git": get_version_from_git(),
+                "apt": get_version_from_apt(),
+            },
+        )
 
     setproctitle(f"aleph-vm guest_api on {unix_socket_path}")
     app = web.Application()
diff --git a/src/aleph/vm/orchestrator/__init__.py b/src/aleph/vm/orchestrator/__init__.py
index 8f783cf65..b4c1907a4 100644
--- a/src/aleph/vm/orchestrator/__init__.py
+++ b/src/aleph/vm/orchestrator/__init__.py
@@ -1,3 +1,5 @@
+from aleph.vm.version import __version__
+
 from . import (
     messages,
     metrics,
@@ -8,14 +10,12 @@
     status,
     supervisor,
     tasks,
-    version,
     views,
     vm,
 )
 
-__version__ = version.__version__
-
 __all__ = (
+    "__version__",
     "messages",
     "metrics",
     "pubsub",
@@ -25,7 +25,6 @@
     "status",
     "supervisor",
     "tasks",
-    "version",
     "views",
     "vm",
 )
diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index 190dcc7c1..af1e6be29 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -11,6 +11,8 @@
 
 from aiohttp.web import Request, Response
 
+from aleph.vm.version import get_version_from_apt, get_version_from_git
+
 try:
     import sentry_sdk
 except ImportError:
@@ -313,6 +315,13 @@ def main():
                 # We recommend adjusting this value in production.
                 traces_sample_rate=1.0,
             )
+            sentry_sdk.set_context(
+                "version",
+                {
+                    "git": get_version_from_git(),
+                    "apt": get_version_from_apt(),
+                },
+            )
         else:
             logger.debug("Sentry SDK found with no DSN configured.")
     else:
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 067fc0ff0..2623118f9 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -12,12 +12,13 @@
 
 from aiohttp import web
 
+from aleph.vm.version import __version__
+
 from ..conf import settings
 from ..pool import VmPool
 from .metrics import create_tables, setup_engine
 from .resources import about_system_usage
 from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task
-from .version import __version__
 from .views import (
     about_config,
     about_execution_records,
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 994df3230..876f7396f 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -21,17 +21,21 @@
 )
 from aleph.vm.controllers.firecracker.program import FileTooLargeError
 from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInit
+from aleph.vm.orchestrator import status
+from aleph.vm.orchestrator.metrics import get_execution_records
+from aleph.vm.orchestrator.pubsub import PubSub
+from aleph.vm.orchestrator.resources import Allocation
+from aleph.vm.orchestrator.run import run_code_on_request, start_persistent_vm
+from aleph.vm.pool import VmPool
+from aleph.vm.utils import (
+    HostNotFoundError,
+    b32_to_b16,
+    dumps_for_json,
+    get_ref_from_dns,
+)
+from aleph.vm.version import __version__
 from packaging.version import InvalidVersion, Version
 
-from ...pool import VmPool
-from ...utils import HostNotFoundError, b32_to_b16, dumps_for_json, get_ref_from_dns
-from .. import status
-from ..metrics import get_execution_records
-from ..pubsub import PubSub
-from ..resources import Allocation
-from ..run import run_code_on_request, start_persistent_vm
-from ..version import __version__
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/src/aleph/vm/orchestrator/version.py b/src/aleph/vm/version.py
similarity index 100%
rename from src/aleph/vm/orchestrator/version.py
rename to src/aleph/vm/version.py

From b9d4386c6950dcdb9c3edcb6ebacbcb00b1e1d03 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 9 Nov 2023 16:33:49 +0100
Subject: [PATCH 526/990] Fix: Index did not show IPv6 available

Problem: Node operators could not easily see whether IPv6 is working fine on a node or not.

Solution: We don't want to enforce this yet, so this displays the availability to VMs as additional information with no color indicating a problem, just a black cross.

The connectivity is checked form the diagnostic VM, ensuring that the VM itself has outgoing IPv6 access.
---
 .../orchestrator/views/templates/index.html   | 50 +++++++++++++++----
 1 file changed, 40 insertions(+), 10 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/templates/index.html b/src/aleph/vm/orchestrator/views/templates/index.html
index a6483454c..c846b9707 100644
--- a/src/aleph/vm/orchestrator/views/templates/index.html
+++ b/src/aleph/vm/orchestrator/views/templates/index.html
@@ -14,7 +14,7 @@
             height: 0.5em;
         }
 
-        #loader-container{
+        .loader-container{
             text-align:center;
             padding:20px;
             height:80px;
@@ -29,15 +29,15 @@
             background:#207AC9;
         }
         @keyframes move {
-            0%   {height:20px;}
-            50%  {height:10px;}
-            100% {height:20px;}
+            0%   {height:12px;}
+            50%  {height:6px;}
+            100% {height:12px;}
         }
 
         @keyframes move2 {
-            0%   {height:10px;}
-            50%  {height:20px;}
-            100% {height:10px;}
+            0%   {height:6px;}
+            50%  {height:12px;}
+            100% {height:6px;}
         }
         #loader-one{
             animation-name: move;
@@ -95,7 +95,7 @@ 
    Diagnostic
    
         Virtualization is
         
             ...
-            
+            
                 
                 
                 
@@ -107,6 +107,17 @@ 
    Diagnostic
    
         Diagnostics checks |
         Open diagnostic VM
     
    
+    
    
+        Egress IPv6
+        
+            is ...
+            
+                
+                
+                
+            
+        
+    
    
 
 
    
     
    Version
    
@@ -138,7 +149,7 @@ 
    Tools
    
 -->
 
 
 
 
-
    Aleph.im Compute Node
    
-
-
    
-    
    
-        This is an Aleph.im compute resource node.
-    
    
-    
    
-        It executes user programs stored on the Aleph network in Virtual Machines.
-    
    
-    
    
-        See the repository for more info.
-    
    
-
-
    
-
-
    
-
-
    
-    
    Multiaddr
    
-    
    
-        This node is exposed on the following addresses:
-    
    
-    
-
-
    
-
-
    
-
-
    
-    
    Diagnostic
    
-    
    
-        Virtualization is
-        
-            ...
-            
-                
-                
-                
+    
    
+
+        
    Aleph.im Compute Node
    
+
+        
    
+            
    
+                This is an Aleph.im compute resource node.
+            
    
+            
    
+                It executes user programs stored on the Aleph network in Virtual Machines.
+            
    
+            
    
+                See the repository for more info.
+            
    
+
+        
    
+
+        
    
+
+        
    
+            
    Multiaddr
    
+            
    
+                This node is exposed on the following addresses:
+            
    
+            
+
+        
    
+
+        
    
+
+        
    
+        
    Diagnostic
    
+        
    
+            Virtualization is
+            
+                ...
+                
+                    
+                    
+                    
+                
             
-        
-    
    
-    
    
-    
    
-        Diagnostics checks |
-        Open diagnostic VM
-    
    
-    
    
-        Egress IPv6
-        
-            is ...
-            
-                
-                
-                
+        
    
+        
    
+        
    
+            Diagnostics checks |
+            Open diagnostic VM
+        
    
+        
    
+            Egress IPv6
+            
+                is ...
+                
+                    
+                    
+                    
+                
             
-        
-    
    
-
    
-
    
-    
    Version
    
-    
    
-        Running version $version.
-    
    
-    
    
-        
-        
-        
-        
    
-    
    
-
-
    
-
-
-
-
+    
+    })();
 
+    
 
 
\ No newline at end of file

From 995125b78db06dfe02556ff21771d2109cf4ba7d Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 24 Oct 2023 13:18:47 +0200
Subject: [PATCH 530/990] Fix: Error remapping did not extend the original
 error

---
 src/aleph/vm/conf.py                          |  4 ++--
 .../vm/controllers/firecracker/program.py     | 10 ++++----
 .../firecracker/snapshot_manager.py           |  4 ++--
 .../vm/hypervisors/firecracker/microvm.py     |  4 ++--
 src/aleph/vm/orchestrator/messages.py         | 16 ++++++-------
 src/aleph/vm/orchestrator/run.py              | 22 ++++++++---------
 src/aleph/vm/orchestrator/views/__init__.py   |  8 +++----
 src/aleph/vm/orchestrator/views/operator.py   | 24 +++++++++----------
 8 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 38eb92d9f..432050668 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -93,12 +93,12 @@ def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> list[st
         # "Failed to get global data: Unit dbus-org.freedesktop.resolve1.service not found."
         try:
             return list(resolvectl_dns_servers_ipv4(interface=network_interface))
-        except (FileNotFoundError, CalledProcessError):
+        except (FileNotFoundError, CalledProcessError) as error:
             if Path("/etc/resolv.conf").exists():
                 return list(etc_resolv_conf_dns_servers())
             else:
                 msg = "No DNS resolver found"
-                raise FileNotFoundError(msg)
+                raise FileNotFoundError(msg) from error
 
     elif dns_resolver == DnsResolver.resolv_conf:
         return list(etc_resolv_conf_dns_servers())
diff --git a/src/aleph/vm/controllers/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
index b8590ca98..57a518980 100644
--- a/src/aleph/vm/controllers/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -191,7 +191,7 @@ async def download_code(self) -> None:
         try:
             self.code_path = await get_code_path(code_ref)
         except ClientResponseError as error:
-            raise ResourceDownloadError(error)
+            raise ResourceDownloadError(error) from error
         assert self.code_path.is_file(), f"Code not found on '{self.code_path}'"
 
     async def download_runtime(self) -> None:
@@ -199,7 +199,7 @@ async def download_runtime(self) -> None:
         try:
             self.rootfs_path = await get_runtime_path(runtime_ref)
         except ClientResponseError as error:
-            raise ResourceDownloadError(error)
+            raise ResourceDownloadError(error) from error
         assert self.rootfs_path.is_file(), f"Runtime not found on {self.rootfs_path}"
 
     async def download_data(self) -> None:
@@ -209,7 +209,7 @@ async def download_data(self) -> None:
                 data_path = await get_data_path(data_ref)
                 self.data_path = data_path
             except ClientResponseError as error:
-                raise ResourceDownloadError(error)
+                raise ResourceDownloadError(error) from error
             assert data_path.is_file(), f"Data not found on {data_path}"
         else:
             self.data_path = None
@@ -420,9 +420,9 @@ async def communicate(reader_: StreamReader, writer_: StreamWriter, scope_: dict
 
         try:
             reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
-        except ConnectionRefusedError:
+        except ConnectionRefusedError as error:
             msg = "MicroVM may have crashed"
-            raise VmInitNotConnected(msg)
+            raise VmInitNotConnected(msg) from error
         try:
             return await asyncio.wait_for(
                 communicate(reader, writer, scope),
diff --git a/src/aleph/vm/controllers/firecracker/snapshot_manager.py b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
index 1b5eac64e..e3fd42032 100644
--- a/src/aleph/vm/controllers/firecracker/snapshot_manager.py
+++ b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
@@ -34,9 +34,9 @@ async def do_vm_snapshot(vm: AlephFirecrackerExecutable) -> CompressedDiskVolume
 
         logger.debug(f"New snapshots for VM {vm.vm_hash} created in {snapshot.path}")
         return snapshot
-    except ValueError:
+    except ValueError as error:
         msg = "Something failed taking an snapshot"
-        raise ValueError(msg)
+        raise ValueError(msg) from error
 
 
 def infinite_run_scheduler_jobs(scheduler: Scheduler) -> None:
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 878134836..a98622c19 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -390,9 +390,9 @@ async def unix_client_connected(reader: asyncio.StreamReader, _writer: asyncio.S
         try:
             self.runtime_config = await asyncio.wait_for(queue.get(), timeout=self.init_timeout)
             logger.debug("...signal from init received")
-        except asyncio.TimeoutError:
+        except asyncio.TimeoutError as error:
             logger.warning("Never received signal from init")
-            raise MicroVMFailedInit()
+            raise MicroVMFailedInit() from error
 
     async def shutdown(self) -> None:
         logger.debug(f"Shutdown vm={self.vm_id}")
diff --git a/src/aleph/vm/orchestrator/messages.py b/src/aleph/vm/orchestrator/messages.py
index a00e22310..3ed4f5cc2 100644
--- a/src/aleph/vm/orchestrator/messages.py
+++ b/src/aleph/vm/orchestrator/messages.py
@@ -12,11 +12,11 @@ async def try_get_message(ref: str) -> ExecutableMessage:
     """Get the message or raise an aiohttp HTTP error"""
     try:
         return await get_message(ref)
-    except ClientConnectorError:
-        raise HTTPServiceUnavailable(reason="Aleph Connector unavailable")
+    except ClientConnectorError as error:
+        raise HTTPServiceUnavailable(reason="Aleph Connector unavailable") from error
     except ClientResponseError as error:
-        if error.status == 404:
-            raise HTTPNotFound(reason="Hash not found", text=f"Hash not found: {ref}")
+        if error.status == HTTPNotFound.status_code:
+            raise HTTPNotFound(reason="Hash not found", text=f"Hash not found: {ref}") from error
         else:
             raise
 
@@ -24,11 +24,11 @@ async def try_get_message(ref: str) -> ExecutableMessage:
 async def get_latest_ref(item_hash: str) -> str:
     try:
         return await get_latest_amend(item_hash)
-    except ClientConnectorError:
-        raise HTTPServiceUnavailable(reason="Aleph Connector unavailable")
+    except ClientConnectorError as error:
+        raise HTTPServiceUnavailable(reason="Aleph Connector unavailable") from error
     except ClientResponseError as error:
-        if error.status == 404:
-            raise HTTPNotFound(reason="Hash not found", text=f"Hash not found: {item_hash}")
+        if error.status == HTTPNotFound.status_code:
+            raise HTTPNotFound(reason="Hash not found", text=f"Hash not found: {item_hash}") from error
         else:
             raise
 
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index 6651256fb..d346c339f 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -61,21 +61,21 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool) -> VmExecution:
     except ResourceDownloadError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPBadRequest(reason="Code, runtime or data not available")
+        raise HTTPBadRequest(reason="Code, runtime or data not available") from error
     except FileTooLargeError as error:
-        raise HTTPInternalServerError(reason=error.args[0])
+        raise HTTPInternalServerError(reason=error.args[0]) from error
     except VmSetupError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(reason="Error during vm initialisation")
+        raise HTTPInternalServerError(reason="Error during vm initialisation") from error
     except MicroVMFailedInit as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(reason="Error during runtime initialisation")
+        raise HTTPInternalServerError(reason="Error during runtime initialisation") from error
     except HostNotFoundError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(reason="Host did not respond to ping")
+        raise HTTPInternalServerError(reason="Host did not respond to ping") from error
 
     if not execution.vm:
         msg = "The VM has not been created"
@@ -90,25 +90,25 @@ async def create_vm_execution_or_raise_http_error(vm_hash: ItemHash, pool: VmPoo
     except ResourceDownloadError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPBadRequest(reason="Code, runtime or data not available")
+        raise HTTPBadRequest(reason="Code, runtime or data not available") from error
     except FileTooLargeError as error:
-        raise HTTPInternalServerError(reason=error.args[0])
+        raise HTTPInternalServerError(reason=error.args[0]) from error
     except VmSetupError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(reason="Error during vm initialisation")
+        raise HTTPInternalServerError(reason="Error during vm initialisation") from error
     except MicroVMFailedInit as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(reason="Error during runtime initialisation")
+        raise HTTPInternalServerError(reason="Error during runtime initialisation") from error
     except HostNotFoundError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(reason="Host did not respond to ping")
+        raise HTTPInternalServerError(reason="Host did not respond to ping") from error
     except Exception as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(reason="unhandled error during initialisation") from error
+        raise HTTPInternalServerError(reason="Unhandled error during initialisation") from error
 
 
 async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, request: web.Request) -> web.Response:
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 90e6a59dd..a28d6fe13 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -81,8 +81,8 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
             try:
                 message_ref = ItemHash(await get_ref_from_dns(domain=f"_aleph-id.{request.host}"))
                 logger.debug(f"Using DNS TXT record to obtain '{message_ref}'")
-            except aiodns.error.DNSError:
-                raise HTTPNotFound(reason="Invalid message reference")
+            except aiodns.error.DNSError as error:
+                raise HTTPNotFound(reason="Invalid message reference") from error
 
     pool = request.app["vm_pool"]
     return await run_code_on_request(message_ref, path, pool, request)
@@ -177,12 +177,12 @@ async def status_check_version(request: web.Request):
     try:
         reference = Version(reference_str)
     except InvalidVersion as error:
-        raise web.HTTPBadRequest(text=error.args[0])
+        raise web.HTTPBadRequest(text=error.args[0]) from error
 
     try:
         current = Version(__version__)
     except InvalidVersion as error:
-        raise web.HTTPServiceUnavailable(text=error.args[0])
+        raise web.HTTPServiceUnavailable(text=error.args[0]) from error
 
     if current >= reference:
         return web.Response(status=200, text=f"Up-to-date: version {current} >= {reference}")
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index c4980afda..a339b1ba8 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -59,29 +59,29 @@ async def authenticate_jwk(request: web.Request):
         payload = keypair_dict.get("payload")
         signature = keypair_dict.get("signature")
     except (json.JSONDecodeError, KeyError):
-        raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey format")
+        raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey format") from None
 
     try:
         json_payload = get_json_from_hex(payload)
-    except json.JSONDecodeError:
-        raise web.HTTPBadRequest(reason="")
+    except json.JSONDecodeError as error:
+        raise web.HTTPBadRequest(reason="") from error
 
     if not verify_wallet_signature(signature, payload, json_payload.get("address")):
-        raise web.HTTPUnauthorized(reason="Invalid signature")
+        raise web.HTTPUnauthorized(reason="Invalid signature") from None
 
     expires = json_payload.get("expires")
     if not expires or not is_token_still_valid(expires):
-        raise web.HTTPUnauthorized(reason="Token expired")
+        raise web.HTTPUnauthorized(reason="Token expired") from None
 
     signed_operation = request.headers.get("X-SignedOperation")
     if not signed_operation:
-        raise web.HTTPBadRequest(reason="Missing X-SignedOperation header")
+        raise web.HTTPBadRequest(reason="Missing X-SignedOperation header") from None
 
     json_web_key = Jwk(json_payload.get("pubkey"))
     try:
         payload = json.loads(signed_operation)
     except json.JSONDecodeError:
-        raise web.HTTPBadRequest(reason="Could not decode X-SignedOperation")
+        raise web.HTTPBadRequest(reason="Could not decode X-SignedOperation") from None
 
     # The signature is not part of the signed payload, remove it
     payload_signature = payload.pop("signature")
@@ -94,7 +94,7 @@ async def authenticate_jwk(request: web.Request):
     ):
         logger.debug("Signature verified")
     else:
-        raise web.HTTPUnauthorized(reason="Signature could not verified")
+        raise web.HTTPUnauthorized(reason="Signature could not verified") from None
 
 
 def require_jwk_authentication(handler: Callable[[web.Request], Awaitable[web.StreamResponse]]):
@@ -113,12 +113,12 @@ async def wrapper(request):
 def get_itemhash_or_400(match_info: UrlMappingMatchInfo) -> ItemHash:
     try:
         ref = match_info["ref"]
-    except KeyError:
-        raise aiohttp.web_exceptions.HTTPBadRequest(body="Missing field: 'ref'")
+    except KeyError as error:
+        raise aiohttp.web_exceptions.HTTPBadRequest(body="Missing field: 'ref'") from error
     try:
         return ItemHash(ref)
-    except UnknownHashError:
-        raise aiohttp.web_exceptions.HTTPBadRequest(body=f"Invalid ref: '{ref}'")
+    except UnknownHashError as error:
+        raise aiohttp.web_exceptions.HTTPBadRequest(body=f"Invalid ref: '{ref}'") from error
 
 
 def get_execution_or_404(ref: ItemHash, pool: VmPool) -> VmExecution:

From 0069f9a1dbf28879e055e5995963c75380063c79 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 24 Oct 2023 13:21:35 +0200
Subject: [PATCH 531/990] Fix: Function arguments not to standard

Multiple small fixes of function arguments not respecting Python standards.
---
 src/aleph/vm/controllers/firecracker/executable.py | 4 ++--
 src/aleph/vm/controllers/firecracker/instance.py   | 4 ++--
 src/aleph/vm/orchestrator/resources.py             | 2 +-
 src/aleph/vm/orchestrator/views/__init__.py        | 2 +-
 src/aleph/vm/orchestrator/views/operator.py        | 4 ++--
 src/aleph/vm/utils.py                              | 4 ++--
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index 9ddd39bdb..cf291f522 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -158,7 +158,7 @@ def __init__(
         resources: AlephFirecrackerResources,
         enable_networking: bool = False,
         enable_console: Optional[bool] = None,
-        hardware_resources: MachineResources = MachineResources(),
+        hardware_resources: Optional[MachineResources] = None,
         tap_interface: Optional[TapInterface] = None,
     ):
         self.vm_id = vm_id
@@ -168,7 +168,7 @@ def __init__(
             enable_console = settings.PRINT_SYSTEM_LOGS
         self.enable_console = enable_console
         self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING
-        self.hardware_resources = hardware_resources
+        self.hardware_resources = hardware_resources or MachineResources()
         self.tap_interface = tap_interface
 
         self.fvm = MicroVM(
diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index 179385127..efc7a981a 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -66,7 +66,7 @@ def __init__(
         resources: AlephInstanceResources,
         enable_networking: bool = False,
         enable_console: Optional[bool] = None,
-        hardware_resources: MachineResources = MachineResources(),
+        hardware_resources: Optional[MachineResources] = None,
         tap_interface: Optional[TapInterface] = None,
     ):
         self.latest_snapshot = None
@@ -76,7 +76,7 @@ def __init__(
             resources,
             enable_networking,
             enable_console,
-            hardware_resources,
+            hardware_resources or MachineResources(),
             tap_interface,
         )
 
diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 7f5fdbdf6..871d4aaf3 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -91,7 +91,7 @@ def get_machine_properties() -> MachineProperties:
     )
 
 
-async def about_system_usage(request: web.Request):
+async def about_system_usage(_: web.Request):
     period_start = datetime.now(timezone.utc).replace(second=0, microsecond=0)
 
     usage: MachineUsage = MachineUsage(
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index a28d6fe13..f5b349c32 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -121,7 +121,7 @@ async def about_config(request: web.Request) -> web.Response:
     )
 
 
-async def about_execution_records(request: web.Request):
+async def about_execution_records(_: web.Request):
     records = await get_execution_records()
     return web.json_response(
         records,
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index a339b1ba8..d6f81d0bd 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -42,11 +42,11 @@ def verify_wallet_signature(signature, message, address):
     return computed_address.lower() == address.lower()
 
 
-def get_json_from_hex(str: str):
+def get_json_from_hex(string: str):
     """
     Converts a hex string to a json object
     """
-    return json.loads(bytes.fromhex(str).decode("utf-8"))
+    return json.loads(bytes.fromhex(string).decode("utf-8"))
 
 
 async def authenticate_jwk(request: web.Request):
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index f0858ea4e..d8324bd4d 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -32,10 +32,10 @@ def as_msgpack(self) -> bytes:
             raise TypeError(msg)
 
 
-def b32_to_b16(hash: str) -> bytes:
+def b32_to_b16(string: str) -> bytes:
     """Convert base32 encoded bytes to base16 encoded bytes."""
     # Add padding
-    hash_b32: str = hash.upper() + "=" * (56 - len(hash))
+    hash_b32: str = string.upper() + "=" * (56 - len(string))
     hash_bytes: bytes = b32decode(hash_b32.encode())
     return b16encode(hash_bytes).lower()
 

From a873b6aebf7f41a6fd79b15871b7428185059b0a Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 24 Oct 2023 14:40:45 +0200
Subject: [PATCH 532/990] Fix: Path.link_to is deprecated

---
 src/aleph/vm/hypervisors/firecracker/microvm.py | 13 +++++++++++--
 src/aleph/vm/orchestrator/views/__init__.py     |  6 +++---
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index a98622c19..a40faa113 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -4,6 +4,7 @@
 import os.path
 import shutil
 import string
+import sys
 import traceback
 from asyncio import Task
 from asyncio.base_events import Server
@@ -261,7 +262,12 @@ def enable_kernel(self, kernel_image_path: Path) -> Path:
         if self.use_jailer:
             kernel_filename = kernel_image_path.name
             jailer_kernel_image_path = f"/opt/{kernel_filename}"
-            kernel_image_path.link_to(f"{self.jailer_path}{jailer_kernel_image_path}")
+
+            if sys.version_info >= (3, 10):
+                Path(f"{self.jailer_path}{jailer_kernel_image_path}").hardlink_to(kernel_image_path)
+            else:
+                kernel_image_path.link_to(f"{self.jailer_path}{jailer_kernel_image_path}")
+
             return Path(jailer_kernel_image_path)
         else:
             return kernel_image_path
@@ -325,7 +331,10 @@ def enable_drive(self, drive_path: Path, read_only: bool = True) -> Drive:
         if self.use_jailer:
             drive_filename = drive_path.name
             jailer_path_on_host = f"/opt/{drive_filename}"
-            drive_path.link_to(f"{self.jailer_path}/{jailer_path_on_host}")
+            if sys.version_info >= (3, 10):
+                Path(f"{self.jailer_path}/{jailer_path_on_host}").hardlink_to(drive_path)
+            else:
+                drive_path.link_to(f"{self.jailer_path}/{jailer_path_on_host}")
             drive_path = Path(jailer_path_on_host)
 
         drive = Drive(
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index f5b349c32..796c9aa48 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -285,12 +285,12 @@ async def update_allocations(request: web.Request):
     # Schedule the start of instances:
     for instance_hash in allocation.instances:
         logger.info(f"Starting instance '{instance_hash}'")
+        instance_item_hash = ItemHash(instance_hash)
         try:
-            instance_hash = ItemHash(instance_hash)
-            await start_persistent_vm(instance_hash, pubsub, pool)
+            await start_persistent_vm(instance_item_hash, pubsub, pool)
         except vm_creation_exceptions as error:
             logger.exception(error)
-            scheduling_errors[instance_hash] = error
+            scheduling_errors[instance_item_hash] = error
 
     # Log unsupported features
     if allocation.on_demand_vms:

From 73479cf8bde56717a7c1b31dfa1470548c1abd47 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 24 Oct 2023 14:45:40 +0200
Subject: [PATCH 533/990] Fix: Hardcoded HTTP error codes were used

Problem: Hardcoded HTTP error codes were used. Looking into the specification was required to understand their meaning.

Solution: Use HTTP error codes from aiohttp.
---
 src/aleph/vm/orchestrator/run.py    | 20 ++++++++++++--------
 src/aleph/vm/orchestrator/status.py |  7 ++++---
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index d346c339f..6febeca2a 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -4,7 +4,11 @@
 
 import msgpack
 from aiohttp import web
-from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError
+from aiohttp.web_exceptions import (
+    HTTPBadGateway,
+    HTTPBadRequest,
+    HTTPInternalServerError,
+)
 from aleph_message.models import ItemHash
 from msgpack import UnpackValueError
 from multidict import CIMultiDict
@@ -138,7 +142,7 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques
             await execution.stop()
 
             return web.Response(
-                status=502,
+                status=HTTPBadGateway.status_code,
                 reason="No response from VM",
                 text="VM did not respond and was shut down",
             )
@@ -148,7 +152,7 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques
         return web.HTTPGatewayTimeout(body="Program did not respond within `resource.seconds`")
     except UnpackValueError as error:
         logger.exception(error)
-        return web.Response(status=502, reason="Invalid response from VM")
+        return web.Response(status=HTTPBadGateway.status_code, reason="Invalid response from VM")
 
     try:
         result = msgpack.loads(result_raw, raw=False)
@@ -169,7 +173,7 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques
                 logger.warning(result["traceback"])
 
             return web.Response(
-                status=500,
+                status=HTTPInternalServerError.status_code,
                 reason="Error in VM execution",
                 body=result["traceback"],
                 content_type="text/plain",
@@ -198,7 +202,7 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques
         )
     except UnpackValueError as error:
         logger.exception(error)
-        return web.Response(status=502, reason="Invalid response from VM")
+        return web.Response(status=HTTPBadGateway.status_code, reason="Invalid response from VM")
     finally:
         if settings.REUSE_TIMEOUT > 0:
             if settings.WATCH_FOR_UPDATES:
@@ -227,7 +231,7 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub, pool: VmPo
         result_raw: bytes = await execution.run_code(scope=scope)
     except UnpackValueError as error:
         logger.exception(error)
-        return web.Response(status=502, reason="Invalid response from VM")
+        return web.Response(status=HTTPBadGateway.status_code, reason="Invalid response from VM")
 
     try:
         result = msgpack.loads(result_raw, raw=False)
@@ -237,7 +241,7 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub, pool: VmPo
         if "traceback" in result:
             logger.warning(result["traceback"])
             return web.Response(
-                status=500,
+                status=HTTPInternalServerError.status_code,
                 reason="Error in VM execution",
                 body=result["traceback"],
                 content_type="text/plain",
@@ -248,7 +252,7 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub, pool: VmPo
 
     except UnpackValueError as error:
         logger.exception(error)
-        return web.Response(status=502, reason="Invalid response from VM")
+        return web.Response(status=HTTPBadGateway.status_code, reason="Invalid response from VM")
     finally:
         if settings.REUSE_TIMEOUT > 0:
             if settings.WATCH_FOR_UPDATES:
diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py
index 4c4a332c6..70829971b 100644
--- a/src/aleph/vm/orchestrator/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -6,6 +6,7 @@
 from typing import Any
 
 from aiohttp import ClientResponseError, ClientSession
+from aiohttp.web_exceptions import HTTPBadGateway, HTTPInternalServerError, HTTPOk
 
 from ..conf import settings
 
@@ -94,7 +95,7 @@ async def check_ipv6(session: ClientSession) -> bool:
 async def check_internet(session: ClientSession) -> bool:
     try:
         result: dict = await get_json_from_vm(session, "/internet")
-        assert result["result"] == 200
+        assert result["result"] == HTTPOk.status_code
         assert "Server" in result["headers"]
         return True
     except ClientResponseError:
@@ -132,7 +133,7 @@ async def check_error_raised(session: ClientSession) -> bool:
     try:
         async with session.get(f"{CHECK_VM_URL}/raise") as resp:
             text = await resp.text()
-            return resp.status == 500 and "Traceback" in text
+            return resp.status == HTTPInternalServerError.status_code and "Traceback" in text
     except ClientResponseError:
         return False
 
@@ -140,7 +141,7 @@ async def check_error_raised(session: ClientSession) -> bool:
 async def check_crash_and_restart(session: ClientSession) -> bool:
     # Crash the VM init.
     async with session.get(f"{CHECK_VM_URL}/crash") as resp:
-        if resp.status != 502:
+        if resp.status != HTTPBadGateway.status_code:
             return False
 
     # Try loading the index page. A new execution should be created.

From 0a899a93908d380a59a3e4f59e3465abb367f82e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 24 Oct 2023 14:47:24 +0200
Subject: [PATCH 534/990] Fix: Minor code cleanups

---
 src/aleph/vm/conf.py                   | 2 +-
 src/aleph/vm/orchestrator/reactor.py   | 6 ++----
 src/aleph/vm/orchestrator/resources.py | 6 +++---
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 432050668..2044368e2 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -107,7 +107,7 @@ def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> list[st
         return list(resolvectl_dns_servers_ipv4(interface=network_interface))
 
     else:
-        assert "No DNS resolve defined, this should never happen."
+        assert False, "No DNS resolve defined, this should never happen."
 
 
 class Settings(BaseSettings):
diff --git a/src/aleph/vm/orchestrator/reactor.py b/src/aleph/vm/orchestrator/reactor.py
index 4c2ec0002..a9713519b 100644
--- a/src/aleph/vm/orchestrator/reactor.py
+++ b/src/aleph/vm/orchestrator/reactor.py
@@ -54,7 +54,7 @@ async def trigger(self, message: AlephMessage):
         for listener in self.listeners:
             if not listener.content.on.message:
                 logger.warning(
-                    "Program with no subscription was registered in reactor listeners: " f"{listener.item_hash}"
+                    r"Program with no subscription was registered in reactor listeners: {listener.item_hash}"
                 )
                 continue
 
@@ -74,6 +74,4 @@ def register(self, message: AlephMessage):
         if message.content.on.message:
             self.listeners.append(message)
         else:
-            logger.debug(
-                "Program with no subscription cannot be registered in reactor listeners: " f"{message.item_hash}"
-            )
+            logger.debug(f"Program with no subscription cannot be registered in reactor listeners: {message.item_hash}")
diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 871d4aaf3..8d3343f3a 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -36,9 +36,9 @@ class CoreFrequencies(BaseModel):
 
     @classmethod
     def from_psutil(cls, psutil_freq: psutil._common.scpufreq):
-        min = psutil_freq.min or psutil_freq.current
-        max = psutil_freq.max or psutil_freq.current
-        return cls(min=min, max=max)
+        min_ = psutil_freq.min or psutil_freq.current
+        max_ = psutil_freq.max or psutil_freq.current
+        return cls(min=min_, max=max_)
 
 
 class CpuUsage(BaseModel):

From c21c8881e97e719720e40859ad4905813497f4d8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 24 Oct 2023 14:52:01 +0200
Subject: [PATCH 535/990] Fix: Fixes for Python < 3.8 are now useless

We only support Python >= 3.9, so fixes for Python 3.7 are not relevant anymore.
---
 src/aleph/vm/models.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 9a92b6300..6c1e607a7 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -188,12 +188,8 @@ def stop_after_timeout(self, timeout: float = 5.0) -> Optional[Task]:
             logger.debug("VM already has a timeout. Extending it.")
             self.expire_task.cancel()
 
-        if sys.version_info.major >= 3 and sys.version_info.minor >= 8:
-            # Task can be named
-            vm_id: str = str(self.vm.vm_id if self.vm else None)
-            self.expire_task = create_task_log_exceptions(self.expire(timeout), name=f"expire {vm_id}")
-        else:
-            self.expire_task = create_task_log_exceptions(self.expire(timeout))
+        vm_id: str = str(self.vm.vm_id if self.vm else None)
+        self.expire_task = create_task_log_exceptions(self.expire(timeout), name=f"expire {vm_id}")
         return self.expire_task
 
     async def expire(self, timeout: float) -> None:

From 4cccfc10eaa92977a910a56f53a7ea1895dfaa7e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 24 Oct 2023 14:52:59 +0200
Subject: [PATCH 536/990] Fix: Useless print()

---
 src/aleph/vm/orchestrator/status.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py
index 70829971b..3ee85eaa3 100644
--- a/src/aleph/vm/orchestrator/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -109,7 +109,6 @@ async def check_cache(session: ClientSession) -> bool:
         result2: int = await get_json_from_vm(session, "/cache/get/a")
         assert result2 == "42"
         keys: list[str] = await get_json_from_vm(session, "/cache/keys")
-        print("KEYS", keys)
         assert "a" in keys
         return True
     except ClientResponseError:

From c2669fe81ddf64a0adcd9a9c418714ec640ea302 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 24 Oct 2023 14:58:49 +0200
Subject: [PATCH 537/990] Fix: Relative imports of parent not recommended

---
 src/aleph/vm/controllers/firecracker/instance.py | 2 +-
 src/aleph/vm/models.py                           | 1 -
 src/aleph/vm/orchestrator/cli.py                 | 5 +++--
 src/aleph/vm/orchestrator/metrics.py             | 2 +-
 src/aleph/vm/orchestrator/reactor.py             | 2 +-
 src/aleph/vm/orchestrator/resources.py           | 2 +-
 src/aleph/vm/orchestrator/run.py                 | 6 +++---
 src/aleph/vm/orchestrator/status.py              | 2 +-
 src/aleph/vm/orchestrator/supervisor.py          | 4 ++--
 src/aleph/vm/orchestrator/tasks.py               | 4 ++--
 src/aleph/vm/orchestrator/views/operator.py      | 4 ++--
 11 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index efc7a981a..d673175c0 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -21,6 +21,7 @@
 )
 from aleph.vm.hypervisors.firecracker.microvm import setfacl
 from aleph.vm.network.interfaces import TapInterface
+from aleph.vm.storage import create_devmapper, create_volume_file
 from aleph.vm.utils import (
     HostNotFoundError,
     NotEnoughDiskSpace,
@@ -29,7 +30,6 @@
     run_in_subprocess,
 )
 
-from ...storage import create_devmapper, create_volume_file
 from .executable import (
     AlephFirecrackerExecutable,
     AlephFirecrackerResources,
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 6c1e607a7..0bf7edb82 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -1,6 +1,5 @@
 import asyncio
 import logging
-import sys
 import uuid
 from asyncio import Task
 from dataclasses import dataclass
diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index af1e6be29..4a19d934f 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -22,8 +22,9 @@
 import alembic.config
 from aleph_message.models import ItemHash
 
-from ..conf import ALLOW_DEVELOPER_SSH_KEYS, make_db_url, settings
-from ..pool import VmPool
+from aleph.vm.conf import ALLOW_DEVELOPER_SSH_KEYS, make_db_url, settings
+from aleph.vm.pool import VmPool
+
 from . import metrics, supervisor
 from .pubsub import PubSub
 from .run import run_code_on_event, run_code_on_request, start_persistent_vm
diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index 125ff90b4..d195af215 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -14,7 +14,7 @@
     from sqlalchemy.ext.declarative import declarative_base
 
 
-from ..conf import make_db_url, settings
+from aleph.vm.conf import make_db_url, settings
 
 Session: sessionmaker
 
diff --git a/src/aleph/vm/orchestrator/reactor.py b/src/aleph/vm/orchestrator/reactor.py
index a9713519b..785f2c233 100644
--- a/src/aleph/vm/orchestrator/reactor.py
+++ b/src/aleph/vm/orchestrator/reactor.py
@@ -4,9 +4,9 @@
 from aleph_message.models import AlephMessage
 from aleph_message.models.execution.environment import Subscription
 
+from aleph.vm.pool import VmPool
 from aleph.vm.utils import create_task_log_exceptions
 
-from ..pool import VmPool
 from .pubsub import PubSub
 from .run import run_code_on_event
 
diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 8d3343f3a..175b61bfd 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -9,7 +9,7 @@
 from aleph_message.models.execution.environment import CpuProperties
 from pydantic import BaseModel, Field
 
-from ..conf import settings
+from aleph.vm.conf import settings
 
 
 class Period(BaseModel):
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index 6febeca2a..c8959b755 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -13,17 +13,17 @@
 from msgpack import UnpackValueError
 from multidict import CIMultiDict
 
+from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.program import (
     FileTooLargeError,
     ResourceDownloadError,
     VmSetupError,
 )
 from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInit
+from aleph.vm.models import VmExecution
+from aleph.vm.pool import VmPool
 from aleph.vm.utils import HostNotFoundError
 
-from ..conf import settings
-from ..models import VmExecution
-from ..pool import VmPool
 from .messages import load_updated_message
 from .pubsub import PubSub
 
diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py
index 3ee85eaa3..4aadd4612 100644
--- a/src/aleph/vm/orchestrator/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -8,7 +8,7 @@
 from aiohttp import ClientResponseError, ClientSession
 from aiohttp.web_exceptions import HTTPBadGateway, HTTPInternalServerError, HTTPOk
 
-from ..conf import settings
+from aleph.vm.conf import settings
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index fc70a1927..9357ad2a9 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -13,10 +13,10 @@
 
 from aiohttp import web
 
+from aleph.vm.conf import settings
+from aleph.vm.pool import VmPool
 from aleph.vm.version import __version__
 
-from ..conf import settings
-from ..pool import VmPool
 from .metrics import create_tables, setup_engine
 from .resources import about_system_usage
 from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 1a66c8d54..2cdeeb284 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -12,10 +12,10 @@
 from aleph_message.models import AlephMessage, ItemHash, ProgramMessage, parse_message
 from yarl import URL
 
+from aleph.vm.conf import settings
+from aleph.vm.pool import VmPool
 from aleph.vm.utils import create_task_log_exceptions
 
-from ..conf import settings
-from ..pool import VmPool
 from .messages import load_updated_message
 from .pubsub import PubSub
 from .reactor import Reactor
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index d6f81d0bd..fab97af52 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -15,8 +15,8 @@
 from eth_account.messages import encode_defunct
 from jwskate import Jwk
 
-from ...models import VmExecution
-from ...pool import VmPool
+from aleph.vm.models import VmExecution
+from aleph.vm.pool import VmPool
 
 logger = logging.getLogger(__name__)
 

From 4cb2cccc99859825a2d9482c2b568fe68c41a27c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 24 Oct 2023 15:06:55 +0200
Subject: [PATCH 538/990] Fix: Exception classes must end with `Error`

---
 src/aleph/vm/controllers/firecracker/executable.py | 2 +-
 src/aleph/vm/controllers/firecracker/instance.py   | 4 ++--
 src/aleph/vm/controllers/firecracker/program.py    | 4 ++--
 src/aleph/vm/hypervisors/firecracker/microvm.py    | 4 ++--
 src/aleph/vm/orchestrator/run.py                   | 6 +++---
 src/aleph/vm/orchestrator/views/__init__.py        | 4 ++--
 src/aleph/vm/utils.py                              | 2 +-
 7 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index cf291f522..4f82728cf 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -130,7 +130,7 @@ class VmSetupError(Exception):
     pass
 
 
-class VmInitNotConnected(Exception):
+class VmInitNotConnectedError(Exception):
     pass
 
 
diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index d673175c0..485f5e6a4 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -24,7 +24,7 @@
 from aleph.vm.storage import create_devmapper, create_volume_file
 from aleph.vm.utils import (
     HostNotFoundError,
-    NotEnoughDiskSpace,
+    NotEnoughDiskSpaceError,
     check_disk_space,
     ping,
     run_in_subprocess,
@@ -149,7 +149,7 @@ async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
         volume = DiskVolume(path=volume_path)
 
         if not check_disk_space(volume.size):
-            raise NotEnoughDiskSpace
+            raise NotEnoughDiskSpaceError
 
         snapshot = await volume.take_snapshot()
         compressed_snapshot = await snapshot.compress(settings.SNAPSHOT_COMPRESSION_ALGORITHM)
diff --git a/src/aleph/vm/controllers/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
index 57a518980..c98e817d5 100644
--- a/src/aleph/vm/controllers/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -33,7 +33,7 @@
     AlephFirecrackerExecutable,
     AlephFirecrackerResources,
     ResourceDownloadError,
-    VmInitNotConnected,
+    VmInitNotConnectedError,
     VmSetupError,
     Volume,
 )
@@ -422,7 +422,7 @@ async def communicate(reader_: StreamReader, writer_: StreamWriter, scope_: dict
             reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
         except ConnectionRefusedError as error:
             msg = "MicroVM may have crashed"
-            raise VmInitNotConnected(msg) from error
+            raise VmInitNotConnectedError(msg) from error
         try:
             return await asyncio.wait_for(
                 communicate(reader, writer, scope),
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index a40faa113..f0cd0832b 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -26,7 +26,7 @@
 DEVICE_BASE_DIRECTORY = "/dev/mapper"
 
 
-class MicroVMFailedInit(Exception):
+class MicroVMFailedInitError(Exception):
     pass
 
 
@@ -401,7 +401,7 @@ async def unix_client_connected(reader: asyncio.StreamReader, _writer: asyncio.S
             logger.debug("...signal from init received")
         except asyncio.TimeoutError as error:
             logger.warning("Never received signal from init")
-            raise MicroVMFailedInit() from error
+            raise MicroVMFailedInitError() from error
 
     async def shutdown(self) -> None:
         logger.debug(f"Shutdown vm={self.vm_id}")
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index c8959b755..20641b761 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -19,7 +19,7 @@
     ResourceDownloadError,
     VmSetupError,
 )
-from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInit
+from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInitError
 from aleph.vm.models import VmExecution
 from aleph.vm.pool import VmPool
 from aleph.vm.utils import HostNotFoundError
@@ -72,7 +72,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool) -> VmExecution:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
         raise HTTPInternalServerError(reason="Error during vm initialisation") from error
-    except MicroVMFailedInit as error:
+    except MicroVMFailedInitError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
         raise HTTPInternalServerError(reason="Error during runtime initialisation") from error
@@ -101,7 +101,7 @@ async def create_vm_execution_or_raise_http_error(vm_hash: ItemHash, pool: VmPoo
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
         raise HTTPInternalServerError(reason="Error during vm initialisation") from error
-    except MicroVMFailedInit as error:
+    except MicroVMFailedInitError as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
         raise HTTPInternalServerError(reason="Error during runtime initialisation") from error
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 796c9aa48..6f51c4faf 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -20,7 +20,7 @@
     VmSetupError,
 )
 from aleph.vm.controllers.firecracker.program import FileTooLargeError
-from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInit
+from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInitError
 from aleph.vm.orchestrator import status
 from aleph.vm.orchestrator.metrics import get_execution_records
 from aleph.vm.orchestrator.pubsub import PubSub
@@ -266,7 +266,7 @@ async def update_allocations(request: web.Request):
         ResourceDownloadError,
         FileTooLargeError,
         VmSetupError,
-        MicroVMFailedInit,
+        MicroVMFailedInitError,
         HostNotFoundError,
     )
 
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index d8324bd4d..ee3deed6a 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -134,5 +134,5 @@ def check_disk_space(bytes_to_use: int) -> bool:
     return host_disk_usage.free >= bytes_to_use
 
 
-class NotEnoughDiskSpace(OSError):
+class NotEnoughDiskSpaceError(OSError):
     pass

From 21183b4e48c3f8385da8d6b407282c74d3efb74d Mon Sep 17 00:00:00 2001
From: Bonjour Internet 
Date: Tue, 14 Nov 2023 18:11:14 +0100
Subject: [PATCH 539/990] Fix virtualization message when everything is ok
 (#468)

fix: Virtualization msg when everything is ok
---
 .../vm/orchestrator/views/static/helpers.js    | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/static/helpers.js b/src/aleph/vm/orchestrator/views/static/helpers.js
index ebaec4fae..7648f4460 100644
--- a/src/aleph/vm/orchestrator/views/static/helpers.js
+++ b/src/aleph/vm/orchestrator/views/static/helpers.js
@@ -7,14 +7,16 @@ async function fetchApiStatus () {
     if(q.ok){
         res.status = "working properly ✅";
     }
-    switch(Number(q.status)){
-        case 503:
-            res.status = "not working properly ❌";
-            res.details = await q.json();
-        case 500:
-            res.status = "❌ Failed";
-        default:
-            res.status = q.status;
+    else {
+        switch(Number(q.status)){
+            case 503:
+                res.status = "not working properly ❌";
+                res.details = await q.json();
+            case 500:
+                res.status = "❌ Failed";
+            default:
+                res.status = q.status;
+        }
     }
 
     return res;

From 1480f7910cc8f854d69f1fca5c8d6d2bbe3de84d Mon Sep 17 00:00:00 2001
From: Mike Hukiewitz <70762838+MHHukiewitz@users.noreply.github.com>
Date: Wed, 15 Nov 2023 11:19:18 +0100
Subject: [PATCH 540/990] Update INSTALL-Ubuntu-22.04.md (#469)

Update READMEs to point to latest aleph-vm version
---
 doc/INSTALL-Debian-11.md    | 2 +-
 doc/INSTALL-Debian-12.md    | 2 +-
 doc/INSTALL-Ubuntu-22.04.md | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index ee297c835..a8bf92bfc 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.9-rc1/aleph-vm.debian-11.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.0/aleph-vm.debian-11.deb
 apt install /opt/aleph-vm.debian-11.deb
 ```
 
diff --git a/doc/INSTALL-Debian-12.md b/doc/INSTALL-Debian-12.md
index b840259f2..0740a1196 100644
--- a/doc/INSTALL-Debian-12.md
+++ b/doc/INSTALL-Debian-12.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.9-rc1/aleph-vm.debian-12.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.0/aleph-vm.debian-12.deb
 apt install /opt/aleph-vm.debian-12.deb
 ```
 
diff --git a/doc/INSTALL-Ubuntu-22.04.md b/doc/INSTALL-Ubuntu-22.04.md
index efd5693f1..ead2b68be 100644
--- a/doc/INSTALL-Ubuntu-22.04.md
+++ b/doc/INSTALL-Ubuntu-22.04.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.2.9-rc1/aleph-vm.ubuntu-22.04.deb
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.0/aleph-vm.ubuntu-22.04.deb
 sudo apt install /opt/aleph-vm.ubuntu-22.04.deb
 ```
 

From 3dc325b9a13539725829fbf762a0f5b8b724e846 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 16 Nov 2023 17:00:02 +0100
Subject: [PATCH 541/990] Improve download integrity (#467)

* Fix: Integrity of runtime filesystems was not checked

Problem: Runtime filesystems could be corrupted on disk. This checks their integrity before launching a VM.

* Fix: Prevent concurrent downloads and failed attempts

Problem: The same file could be downloaded multiple times in parallel (ex: same runtime for multiple programs).

Solution: Ensure that only one task is downloading a file at a time by using the temporary file on disk as a lock.

This also adds multiple attempts to download files in case of network issues.

* Fix: Concurrent VM creation can lead to the same resources
---
 .../vm/hypervisors/firecracker/microvm.py     | 28 ++++--
 src/aleph/vm/storage.py                       | 98 ++++++++++++++-----
 2 files changed, 95 insertions(+), 31 deletions(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index f0cd0832b..3ee5dd5a3 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -263,10 +263,13 @@ def enable_kernel(self, kernel_image_path: Path) -> Path:
             kernel_filename = kernel_image_path.name
             jailer_kernel_image_path = f"/opt/{kernel_filename}"
 
-            if sys.version_info >= (3, 10):
-                Path(f"{self.jailer_path}{jailer_kernel_image_path}").hardlink_to(kernel_image_path)
-            else:
-                kernel_image_path.link_to(f"{self.jailer_path}{jailer_kernel_image_path}")
+            try:
+                if sys.version_info >= (3, 10):
+                    Path(f"{self.jailer_path}{jailer_kernel_image_path}").hardlink_to(kernel_image_path)
+                else:
+                    kernel_image_path.link_to(f"{self.jailer_path}{jailer_kernel_image_path}")
+            except FileExistsError:
+                logger.debug(f"File {jailer_kernel_image_path} already exists")
 
             return Path(jailer_kernel_image_path)
         else:
@@ -289,7 +292,10 @@ def enable_file_rootfs(self, path_on_host: Path) -> Path:
         if self.use_jailer:
             rootfs_filename = Path(path_on_host).name
             jailer_path_on_host = f"/opt/{rootfs_filename}"
-            os.link(path_on_host, f"{self.jailer_path}/{jailer_path_on_host}")
+            try:
+                os.link(path_on_host, f"{self.jailer_path}/{jailer_path_on_host}")
+            except FileExistsError:
+                logger.debug(f"File {jailer_path_on_host} already exists")
             return Path(jailer_path_on_host)
         else:
             return path_on_host
@@ -331,10 +337,14 @@ def enable_drive(self, drive_path: Path, read_only: bool = True) -> Drive:
         if self.use_jailer:
             drive_filename = drive_path.name
             jailer_path_on_host = f"/opt/{drive_filename}"
-            if sys.version_info >= (3, 10):
-                Path(f"{self.jailer_path}/{jailer_path_on_host}").hardlink_to(drive_path)
-            else:
-                drive_path.link_to(f"{self.jailer_path}/{jailer_path_on_host}")
+
+            try:
+                if sys.version_info >= (3, 10):
+                    Path(f"{self.jailer_path}/{jailer_path_on_host}").hardlink_to(drive_path)
+                else:
+                    drive_path.link_to(f"{self.jailer_path}/{jailer_path_on_host}")
+            except FileExistsError:
+                logger.debug(f"File {jailer_path_on_host} already exists")
             drive_path = Path(jailer_path_on_host)
 
         drive = Drive(
diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index 9dd9ef380..447faecde 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -4,6 +4,7 @@
 In this prototype, it returns a hardcoded example.
 In the future, it should connect to an Aleph node and retrieve the code from there.
 """
+import asyncio
 import json
 import logging
 import re
@@ -11,6 +12,7 @@
 from datetime import datetime, timezone
 from pathlib import Path
 from shutil import copy2, make_archive
+from subprocess import CalledProcessError
 from typing import Union
 
 import aiohttp
@@ -36,6 +38,10 @@
 DEVICE_MAPPER_DIRECTORY = "/dev/mapper"
 
 
+class CorruptedFilesystemError(Exception):
+    """Raised when a file containing a filesystem is corrupted."""
+
+
 async def chown_to_jailman(path: Path) -> None:
     """Changes ownership of the target when running firecracker inside jailer isolation."""
     if not path.exists():
@@ -45,39 +51,72 @@ async def chown_to_jailman(path: Path) -> None:
         await run_in_subprocess(["chown", "jailman:jailman", str(path)])
 
 
+async def file_downloaded_by_another_task(final_path: Path) -> None:
+    """Wait for a file to be downloaded by another task in parallel."""
+
+    # Wait for the file to be created
+    while not final_path.is_file():
+        await asyncio.sleep(0.1)
+
+
+async def download_file_in_chunks(url: str, tmp_path: Path) -> None:
+    async with aiohttp.ClientSession() as session:
+        resp = await session.get(url)
+        resp.raise_for_status()
+
+        with open(tmp_path, "wb") as cache_file:
+            counter = 0
+            while True:
+                chunk = await resp.content.read(65536)
+                if not chunk:
+                    break
+                cache_file.write(chunk)
+                counter += 1
+                if not (counter % 20):
+                    sys.stdout.write("")
+                    sys.stdout.flush()
+
+        sys.stdout.write("\n")
+        sys.stdout.flush()
+
+
 async def download_file(url: str, local_path: Path) -> None:
     # TODO: Limit max size of download to the message specification
     if local_path.is_file():
         logger.debug(f"File already exists: {local_path}")
         return
 
+    # Avoid partial downloads and incomplete files by only moving the file when it's complete.
     tmp_path = Path(f"{local_path}.part")
+
+    # Ensure the file is not being downloaded by another task in parallel.
+    try:
+        tmp_path.touch(exist_ok=False)
+    except FileExistsError:
+        # Another task is already downloading the file
+        # Use `asyncio.timeout` manager after dropping support for Python 3.10
+        await asyncio.wait_for(file_downloaded_by_another_task(local_path), timeout=300)
+
     logger.debug(f"Downloading {url} -> {tmp_path}")
-    async with aiohttp.ClientSession() as session:
-        resp = await session.get(url)
-        resp.raise_for_status()
+    download_attempts = 3
+    for attempt in range(download_attempts):
         try:
-            with open(tmp_path, "wb") as cache_file:
-                counter = 0
-                while True:
-                    chunk = await resp.content.read(65536)
-                    if not chunk:
-                        break
-                    cache_file.write(chunk)
-                    counter += 1
-                    if not (counter % 20):
-                        sys.stdout.write("")
-                        sys.stdout.flush()
-
-            sys.stdout.write("\n")
-            sys.stdout.flush()
-
+            await download_file_in_chunks(url, tmp_path)
             tmp_path.rename(local_path)
             logger.debug(f"Download complete, moved {tmp_path} -> {local_path}")
-        except Exception:
-            # Ensure no partial file is left
+        except (
+            aiohttp.ClientConnectionError,
+            aiohttp.ClientResponseError,
+            aiohttp.ClientPayloadError,
+        ) as error:
+            if attempt < (download_attempts - 1):
+                logger.warning(f"Download failed, retrying attempt {attempt + 1}/3...")
+                continue
+            else:
+                raise error
+        finally:
+            # Ensure no partial file is left behind
             tmp_path.unlink(missing_ok=True)
-            raise
 
 
 async def get_latest_amend(item_hash: str) -> str:
@@ -154,14 +193,29 @@ async def get_data_path(ref: str) -> Path:
     return cache_path
 
 
+async def check_squashfs_integrity(path: Path) -> None:
+    """Check that the squashfs file is not corrupted."""
+    try:
+        await run_in_subprocess(["unsquashfs", "-stat", "-no-progress", str(path)], check=True)
+    except CalledProcessError as error:
+        msg = f"Corrupted squashfs file: {path}"
+        raise CorruptedFilesystemError(msg) from error
+
+
 async def get_runtime_path(ref: str) -> Path:
     """Obtain the runtime used for the rootfs of a program."""
     if settings.FAKE_DATA_PROGRAM:
+        await check_squashfs_integrity(Path(settings.FAKE_DATA_RUNTIME))
         return Path(settings.FAKE_DATA_RUNTIME)
 
     cache_path = Path(settings.RUNTIME_CACHE) / ref
     url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}"
-    await download_file(url, cache_path)
+
+    if not cache_path.is_file():
+        # File does not exist, download it
+        await download_file(url, cache_path)
+
+    await check_squashfs_integrity(cache_path)
     await chown_to_jailman(cache_path)
     return cache_path
 

From 6d19a80dd6ba4dea0289c102dfdd31d4d6d2372f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 8 Nov 2023 16:06:30 +0100
Subject: [PATCH 542/990] Fix: Users could fill all RAM using queues

Users could open an unlimited number of queues containing an unlimited number of messages.
---
 src/aleph/vm/hypervisors/firecracker/microvm.py | 10 ++++++++--
 src/aleph/vm/orchestrator/views/operator.py     |  7 ++++++-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 3ee5dd5a3..0c5c26554 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -362,7 +362,10 @@ async def print_logs(self):
         while True:
             stdout = await self.proc.stdout.readline()
             for queue in self.log_queues:
-                await queue.put(("stdout", stdout))
+                if queue.full():
+                    logger.warning("Log queue is full")
+                else:
+                    await queue.put(("stdout", stdout))
             if stdout:
                 print(stdout.decode().strip())
             else:
@@ -374,7 +377,10 @@ async def print_logs_stderr(self):
         while True:
             stderr = await self.proc.stderr.readline()
             for queue in self.log_queues:
-                await queue.put(("stderr", stderr))
+                if queue.full():
+                    logger.warning("Log queue is full")
+                else:
+                    await queue.put(("stderr", stderr))
             if stderr:
                 print(stderr.decode().strip())
             else:
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index fab97af52..c2db8266d 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -140,12 +140,17 @@ async def stream_logs(request: web.Request):
     if execution.vm is None:
         raise web.HTTPBadRequest(body=f"VM {vm_hash} is not running")
 
-    queue: asyncio.Queue = asyncio.Queue()
+    queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
     try:
         ws = web.WebSocketResponse()
         try:
             await ws.prepare(request)
 
+            # Limit the number of queues per VM
+            if len(execution.vm.fvm.log_queues) > 20:
+                logger.warning("Too many log queues, dropping the oldest one")
+                execution.vm.fvm.log_queues.pop(0)
+
             execution.vm.fvm.log_queues.append(queue)
 
             while True:

From 7d2060eb0ab8441850ce3f87d1c25350edbcb7cb Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 8 Nov 2023 16:45:47 +0100
Subject: [PATCH 543/990] Problem: print_log tasks keep reading after vm end

After the vm proc and the related stdout and stderr were closed the task keep trying to read them, creating a busy loop
which wasted cpu.

Solution: end the task when the file descriptor is closed
---
 .../vm/hypervisors/firecracker/microvm.py     | 23 +++++++++----------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 0c5c26554..702473dfb 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -360,31 +360,30 @@ async def print_logs(self):
         while not self.proc:
             await asyncio.sleep(0.01)  # Todo: Use signal here
         while True:
-            stdout = await self.proc.stdout.readline()
+            line = await self.proc.stdout.readline()
+            if not line:  # EOF, FD is closed nothing more will come
+                return
             for queue in self.log_queues:
                 if queue.full():
                     logger.warning("Log queue is full")
                 else:
-                    await queue.put(("stdout", stdout))
-            if stdout:
-                print(stdout.decode().strip())
-            else:
-                await asyncio.sleep(0.001)
+                    await queue.put(("stdout", line))
+            print(self, line.decode().strip())
 
     async def print_logs_stderr(self):
         while not self.proc:
             await asyncio.sleep(0.01)  # Todo: Use signal here
         while True:
-            stderr = await self.proc.stderr.readline()
+            line = await self.proc.stderr.readline()
+            if not line:  # EOF, FD is closed nothing more will come
+                return
             for queue in self.log_queues:
                 if queue.full():
                     logger.warning("Log queue is full")
                 else:
-                    await queue.put(("stderr", stderr))
-            if stderr:
-                print(stderr.decode().strip())
-            else:
-                await asyncio.sleep(0.001)
+                    await queue.put(("stderr", line))
+                await queue.put(("stderr", line))
+            print(self, line.decode().strip(), file=sys.stderr)
 
     def start_printing_logs(self) -> tuple[Task, Task]:
         loop = asyncio.get_running_loop()

From 5daf33a9a590481d1cee986ae61ac6e696d58201 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 16 Nov 2023 17:49:30 +0100
Subject: [PATCH 544/990] Fix errors cascade on Websocket.prepare() error

This fix the error cascade that happened when opening the logs endpoint
not as websocket. This was problematic because it returned to the http
client a 500 error instead of the actual error code and message

To reproduce
* start aleph supervisor
* open in Firefox: http://localhost:4020/vm/3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af
* then in another tab http://localhost:4020/control/machine/3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af/logs

Error that happened before
2023-11-16 16:16:58,089 | ERROR | Error handling request
Traceback (most recent call last):
  File "/home/ubuntu/remote-aleph/src/aleph/vm/orchestrator/views/operator.py", line 147, in stream_logs
    await ws.prepare(request)
  File "/home/ubuntu/.virtualenvs/aleph-vm/lib/python3.10/site-packages/aiohttp/web_ws.py", line 137, in prepare
    protocol, writer = self._pre_start(request)
  File "/home/ubuntu/.virtualenvs/aleph-vm/lib/python3.10/site-packages/aiohttp/web_ws.py", line 232, in _pre_start
    headers, protocol, compress, notakeover = self._handshake(request)
  File "/home/ubuntu/.virtualenvs/aleph-vm/lib/python3.10/site-packages/aiohttp/web_ws.py", line 149, in _handshake
    raise HTTPBadRequest(
aiohttp.web_exceptions.HTTPBadRequest: Bad Request

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/remote-aleph/src/aleph/vm/orchestrator/views/operator.py", line 157, in stream_logs
    await ws.close()
  File "/home/ubuntu/.virtualenvs/aleph-vm/lib/python3.10/site-packages/aiohttp/web_ws.py", line 337, in close
    raise RuntimeError("Call .prepare() first")
RuntimeError: Call .prepare() first

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/.virtualenvs/aleph-vm/lib/python3.10/site-packages/aiohttp/web_protocol.py", line 433, in _handle_request
    resp = await request_handler(request)
  File "/home/ubuntu/.virtualenvs/aleph-vm/lib/python3.10/site-packages/aiohttp/web_app.py", line 504, in _handle
    resp = await handler(request)
  File "/home/ubuntu/.virtualenvs/aleph-vm/lib/python3.10/site-packages/aiohttp/web_middlewares.py", line 117, in impl
    return await handler(request)
  File "/home/ubuntu/remote-aleph/src/aleph/vm/orchestrator/supervisor.py", line 46, in server_version_middleware
    resp: web.StreamResponse = await handler(request)
  File "/home/ubuntu/remote-aleph/src/aleph/vm/orchestrator/views/operator.py", line 159, in stream_logs
    execution.vm.fvm.log_queues.remove(queue)
ValueError: list.remove(x): x not in list
---
 src/aleph/vm/orchestrator/views/operator.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index c2db8266d..389d7d8c4 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -143,14 +143,12 @@ async def stream_logs(request: web.Request):
     queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
     try:
         ws = web.WebSocketResponse()
+        await ws.prepare(request)
         try:
-            await ws.prepare(request)
-
             # Limit the number of queues per VM
             if len(execution.vm.fvm.log_queues) > 20:
                 logger.warning("Too many log queues, dropping the oldest one")
                 execution.vm.fvm.log_queues.pop(0)
-
             execution.vm.fvm.log_queues.append(queue)
 
             while True:
@@ -161,7 +159,8 @@ async def stream_logs(request: web.Request):
         finally:
             await ws.close()
     finally:
-        execution.vm.fvm.log_queues.remove(queue)
+        if queue in execution.vm.fvm.log_queues:
+            execution.vm.fvm.log_queues.remove(queue)
         queue.empty()
 
 

From 27a7780cd36a9fcdf379db0db5c89b1223298811 Mon Sep 17 00:00:00 2001
From: mhh 
Date: Fri, 17 Nov 2023 10:36:01 +0100
Subject: [PATCH 545/990] Add PR Difficulty Rating workflow to the
 .github/workflows directory with necessary steps and configurations.

---
 .github/workflows/pr-rating.yml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 .github/workflows/pr-rating.yml

diff --git a/.github/workflows/pr-rating.yml b/.github/workflows/pr-rating.yml
new file mode 100644
index 000000000..2bbcd27dd
--- /dev/null
+++ b/.github/workflows/pr-rating.yml
@@ -0,0 +1,19 @@
+name: Test PR Difficulty Rating Action
+
+permissions:
+  pull-requests: write
+
+on:
+  pull_request:
+    types: [opened, reopened, ready_for_review]
+
+jobs:
+  difficulty-rating:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+    steps:
+      - name: PR Difficulty Rating
+        uses: rate-my-pr/rate@v1
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          LLAMA_URL: ${{ secrets.LLAMA_URL }}

From 9c78d1eaba1f75bd9cb21276835c15d690e5e955 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 16 Nov 2023 16:59:33 +0100
Subject: [PATCH 546/990] Fix: Download progress printed nothing ("")

Solution: Print a dot "." instead.
---
 src/aleph/vm/storage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index 447faecde..976666d60 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -73,7 +73,7 @@ async def download_file_in_chunks(url: str, tmp_path: Path) -> None:
                 cache_file.write(chunk)
                 counter += 1
                 if not (counter % 20):
-                    sys.stdout.write("")
+                    sys.stdout.write(".")
                     sys.stdout.flush()
 
         sys.stdout.write("\n")

From 2a0a290caaf5970c487dbd5c885e586a0d7ac3fb Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 17 Nov 2023 12:18:25 +0100
Subject: [PATCH 547/990] Problem: MicroVM had no str which made log ugly

This made representation in the log like this
 2092.023134 |V DEBUG | Init received msg

Solution : Add a __str__ method on MicroVM
---
 src/aleph/vm/hypervisors/firecracker/microvm.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 702473dfb..344cb3bf5 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -88,6 +88,9 @@ class MicroVM:
     mounted_rootfs: Optional[Path] = None
     _unix_socket: Optional[Server] = None
 
+    def __str__(self):
+        return f""
+
     @property
     def namespace_path(self):
         firecracker_bin_name = os.path.basename(self.firecracker_bin_path)

From 9f243c3cdeeefa63ca94c88dc8af93259e37939c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 17 Nov 2023 12:56:18 +0100
Subject: [PATCH 548/990] Add easier parsable format for logs

---
 src/aleph/vm/hypervisors/firecracker/microvm.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 344cb3bf5..534f53f36 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -88,9 +88,12 @@ class MicroVM:
     mounted_rootfs: Optional[Path] = None
     _unix_socket: Optional[Server] = None
 
-    def __str__(self):
+    def __repr__(self):
         return f""
 
+    def __str__(self):
+        return f"vm-{self.vm_id}"
+
     @property
     def namespace_path(self):
         firecracker_bin_name = os.path.basename(self.firecracker_bin_path)

From bd645901bea2700af1c71ed21ed62e8f419e9061 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Mon, 23 Oct 2023 17:01:29 +0200
Subject: [PATCH 549/990] Problem: Class VMType are coupled to orchestration
 path.

Solution: Decouple VMType class to be outside orchestrator path.
---
 src/aleph/vm/network/hostnetwork.py           | 2 +-
 src/aleph/vm/pool.py                          | 2 +-
 src/aleph/vm/{orchestrator/vm => }/vm_type.py | 0
 tests/supervisor/test_ipv6_allocator.py       | 2 +-
 4 files changed, 3 insertions(+), 3 deletions(-)
 rename src/aleph/vm/{orchestrator/vm => }/vm_type.py (100%)

diff --git a/src/aleph/vm/network/hostnetwork.py b/src/aleph/vm/network/hostnetwork.py
index 22f76acf7..566a4b94c 100644
--- a/src/aleph/vm/network/hostnetwork.py
+++ b/src/aleph/vm/network/hostnetwork.py
@@ -6,7 +6,7 @@
 from aleph_message.models import ItemHash
 
 from aleph.vm.conf import IPv6AllocationPolicy
-from aleph.vm.orchestrator.vm.vm_type import VmType
+from aleph.vm.vm_type import VmType
 
 from .firewall import initialize_nftables, setup_nftables_for_vm, teardown_nftables
 from .interfaces import TapInterface
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index cba891c7f..ce77e1413 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -9,7 +9,7 @@
 from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
 from aleph.vm.network.hostnetwork import Network, make_ipv6_allocator
-from aleph.vm.orchestrator.vm.vm_type import VmType
+from aleph.vm.vm_type import VmType
 
 from .models import ExecutableContent, VmExecution
 
diff --git a/src/aleph/vm/orchestrator/vm/vm_type.py b/src/aleph/vm/vm_type.py
similarity index 100%
rename from src/aleph/vm/orchestrator/vm/vm_type.py
rename to src/aleph/vm/vm_type.py
diff --git a/tests/supervisor/test_ipv6_allocator.py b/tests/supervisor/test_ipv6_allocator.py
index bdf8d2a7b..a3fdf11aa 100644
--- a/tests/supervisor/test_ipv6_allocator.py
+++ b/tests/supervisor/test_ipv6_allocator.py
@@ -1,7 +1,7 @@
 import os
 
 from aleph.vm.network.hostnetwork import StaticIPv6Allocator
-from aleph.vm.orchestrator.vm.vm_type import VmType
+from aleph.vm.vm_type import VmType
 
 # Avoid failures linked to settings when initializing the global VmPool object
 os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"

From c5e9ce285755d53efc23f993133059416bda8a8a Mon Sep 17 00:00:00 2001
From: Bonjour Internet 
Date: Tue, 14 Nov 2023 14:56:34 +0100
Subject: [PATCH 550/990] chore: ship chart lib as part of aleph-vm

---
 .../static/lightweight-charts.standalone.production.js     | 7 +++++++
 src/aleph/vm/orchestrator/views/templates/index.html       | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 src/aleph/vm/orchestrator/views/static/lightweight-charts.standalone.production.js

diff --git a/src/aleph/vm/orchestrator/views/static/lightweight-charts.standalone.production.js b/src/aleph/vm/orchestrator/views/static/lightweight-charts.standalone.production.js
new file mode 100644
index 000000000..73c825b46
--- /dev/null
+++ b/src/aleph/vm/orchestrator/views/static/lightweight-charts.standalone.production.js
@@ -0,0 +1,7 @@
+/*!
+ * @license
+ * TradingView Lightweight Charts™ v4.1.1
+ * Copyright (c) 2023 TradingView, Inc.
+ * Licensed under Apache License 2.0 https://www.apache.org/licenses/LICENSE-2.0
+ */
+!function(){"use strict";const t={upColor:"#26a69a",downColor:"#ef5350",wickVisible:!0,borderVisible:!0,borderColor:"#378658",borderUpColor:"#26a69a",borderDownColor:"#ef5350",wickColor:"#737375",wickUpColor:"#26a69a",wickDownColor:"#ef5350"},i={upColor:"#26a69a",downColor:"#ef5350",openVisible:!0,thinBars:!0},n={color:"#2196f3",lineStyle:0,lineWidth:3,lineType:0,lineVisible:!0,crosshairMarkerVisible:!0,crosshairMarkerRadius:4,crosshairMarkerBorderColor:"",crosshairMarkerBorderWidth:2,crosshairMarkerBackgroundColor:"",lastPriceAnimation:0,pointMarkersVisible:!1},s={topColor:"rgba( 46, 220, 135, 0.4)",bottomColor:"rgba( 40, 221, 100, 0)",invertFilledArea:!1,lineColor:"#33D778",lineStyle:0,lineWidth:3,lineType:0,lineVisible:!0,crosshairMarkerVisible:!0,crosshairMarkerRadius:4,crosshairMarkerBorderColor:"",crosshairMarkerBorderWidth:2,crosshairMarkerBackgroundColor:"",lastPriceAnimation:0,pointMarkersVisible:!1},e={baseValue:{type:"price",price:0},topFillColor1:"rgba(38, 166, 154, 0.28)",topFillColor2:"rgba(38, 166, 154, 0.05)",topLineColor:"rgba(38, 166, 154, 1)",bottomFillColor1:"rgba(239, 83, 80, 0.05)",bottomFillColor2:"rgba(239, 83, 80, 0.28)",bottomLineColor:"rgba(239, 83, 80, 1)",lineWidth:3,lineStyle:0,lineType:0,lineVisible:!0,crosshairMarkerVisible:!0,crosshairMarkerRadius:4,crosshairMarkerBorderColor:"",crosshairMarkerBorderWidth:2,crosshairMarkerBackgroundColor:"",lastPriceAnimation:0,pointMarkersVisible:!1},r={color:"#26a69a",base:0},h={color:"#2196f3"},l={title:"",visible:!0,lastValueVisible:!0,priceLineVisible:!0,priceLineSource:0,priceLineWidth:1,priceLineColor:"",priceLineStyle:2,baseLineVisible:!0,baseLineWidth:1,baseLineColor:"#B2B5BE",baseLineStyle:0,priceFormat:{type:"price",precision:2,minMove:.01}};var a,o;function _(t,i){const n={0:[],1:[t.lineWidth,t.lineWidth],2:[2*t.lineWidth,2*t.lineWidth],3:[6*t.lineWidth,6*t.lineWidth],4:[t.lineWidth,4*t.lineWidth]}[i];t.setLineDash(n)}function u(t,i,n,s){t.beginPath();const e=t.lineWidth%2?.5:0;t.moveTo(n,i+e),t.lineTo(s,i+e),t.stroke()}function c(t,i){if(!t)throw new Error("Assertion failed"+(i?": "+i:""))}function d(t){if(void 0===t)throw new Error("Value is undefined");return t}function f(t){if(null===t)throw new Error("Value is null");return t}function v(t){return f(d(t))}!function(t){t[t.Simple=0]="Simple",t[t.WithSteps=1]="WithSteps",t[t.Curved=2]="Curved"}(a||(a={})),function(t){t[t.Solid=0]="Solid",t[t.Dotted=1]="Dotted",t[t.Dashed=2]="Dashed",t[t.LargeDashed=3]="LargeDashed",t[t.SparseDotted=4]="SparseDotted"}(o||(o={}));const p={khaki:"#f0e68c",azure:"#f0ffff",aliceblue:"#f0f8ff",ghostwhite:"#f8f8ff",gold:"#ffd700",goldenrod:"#daa520",gainsboro:"#dcdcdc",gray:"#808080",green:"#008000",honeydew:"#f0fff0",floralwhite:"#fffaf0",lightblue:"#add8e6",lightcoral:"#f08080",lemonchiffon:"#fffacd",hotpink:"#ff69b4",lightyellow:"#ffffe0",greenyellow:"#adff2f",lightgoldenrodyellow:"#fafad2",limegreen:"#32cd32",linen:"#faf0e6",lightcyan:"#e0ffff",magenta:"#f0f",maroon:"#800000",olive:"#808000",orange:"#ffa500",oldlace:"#fdf5e6",mediumblue:"#0000cd",transparent:"#0000",lime:"#0f0",lightpink:"#ffb6c1",mistyrose:"#ffe4e1",moccasin:"#ffe4b5",midnightblue:"#191970",orchid:"#da70d6",mediumorchid:"#ba55d3",mediumturquoise:"#48d1cc",orangered:"#ff4500",royalblue:"#4169e1",powderblue:"#b0e0e6",red:"#f00",coral:"#ff7f50",turquoise:"#40e0d0",white:"#fff",whitesmoke:"#f5f5f5",wheat:"#f5deb3",teal:"#008080",steelblue:"#4682b4",bisque:"#ffe4c4",aquamarine:"#7fffd4",aqua:"#0ff",sienna:"#a0522d",silver:"#c0c0c0",springgreen:"#00ff7f",antiquewhite:"#faebd7",burlywood:"#deb887",brown:"#a52a2a",beige:"#f5f5dc",chocolate:"#d2691e",chartreuse:"#7fff00",cornflowerblue:"#6495ed",cornsilk:"#fff8dc",crimson:"#dc143c",cadetblue:"#5f9ea0",tomato:"#ff6347",fuchsia:"#f0f",blue:"#00f",salmon:"#fa8072",blanchedalmond:"#ffebcd",slateblue:"#6a5acd",slategray:"#708090",thistle:"#d8bfd8",tan:"#d2b48c",cyan:"#0ff",darkblue:"#00008b",darkcyan:"#008b8b",darkgoldenrod:"#b8860b",darkgray:"#a9a9a9",blueviolet:"#8a2be2",black:"#000",darkmagenta:"#8b008b",darkslateblue:"#483d8b",darkkhaki:"#bdb76b",darkorchid:"#9932cc",darkorange:"#ff8c00",darkgreen:"#006400",darkred:"#8b0000",dodgerblue:"#1e90ff",darkslategray:"#2f4f4f",dimgray:"#696969",deepskyblue:"#00bfff",firebrick:"#b22222",forestgreen:"#228b22",indigo:"#4b0082",ivory:"#fffff0",lavenderblush:"#fff0f5",feldspar:"#d19275",indianred:"#cd5c5c",lightgreen:"#90ee90",lightgrey:"#d3d3d3",lightskyblue:"#87cefa",lightslategray:"#789",lightslateblue:"#8470ff",snow:"#fffafa",lightseagreen:"#20b2aa",lightsalmon:"#ffa07a",darksalmon:"#e9967a",darkviolet:"#9400d3",mediumpurple:"#9370d8",mediumaquamarine:"#66cdaa",skyblue:"#87ceeb",lavender:"#e6e6fa",lightsteelblue:"#b0c4de",mediumvioletred:"#c71585",mintcream:"#f5fffa",navajowhite:"#ffdead",navy:"#000080",olivedrab:"#6b8e23",palevioletred:"#d87093",violetred:"#d02090",yellow:"#ff0",yellowgreen:"#9acd32",lawngreen:"#7cfc00",pink:"#ffc0cb",paleturquoise:"#afeeee",palegoldenrod:"#eee8aa",darkolivegreen:"#556b2f",darkseagreen:"#8fbc8f",darkturquoise:"#00ced1",peachpuff:"#ffdab9",deeppink:"#ff1493",violet:"#ee82ee",palegreen:"#98fb98",mediumseagreen:"#3cb371",peru:"#cd853f",saddlebrown:"#8b4513",sandybrown:"#f4a460",rosybrown:"#bc8f8f",purple:"#800080",seagreen:"#2e8b57",seashell:"#fff5ee",papayawhip:"#ffefd5",mediumslateblue:"#7b68ee",plum:"#dda0dd",mediumspringgreen:"#00fa9a"};function m(t){return t<0?0:t>255?255:Math.round(t)||0}function b(t){return t<=0||t>0?t<0?0:t>1?1:Math.round(1e4*t)/1e4:0}const w=/^#([0-9a-f])([0-9a-f])([0-9a-f])([0-9a-f])?$/i,g=/^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})?$/i,M=/^rgb\(\s*(-?\d{1,10})\s*,\s*(-?\d{1,10})\s*,\s*(-?\d{1,10})\s*\)$/,x=/^rgba\(\s*(-?\d{1,10})\s*,\s*(-?\d{1,10})\s*,\s*(-?\d{1,10})\s*,\s*(-?[\d]{0,10}(?:\.\d+)?)\s*\)$/;function S(t){(t=t.toLowerCase())in p&&(t=p[t]);{const i=x.exec(t)||M.exec(t);if(i)return[m(parseInt(i[1],10)),m(parseInt(i[2],10)),m(parseInt(i[3],10)),b(i.length<5?1:parseFloat(i[4]))]}{const i=g.exec(t);if(i)return[m(parseInt(i[1],16)),m(parseInt(i[2],16)),m(parseInt(i[3],16)),1]}{const i=w.exec(t);if(i)return[m(17*parseInt(i[1],16)),m(17*parseInt(i[2],16)),m(17*parseInt(i[3],16)),1]}throw new Error(`Cannot parse color: ${t}`)}function y(t){const i=S(t);return{t:`rgb(${i[0]}, ${i[1]}, ${i[2]})`,i:(n=i,.199*n[0]+.687*n[1]+.114*n[2]>160?"black":"white")};var n}class k{constructor(){this.h=[]}l(t,i,n){const s={o:t,_:i,u:!0===n};this.h.push(s)}v(t){const i=this.h.findIndex((i=>t===i.o));i>-1&&this.h.splice(i,1)}p(t){this.h=this.h.filter((i=>i._!==t))}m(t,i,n){const s=[...this.h];this.h=this.h.filter((t=>!t.u)),s.forEach((s=>s.o(t,i,n)))}M(){return this.h.length>0}S(){this.h=[]}}function C(t,...i){for(const n of i)for(const i in n)void 0!==n[i]&&("object"!=typeof n[i]||void 0===t[i]||Array.isArray(n[i])?t[i]=n[i]:C(t[i],n[i]));return t}function T(t){return"number"==typeof t&&isFinite(t)}function P(t){return"number"==typeof t&&t%1==0}function R(t){return"string"==typeof t}function D(t){return"boolean"==typeof t}function O(t){const i=t;if(!i||"object"!=typeof i)return i;let n,s,e;for(s in n=Array.isArray(i)?[]:{},i)i.hasOwnProperty(s)&&(e=i[s],n[s]=e&&"object"==typeof e?O(e):e);return n}function A(t){return null!==t}function B(t){return null===t?void 0:t}const V="-apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif";function z(t,i,n){return void 0===i&&(i=V),`${n=void 0!==n?`${n} `:""}${t}px ${i}`}class E{constructor(t){this.k={C:1,T:5,P:NaN,R:"",D:"",O:"",A:"",B:0,V:0,I:0,L:0,N:0},this.F=t}W(){const t=this.k,i=this.j(),n=this.H();return t.P===i&&t.D===n||(t.P=i,t.D=n,t.R=z(i,n),t.L=2.5/12*i,t.B=t.L,t.V=i/12*t.T,t.I=i/12*t.T,t.N=0),t.O=this.$(),t.A=this.U(),this.k}$(){return this.F.W().layout.textColor}U(){return this.F.q()}j(){return this.F.W().layout.fontSize}H(){return this.F.W().layout.fontFamily}}class I{constructor(){this.Y=[]}X(t){this.Y=t}K(t,i,n){this.Y.forEach((s=>{s.K(t,i,n)}))}}class L{K(t,i,n){t.useMediaCoordinateSpace((t=>this.Z(t,i,n)))}G(t,i,n){t.useMediaCoordinateSpace((t=>this.J(t,i,n)))}J(t,i,n){}}class N extends L{constructor(){super(...arguments),this.tt=null}it(t){this.tt=t}Z({context:t}){if(null===this.tt||null===this.tt.nt)return;const i=this.tt.nt,n=this.tt,s=s=>{t.beginPath();for(let e=i.to-1;e>=i.from;--e){const i=n.st[e];t.moveTo(i.et,i.rt),t.arc(i.et,i.rt,s,0,2*Math.PI)}t.fill()};n.ht>0&&(t.fillStyle=n.lt,s(n.ot+n.ht)),t.fillStyle=n._t,s(n.ot)}}function F(){return{st:[{et:0,rt:0,ut:0,ct:0}],_t:"",lt:"",ot:0,ht:0,nt:null}}const W={from:0,to:1};class j{constructor(t,i){this.dt=new I,this.ft=[],this.vt=[],this.bt=!0,this.F=t,this.wt=i,this.dt.X(this.ft)}gt(t){const i=this.F.Mt();i.length!==this.ft.length&&(this.vt=i.map(F),this.ft=this.vt.map((t=>{const i=new N;return i.it(t),i})),this.dt.X(this.ft)),this.bt=!0}xt(){return this.bt&&(this.St(),this.bt=!1),this.dt}St(){const t=2===this.wt.W().mode,i=this.F.Mt(),n=this.wt.yt(),s=this.F.kt();i.forEach(((i,e)=>{var r;const h=this.vt[e],l=i.Ct(n);if(t||null===l||!i.Tt())return void(h.nt=null);const a=f(i.Pt());h._t=l.Rt,h.ot=l.ot,h.ht=l.Dt,h.st[0].ct=l.ct,h.st[0].rt=i.At().Ot(l.ct,a.Bt),h.lt=null!==(r=l.Vt)&&void 0!==r?r:this.F.zt(h.st[0].rt/i.At().Et()),h.st[0].ut=n,h.st[0].et=s.It(n),h.nt=W}))}}class H{K(t,i,n){t.useBitmapCoordinateSpace((t=>this.Z(t,i,n)))}}class $ extends H{constructor(t){super(),this.Lt=t}Z({context:t,bitmapSize:i,horizontalPixelRatio:n,verticalPixelRatio:s}){if(null===this.Lt)return;const e=this.Lt.Nt.Tt,r=this.Lt.Ft.Tt;if(!e&&!r)return;const h=Math.round(this.Lt.et*n),l=Math.round(this.Lt.rt*s);t.lineCap="butt",e&&h>=0&&(t.lineWidth=Math.floor(this.Lt.Nt.ht*n),t.strokeStyle=this.Lt.Nt.O,t.fillStyle=this.Lt.Nt.O,_(t,this.Lt.Nt.Wt),function(t,i,n,s){t.beginPath();const e=t.lineWidth%2?.5:0;t.moveTo(i+e,n),t.lineTo(i+e,s),t.stroke()}(t,h,0,i.height)),r&&l>=0&&(t.lineWidth=Math.floor(this.Lt.Ft.ht*s),t.strokeStyle=this.Lt.Ft.O,t.fillStyle=this.Lt.Ft.O,_(t,this.Lt.Ft.Wt),u(t,l,0,i.width))}}class U{constructor(t){this.bt=!0,this.jt={Nt:{ht:1,Wt:0,O:"",Tt:!1},Ft:{ht:1,Wt:0,O:"",Tt:!1},et:0,rt:0},this.Ht=new $(this.jt),this.$t=t}gt(){this.bt=!0}xt(){return this.bt&&(this.St(),this.bt=!1),this.Ht}St(){const t=this.$t.Tt(),i=f(this.$t.Ut()),n=i.qt().W().crosshair,s=this.jt;if(2===n.mode)return s.Ft.Tt=!1,void(s.Nt.Tt=!1);s.Ft.Tt=t&&this.$t.Yt(i),s.Nt.Tt=t&&this.$t.Xt(),s.Ft.ht=n.horzLine.width,s.Ft.Wt=n.horzLine.style,s.Ft.O=n.horzLine.color,s.Nt.ht=n.vertLine.width,s.Nt.Wt=n.vertLine.style,s.Nt.O=n.vertLine.color,s.et=this.$t.Kt(),s.rt=this.$t.Zt()}}function q(t,i,n,s,e,r){t.fillRect(i+r,n,s-2*r,r),t.fillRect(i+r,n+e-r,s-2*r,r),t.fillRect(i,n,r,e),t.fillRect(i+s-r,n,r,e)}function Y(t,i,n,s,e,r){t.save(),t.globalCompositeOperation="copy",t.fillStyle=r,t.fillRect(i,n,s,e),t.restore()}function X(t,i){return t.map((t=>0===t?t:t+i))}function K(t,i,n,s,e,r){t.beginPath(),t.lineTo(i+s-r[1],n),0!==r[1]&&t.arcTo(i+s,n,i+s,n+r[1],r[1]),t.lineTo(i+s,n+e-r[2]),0!==r[2]&&t.arcTo(i+s,n+e,i+s-r[2],n+e,r[2]),t.lineTo(i+r[3],n+e),0!==r[3]&&t.arcTo(i,n+e,i,n+e-r[3],r[3]),t.lineTo(i,n+r[0]),0!==r[0]&&t.arcTo(i,n,i+r[0],n,r[0])}function Z(t,i,n,s,e,r,h=0,l=[0,0,0,0],a=""){if(t.save(),!h||!a||a===r)return K(t,i,n,s,e,l),t.fillStyle=r,t.fill(),void t.restore();const o=h/2;if("transparent"!==r){K(t,i+h,n+h,s-2*h,e-2*h,X(l,-h)),t.fillStyle=r,t.fill()}if("transparent"!==a){K(t,i+o,n+o,s-h,e-h,X(l,-o)),t.lineWidth=h,t.strokeStyle=a,t.closePath(),t.stroke()}t.restore()}function G(t,i,n,s,e,r,h){t.save(),t.globalCompositeOperation="copy";const l=t.createLinearGradient(0,0,0,e);l.addColorStop(0,r),l.addColorStop(1,h),t.fillStyle=l,t.fillRect(i,n,s,e),t.restore()}class J{constructor(t,i){this.it(t,i)}it(t,i){this.Lt=t,this.Gt=i}Et(t,i){return this.Lt.Tt?t.P+t.L+t.B:0}K(t,i,n,s){if(!this.Lt.Tt||0===this.Lt.Jt.length)return;const e=this.Lt.O,r=this.Gt.t,h=t.useBitmapCoordinateSpace((t=>{const h=t.context;h.font=i.R;const l=this.Qt(t,i,n,s),a=l.ti,o=(t,i)=>{l.ii?Z(h,a.ni,a.si,a.ei,a.ri,t,a.hi,[a.ot,0,0,a.ot],i):Z(h,a.li,a.si,a.ei,a.ri,t,a.hi,[0,a.ot,a.ot,0],i)};return o(r,"transparent"),this.Lt.ai&&(h.fillStyle=e,h.fillRect(a.li,a.oi,a._i-a.li,a.ui)),o("transparent",r),this.Lt.ci&&(h.fillStyle=i.A,h.fillRect(l.ii?a.di-a.hi:0,a.si,a.hi,a.fi-a.si)),l}));t.useMediaCoordinateSpace((({context:t})=>{const n=h.vi;t.font=i.R,t.textAlign=h.ii?"right":"left",t.textBaseline="middle",t.fillStyle=e,t.fillText(this.Lt.Jt,n.pi,(n.si+n.fi)/2+n.mi)}))}Qt(t,i,n,s){var e;const{context:r,bitmapSize:h,mediaSize:l,horizontalPixelRatio:a,verticalPixelRatio:o}=t,_=this.Lt.ai||!this.Lt.bi?i.T:0,u=this.Lt.wi?i.C:0,c=i.L+this.Gt.gi,d=i.B+this.Gt.Mi,f=i.V,v=i.I,p=this.Lt.Jt,m=i.P,b=n.xi(r,p),w=Math.ceil(n.Si(r,p)),g=m+c+d,M=i.C+f+v+w+_,x=Math.max(1,Math.floor(o));let S=Math.round(g*o);S%2!=x%2&&(S+=1);const y=u>0?Math.max(1,Math.floor(u*a)):0,k=Math.round(M*a),C=Math.round(_*a),T=null!==(e=this.Gt.yi)&&void 0!==e?e:this.Gt.ki,P=Math.round(T*o)-Math.floor(.5*o),R=Math.floor(P+x/2-S/2),D=R+S,O="right"===s,A=O?l.width-u:u,B=O?h.width-y:y;let V,z,E;return O?(V=B-k,z=B-C,E=A-_-f-u):(V=B+k,z=B+C,E=A+_+f),{ii:O,ti:{si:R,oi:P,fi:D,ei:k,ri:S,ot:2*a,hi:y,ni:V,li:B,_i:z,ui:x,di:h.width},vi:{si:R/o,fi:D/o,pi:E,mi:b}}}}class Q{constructor(t){this.Ci={ki:0,t:"#000",Mi:0,gi:0},this.Ti={Jt:"",Tt:!1,ai:!0,bi:!1,Vt:"",O:"#FFF",ci:!1,wi:!1},this.Pi={Jt:"",Tt:!1,ai:!1,bi:!0,Vt:"",O:"#FFF",ci:!0,wi:!0},this.bt=!0,this.Ri=new(t||J)(this.Ti,this.Ci),this.Di=new(t||J)(this.Pi,this.Ci)}Jt(){return this.Oi(),this.Ti.Jt}ki(){return this.Oi(),this.Ci.ki}gt(){this.bt=!0}Et(t,i=!1){return Math.max(this.Ri.Et(t,i),this.Di.Et(t,i))}Ai(){return this.Ci.yi||0}Bi(t){this.Ci.yi=t}Vi(){return this.Oi(),this.Ti.Tt||this.Pi.Tt}zi(){return this.Oi(),this.Ti.Tt}xt(t){return this.Oi(),this.Ti.ai=this.Ti.ai&&t.W().ticksVisible,this.Pi.ai=this.Pi.ai&&t.W().ticksVisible,this.Ri.it(this.Ti,this.Ci),this.Di.it(this.Pi,this.Ci),this.Ri}Ei(){return this.Oi(),this.Ri.it(this.Ti,this.Ci),this.Di.it(this.Pi,this.Ci),this.Di}Oi(){this.bt&&(this.Ti.ai=!0,this.Pi.ai=!1,this.Ii(this.Ti,this.Pi,this.Ci))}}class tt extends Q{constructor(t,i,n){super(),this.$t=t,this.Li=i,this.Ni=n}Ii(t,i,n){if(t.Tt=!1,2===this.$t.W().mode)return;const s=this.$t.W().horzLine;if(!s.labelVisible)return;const e=this.Li.Pt();if(!this.$t.Tt()||this.Li.Fi()||null===e)return;const r=y(s.labelBackgroundColor);n.t=r.t,t.O=r.i;const h=2/12*this.Li.P();n.gi=h,n.Mi=h;const l=this.Ni(this.Li);n.ki=l.ki,t.Jt=this.Li.Wi(l.ct,e),t.Tt=!0}}const it=/[1-9]/g;class nt{constructor(){this.Lt=null}it(t){this.Lt=t}K(t,i){if(null===this.Lt||!1===this.Lt.Tt||0===this.Lt.Jt.length)return;const n=t.useMediaCoordinateSpace((({context:t})=>(t.font=i.R,Math.round(i.ji.Si(t,f(this.Lt).Jt,it)))));if(n<=0)return;const s=i.Hi,e=n+2*s,r=e/2,h=this.Lt.$i;let l=this.Lt.ki,a=Math.floor(l-r)+.5;a<0?(l+=Math.abs(0-a),a=Math.floor(l-r)+.5):a+e>h&&(l-=Math.abs(h-(a+e)),a=Math.floor(l-r)+.5);const o=a+e,_=Math.ceil(0+i.C+i.T+i.L+i.P+i.B);t.useBitmapCoordinateSpace((({context:t,horizontalPixelRatio:n,verticalPixelRatio:s})=>{const e=f(this.Lt);t.fillStyle=e.t;const r=Math.round(a*n),h=Math.round(0*s),l=Math.round(o*n),u=Math.round(_*s),c=Math.round(2*n);if(t.beginPath(),t.moveTo(r,h),t.lineTo(r,u-c),t.arcTo(r,u,r+c,u,c),t.lineTo(l-c,u),t.arcTo(l,u,l,u-c,c),t.lineTo(l,h),t.fill(),e.ai){const r=Math.round(e.ki*n),l=h,a=Math.round((l+i.T)*s);t.fillStyle=e.O;const o=Math.max(1,Math.floor(n)),_=Math.floor(.5*n);t.fillRect(r-_,l,o,a-l)}})),t.useMediaCoordinateSpace((({context:t})=>{const n=f(this.Lt),e=0+i.C+i.T+i.L+i.P/2;t.font=i.R,t.textAlign="left",t.textBaseline="middle",t.fillStyle=n.O;const r=i.ji.xi(t,"Apr0");t.translate(a+s,e+r),t.fillText(n.Jt,0,0)}))}}class st{constructor(t,i,n){this.bt=!0,this.Ht=new nt,this.jt={Tt:!1,t:"#4c525e",O:"white",Jt:"",$i:0,ki:NaN,ai:!0},this.wt=t,this.Ui=i,this.Ni=n}gt(){this.bt=!0}xt(){return this.bt&&(this.St(),this.bt=!1),this.Ht.it(this.jt),this.Ht}St(){const t=this.jt;if(t.Tt=!1,2===this.wt.W().mode)return;const i=this.wt.W().vertLine;if(!i.labelVisible)return;const n=this.Ui.kt();if(n.Fi())return;t.$i=n.$i();const s=this.Ni();if(null===s)return;t.ki=s.ki;const e=n.qi(this.wt.yt());t.Jt=n.Yi(f(e)),t.Tt=!0;const r=y(i.labelBackgroundColor);t.t=r.t,t.O=r.i,t.ai=n.W().ticksVisible}}class et{constructor(){this.Xi=null,this.Ki=0}Zi(){return this.Ki}Gi(t){this.Ki=t}At(){return this.Xi}Ji(t){this.Xi=t}Qi(t){return[]}tn(){return[]}Tt(){return!0}}var rt;!function(t){t[t.Normal=0]="Normal",t[t.Magnet=1]="Magnet",t[t.Hidden=2]="Hidden"}(rt||(rt={}));class ht extends et{constructor(t,i){super(),this.nn=null,this.sn=NaN,this.en=0,this.rn=!0,this.hn=new Map,this.ln=!1,this.an=NaN,this.on=NaN,this._n=NaN,this.un=NaN,this.Ui=t,this.cn=i,this.dn=new j(t,this);this.fn=((t,i)=>n=>{const s=i(),e=t();if(n===f(this.nn).vn())return{ct:e,ki:s};{const t=f(n.Pt());return{ct:n.pn(s,t),ki:s}}})((()=>this.sn),(()=>this.on));const n=((t,i)=>()=>{const n=this.Ui.kt().mn(t()),s=i();return n&&Number.isFinite(s)?{ut:n,ki:s}:null})((()=>this.en),(()=>this.Kt()));this.bn=new st(this,t,n),this.wn=new U(this)}W(){return this.cn}gn(t,i){this._n=t,this.un=i}Mn(){this._n=NaN,this.un=NaN}xn(){return this._n}Sn(){return this.un}yn(t,i,n){this.ln||(this.ln=!0),this.rn=!0,this.kn(t,i,n)}yt(){return this.en}Kt(){return this.an}Zt(){return this.on}Tt(){return this.rn}Cn(){this.rn=!1,this.Tn(),this.sn=NaN,this.an=NaN,this.on=NaN,this.nn=null,this.Mn()}Pn(t){return null!==this.nn?[this.wn,this.dn]:[]}Yt(t){return t===this.nn&&this.cn.horzLine.visible}Xt(){return this.cn.vertLine.visible}Rn(t,i){this.rn&&this.nn===t||this.hn.clear();const n=[];return this.nn===t&&n.push(this.Dn(this.hn,i,this.fn)),n}tn(){return this.rn?[this.bn]:[]}Ut(){return this.nn}On(){this.wn.gt(),this.hn.forEach((t=>t.gt())),this.bn.gt(),this.dn.gt()}An(t){return t&&!t.vn().Fi()?t.vn():null}kn(t,i,n){this.Bn(t,i,n)&&this.On()}Bn(t,i,n){const s=this.an,e=this.on,r=this.sn,h=this.en,l=this.nn,a=this.An(n);this.en=t,this.an=isNaN(t)?NaN:this.Ui.kt().It(t),this.nn=n;const o=null!==a?a.Pt():null;return null!==a&&null!==o?(this.sn=i,this.on=a.Ot(i,o)):(this.sn=NaN,this.on=NaN),s!==this.an||e!==this.on||h!==this.en||r!==this.sn||l!==this.nn}Tn(){const t=this.Ui.Mt().map((t=>t.zn().Vn())).filter(A),i=0===t.length?null:Math.max(...t);this.en=null!==i?i:NaN}Dn(t,i,n){let s=t.get(i);return void 0===s&&(s=new tt(this,i,n),t.set(i,s)),s}}function lt(t){return"left"===t||"right"===t}class at{constructor(t){this.En=new Map,this.In=[],this.Ln=t}Nn(t,i){const n=function(t,i){return void 0===t?i:{Fn:Math.max(t.Fn,i.Fn),Wn:t.Wn||i.Wn}}(this.En.get(t),i);this.En.set(t,n)}jn(){return this.Ln}Hn(t){const i=this.En.get(t);return void 0===i?{Fn:this.Ln}:{Fn:Math.max(this.Ln,i.Fn),Wn:i.Wn}}$n(){this.Un(),this.In=[{qn:0}]}Yn(t){this.Un(),this.In=[{qn:1,Bt:t}]}Xn(t){this.Kn(),this.In.push({qn:5,Bt:t})}Un(){this.Kn(),this.In.push({qn:6})}Zn(){this.Un(),this.In=[{qn:4}]}Gn(t){this.Un(),this.In.push({qn:2,Bt:t})}Jn(t){this.Un(),this.In.push({qn:3,Bt:t})}Qn(){return this.In}ts(t){for(const i of t.In)this.ns(i);this.Ln=Math.max(this.Ln,t.Ln),t.En.forEach(((t,i)=>{this.Nn(i,t)}))}static ss(){return new at(2)}static es(){return new at(3)}ns(t){switch(t.qn){case 0:this.$n();break;case 1:this.Yn(t.Bt);break;case 2:this.Gn(t.Bt);break;case 3:this.Jn(t.Bt);break;case 4:this.Zn();break;case 5:this.Xn(t.Bt);break;case 6:this.Kn()}}Kn(){const t=this.In.findIndex((t=>5===t.qn));-1!==t&&this.In.splice(t,1)}}const ot=".";function _t(t,i){if(!T(t))return"n/a";if(!P(i))throw new TypeError("invalid length");if(i<0||i>16)throw new TypeError("invalid length");if(0===i)return t.toString();return("0000000000000000"+t.toString()).slice(-i)}class ut{constructor(t,i){if(i||(i=1),T(t)&&P(t)||(t=100),t<0)throw new TypeError("invalid base");this.Li=t,this.rs=i,this.hs()}format(t){const i=t<0?"−":"";return t=Math.abs(t),i+this.ls(t)}hs(){if(this.os=0,this.Li>0&&this.rs>0){let t=this.Li;for(;t>1;)t/=10,this.os++}}ls(t){const i=this.Li/this.rs;let n=Math.floor(t),s="";const e=void 0!==this.os?this.os:NaN;if(i>1){let r=+(Math.round(t*i)-n*i).toFixed(this.os);r>=i&&(r-=i,n+=1),s=ot+_t(+r.toFixed(this.os)*this.rs,e)}else n=Math.round(n*i)/i,e>0&&(s=ot+_t(0,e));return n.toFixed(0)+s}}class ct extends ut{constructor(t=100){super(t)}format(t){return`${super.format(t)}%`}}class dt{constructor(t){this._s=t}format(t){let i="";return t<0&&(i="-",t=-t),t<995?i+this.us(t):t<999995?i+this.us(t/1e3)+"K":t<999999995?(t=1e3*Math.round(t/1e3),i+this.us(t/1e6)+"M"):(t=1e6*Math.round(t/1e6),i+this.us(t/1e9)+"B")}us(t){let i;const n=Math.pow(10,this._s);return i=(t=Math.round(t*n)/n)>=1e-15&&t<1?t.toFixed(this._s).replace(/\.?0+$/,""):String(t),i.replace(/(\.[1-9]*)0+$/,((t,i)=>i))}}function ft(t,i,n,s,e,r,h){if(0===i.length||s.from>=i.length||s.to<=0)return;const{context:l,horizontalPixelRatio:a,verticalPixelRatio:o}=t,_=i[s.from];let u=r(t,_),c=_;if(s.to-s.from<2){const i=e/2;l.beginPath();const n={et:_.et-i,rt:_.rt},s={et:_.et+i,rt:_.rt};l.moveTo(n.et*a,n.rt*o),l.lineTo(s.et*a,s.rt*o),h(t,u,n,s)}else{const e=(i,n)=>{h(t,u,c,n),l.beginPath(),u=i,c=n};let d=c;l.beginPath(),l.moveTo(_.et*a,_.rt*o);for(let h=s.from+1;h=s.from;--n){const s=i[n];if(s){const i=e(t,s);i!==a&&(l.beginPath(),null!==a&&l.fill(),l.fillStyle=i,a=i);const n=Math.round(s.et*r)+o,u=s.rt*h;l.moveTo(n,u),l.arc(n,u,_,0,2*Math.PI)}}l.fill()}(t,i,l,n,o)}}class Pt extends Tt{Ds(t,i){return i._t}}function Rt(t,i,n,s,e=0,r=i.length){let h=r-e;for(;0>1,l=e+r;s(i[l],n)===t?(e=l+1,h-=r+1):h=r}return e}const Dt=Rt.bind(null,!0),Ot=Rt.bind(null,!1);function At(t,i){return t.ut0&&r=s&&(l=r-1),h>0&&hObject.assign(Object.assign({},t),this.Is.$s().Hs(t.ut))))}Us(){this.Es=null}Fs(){this.Bs&&(this.qs(),this.Bs=!1),this.Vs&&(this.js(),this.Vs=!1),this.As&&(this.Ys(),this.As=!1)}Ys(){const t=this.Is.At(),i=this.Ls.kt();if(this.Us(),i.Fi()||t.Fi())return;const n=i.Xs();if(null===n)return;if(0===this.Is.zn().Ks())return;const s=this.Is.Pt();null!==s&&(this.Es=Vt(this.zs,n,this.Ns),this.Zs(t,i,s.Bt),this.Gs())}}class Et extends zt{constructor(t,i){super(t,i,!0)}Zs(t,i,n){i.Js(this.zs,B(this.Es)),t.Qs(this.zs,n,B(this.Es))}te(t,i){return{ut:t,ct:i,et:NaN,rt:NaN}}qs(){const t=this.Is.$s();this.zs=this.Is.zn().ie().map((i=>{const n=i.Bt[3];return this.ne(i.se,n,t)}))}}class It extends Et{constructor(t,i){super(t,i),this.Ws=new I,this.ee=new kt,this.re=new Pt,this.Ws.X([this.ee,this.re])}ne(t,i,n){return Object.assign(Object.assign({},this.te(t,i)),n.Hs(t))}Gs(){const t=this.Is.W();this.ee.it({ds:t.lineType,st:this.zs,Wt:t.lineStyle,ht:t.lineWidth,fs:null,vs:t.invertFilledArea,nt:this.Es,cs:this.Ls.kt().he()}),this.re.it({ds:t.lineVisible?t.lineType:void 0,st:this.zs,Wt:t.lineStyle,ht:t.lineWidth,nt:this.Es,cs:this.Ls.kt().he(),Rs:t.pointMarkersVisible?t.pointMarkersRadius||t.lineWidth/2+2:void 0})}}class Lt extends H{constructor(){super(...arguments),this.Lt=null,this.le=0,this.ae=0}it(t){this.Lt=t}Z({context:t,horizontalPixelRatio:i,verticalPixelRatio:n}){if(null===this.Lt||0===this.Lt.zn.length||null===this.Lt.nt)return;if(this.le=this.oe(i),this.le>=2){Math.max(1,Math.floor(i))%2!=this.le%2&&this.le--}this.ae=this.Lt._e?Math.min(this.le,Math.floor(i)):this.le;let s=null;const e=this.ae<=this.le&&this.Lt.he>=Math.floor(1.5*i);for(let r=this.Lt.nt.from;rf+p-1&&(e=f+p-1,s=e-_+1),t.fillRect(i,s,o-i,e-s+1)}const i=a+m;let s=Math.max(f,Math.round(h.pe*n)-l),e=s+_-1;e>f+p-1&&(e=f+p-1,s=e-_+1),t.fillRect(u+1,s,i-u,e-s+1)}}}oe(t){const i=Math.floor(t);return Math.max(i,Math.floor(function(t,i){return Math.floor(.3*t*i)}(f(this.Lt).he,t)))}}class Nt extends zt{constructor(t,i){super(t,i,!1)}Zs(t,i,n){i.Js(this.zs,B(this.Es)),t.me(this.zs,n,B(this.Es))}be(t,i,n){return{ut:t,we:i.Bt[0],ge:i.Bt[1],Me:i.Bt[2],xe:i.Bt[3],et:NaN,ve:NaN,ce:NaN,de:NaN,pe:NaN}}qs(){const t=this.Is.$s();this.zs=this.Is.zn().ie().map((i=>this.ne(i.se,i,t)))}}class Ft extends Nt{constructor(){super(...arguments),this.Ws=new Lt}ne(t,i,n){return Object.assign(Object.assign({},this.be(t,i,n)),n.Hs(t))}Gs(){const t=this.Is.W();this.Ws.it({zn:this.zs,he:this.Ls.kt().he(),fe:t.openVisible,_e:t.thinBars,nt:this.Es})}}class Wt extends gt{constructor(){super(...arguments),this.Cs=new yt}ps(t,i){const n=this.tt;return this.Cs.bs(t,{gs:i.Se,Ms:i.ye,xs:i.ke,Ss:i.Ce,ys:t.bitmapSize.height,fs:n.fs})}}class jt extends Tt{constructor(){super(...arguments),this.Te=new yt}Ds(t,i){const n=this.tt;return this.Te.bs(t,{gs:i.Pe,Ms:i.Pe,xs:i.Re,Ss:i.Re,ys:t.bitmapSize.height,fs:n.fs})}}class Ht extends Et{constructor(t,i){super(t,i),this.Ws=new I,this.De=new Wt,this.Oe=new jt,this.Ws.X([this.De,this.Oe])}ne(t,i,n){return Object.assign(Object.assign({},this.te(t,i)),n.Hs(t))}Gs(){const t=this.Is.Pt();if(null===t)return;const i=this.Is.W(),n=this.Is.At().Ot(i.baseValue.price,t.Bt),s=this.Ls.kt().he();this.De.it({st:this.zs,ht:i.lineWidth,Wt:i.lineStyle,ds:i.lineType,fs:n,vs:!1,nt:this.Es,cs:s}),this.Oe.it({st:this.zs,ht:i.lineWidth,Wt:i.lineStyle,ds:i.lineVisible?i.lineType:void 0,Rs:i.pointMarkersVisible?i.pointMarkersRadius||i.lineWidth/2+2:void 0,fs:n,nt:this.Es,cs:s})}}class $t extends H{constructor(){super(...arguments),this.Lt=null,this.le=0}it(t){this.Lt=t}Z(t){if(null===this.Lt||0===this.Lt.zn.length||null===this.Lt.nt)return;const{horizontalPixelRatio:i}=t;if(this.le=function(t,i){if(t>=2.5&&t<=4)return Math.floor(3*i);const n=1-.2*Math.atan(Math.max(4,t)-4)/(.5*Math.PI),s=Math.floor(t*n*i),e=Math.floor(t*i),r=Math.min(s,e);return Math.max(Math.floor(i),r)}(this.Lt.he,i),this.le>=2){Math.floor(i)%2!=this.le%2&&this.le--}const n=this.Lt.zn;this.Lt.Ae&&this.Be(t,n,this.Lt.nt),this.Lt.ci&&this.Ve(t,n,this.Lt.nt);const s=this.ze(i);(!this.Lt.ci||this.le>2*s)&&this.Ee(t,n,this.Lt.nt)}Be(t,i,n){if(null===this.Lt)return;const{context:s,horizontalPixelRatio:e,verticalPixelRatio:r}=t;let h="",l=Math.min(Math.floor(e),Math.floor(this.Lt.he*e));l=Math.max(Math.floor(e),Math.min(l,this.le));const a=Math.floor(.5*l);let o=null;for(let t=n.from;t2*l)q(s,o,u,_-o+1,c-u+1,l);else{const t=_-o+1;s.fillRect(o,u,t,c-u+1)}a=_}}Ee(t,i,n){if(null===this.Lt)return;const{context:s,horizontalPixelRatio:e,verticalPixelRatio:r}=t;let h="";const l=this.ze(e);for(let t=n.from;to||s.fillRect(_,a,u-_+1,o-a+1)}}}class Ut extends Nt{constructor(){super(...arguments),this.Ws=new $t}ne(t,i,n){return Object.assign(Object.assign({},this.be(t,i,n)),n.Hs(t))}Gs(){const t=this.Is.W();this.Ws.it({zn:this.zs,he:this.Ls.kt().he(),Ae:t.wickVisible,ci:t.borderVisible,nt:this.Es})}}class qt{constructor(t,i){this.Ne=t,this.Li=i}K(t,i,n){this.Ne.draw(t,this.Li,i,n)}}class Yt extends zt{constructor(t,i,n){super(t,i,!1),this.wn=n,this.Ws=new qt(this.wn.renderer(),(i=>{const n=t.Pt();return null===n?null:t.At().Ot(i,n.Bt)}))}Fe(t){return this.wn.priceValueBuilder(t)}We(t){return this.wn.isWhitespace(t)}qs(){const t=this.Is.$s();this.zs=this.Is.zn().ie().map((i=>Object.assign(Object.assign({ut:i.se,et:NaN},t.Hs(i.se)),{je:i.He})))}Zs(t,i){i.Js(this.zs,B(this.Es))}Gs(){this.wn.update({bars:this.zs.map(Xt),barSpacing:this.Ls.kt().he(),visibleRange:this.Es},this.Is.W())}}function Xt(t){return{x:t.et,time:t.ut,originalData:t.je,barColor:t.ue}}class Kt extends H{constructor(){super(...arguments),this.Lt=null,this.$e=[]}it(t){this.Lt=t,this.$e=[]}Z({context:t,horizontalPixelRatio:i,verticalPixelRatio:n}){if(null===this.Lt||0===this.Lt.st.length||null===this.Lt.nt)return;this.$e.length||this.Ue(i);const s=Math.max(1,Math.floor(n)),e=Math.round(this.Lt.qe*n)-Math.floor(s/2),r=e+s;for(let i=this.Lt.nt.from;is.Xe?s.di=n.Os-i-1:n.Os=s.di+i+1))}let s=Math.ceil(this.Lt.he*t);for(let t=this.Lt.nt.from;t0&&s<4)for(let t=this.Lt.nt.from;ts&&(i.Ye>i.Xe?i.di-=1:i.Os+=1)}}}class Zt extends Et{constructor(){super(...arguments),this.Ws=new Kt}ne(t,i,n){return Object.assign(Object.assign({},this.te(t,i)),n.Hs(t))}Gs(){const t={st:this.zs,he:this.Ls.kt().he(),nt:this.Es,qe:this.Is.At().Ot(this.Is.W().base,f(this.Is.Pt()).Bt)};this.Ws.it(t)}}class Gt extends Et{constructor(){super(...arguments),this.Ws=new Pt}ne(t,i,n){return Object.assign(Object.assign({},this.te(t,i)),n.Hs(t))}Gs(){const t=this.Is.W(),i={st:this.zs,Wt:t.lineStyle,ds:t.lineVisible?t.lineType:void 0,ht:t.lineWidth,Rs:t.pointMarkersVisible?t.pointMarkersRadius||t.lineWidth/2+2:void 0,nt:this.Es,cs:this.Ls.kt().he()};this.Ws.it(i)}}const Jt=/[2-9]/g;class Qt{constructor(t=50){this.Ke=0,this.Ze=1,this.Ge=1,this.Je={},this.Qe=new Map,this.tr=t}ir(){this.Ke=0,this.Qe.clear(),this.Ze=1,this.Ge=1,this.Je={}}Si(t,i,n){return this.nr(t,i,n).width}xi(t,i,n){const s=this.nr(t,i,n);return((s.actualBoundingBoxAscent||0)-(s.actualBoundingBoxDescent||0))/2}nr(t,i,n){const s=n||Jt,e=String(i).replace(s,"0");if(this.Qe.has(e))return d(this.Qe.get(e)).sr;if(this.Ke===this.tr){const t=this.Je[this.Ge];delete this.Je[this.Ge],this.Qe.delete(t),this.Ge++,this.Ke--}t.save(),t.textBaseline="middle";const r=t.measureText(e);return t.restore(),0===r.width&&i.length||(this.Qe.set(e,{sr:r,er:this.Ze}),this.Je[this.Ze]=e,this.Ke++,this.Ze++),r}}class ti{constructor(t){this.rr=null,this.k=null,this.hr="right",this.lr=t}ar(t,i,n){this.rr=t,this.k=i,this.hr=n}K(t){null!==this.k&&null!==this.rr&&this.rr.K(t,this.k,this.lr,this.hr)}}class ii{constructor(t,i,n){this._r=t,this.lr=new Qt(50),this.ur=i,this.F=n,this.j=-1,this.Ht=new ti(this.lr)}xt(){const t=this.F.cr(this.ur);if(null===t)return null;const i=t.dr(this.ur)?t.vr():this.ur.At();if(null===i)return null;const n=t.pr(i);if("overlay"===n)return null;const s=this.F.mr();return s.P!==this.j&&(this.j=s.P,this.lr.ir()),this.Ht.ar(this._r.Ei(),s,n),this.Ht}}class ni extends H{constructor(){super(...arguments),this.Lt=null}it(t){this.Lt=t}br(t,i){var n;if(!(null===(n=this.Lt)||void 0===n?void 0:n.Tt))return null;const{rt:s,ht:e,wr:r}=this.Lt;return i>=s-e-7&&i<=s+e+7?{gr:this.Lt,wr:r}:null}Z({context:t,bitmapSize:i,horizontalPixelRatio:n,verticalPixelRatio:s}){if(null===this.Lt)return;if(!1===this.Lt.Tt)return;const e=Math.round(this.Lt.rt*s);e<0||e>i.height||(t.lineCap="butt",t.strokeStyle=this.Lt.O,t.lineWidth=Math.floor(this.Lt.ht*n),_(t,this.Lt.Wt),u(t,e,0,i.width))}}class si{constructor(t){this.Mr={rt:0,O:"rgba(0, 0, 0, 0)",ht:1,Wt:0,Tt:!1},this.Sr=new ni,this.bt=!0,this.Is=t,this.Ls=t.qt(),this.Sr.it(this.Mr)}gt(){this.bt=!0}xt(){return this.Is.Tt()?(this.bt&&(this.yr(),this.bt=!1),this.Sr):null}}class ei extends si{constructor(t){super(t)}yr(){this.Mr.Tt=!1;const t=this.Is.At(),i=t.kr().kr;if(2!==i&&3!==i)return;const n=this.Is.W();if(!n.baseLineVisible||!this.Is.Tt())return;const s=this.Is.Pt();null!==s&&(this.Mr.Tt=!0,this.Mr.rt=t.Ot(s.Bt,s.Bt),this.Mr.O=n.baseLineColor,this.Mr.ht=n.baseLineWidth,this.Mr.Wt=n.baseLineStyle)}}class ri extends H{constructor(){super(...arguments),this.Lt=null}it(t){this.Lt=t}He(){return this.Lt}Z({context:t,horizontalPixelRatio:i,verticalPixelRatio:n}){const s=this.Lt;if(null===s)return;const e=Math.max(1,Math.floor(i)),r=e%2/2,h=Math.round(s.Xe.x*i)+r,l=s.Xe.y*n;t.fillStyle=s.Cr,t.beginPath();const a=Math.max(2,1.5*s.Tr)*i;t.arc(h,l,a,0,2*Math.PI,!1),t.fill(),t.fillStyle=s.Pr,t.beginPath(),t.arc(h,l,s.ot*i,0,2*Math.PI,!1),t.fill(),t.lineWidth=e,t.strokeStyle=s.Rr,t.beginPath(),t.arc(h,l,s.ot*i+e/2,0,2*Math.PI,!1),t.stroke()}}const hi=[{Dr:0,Or:.25,Ar:4,Br:10,Vr:.25,zr:0,Er:.4,Ir:.8},{Dr:.25,Or:.525,Ar:10,Br:14,Vr:0,zr:0,Er:.8,Ir:0},{Dr:.525,Or:1,Ar:14,Br:14,Vr:0,zr:0,Er:0,Ir:0}];function li(t,i,n,s){return function(t,i){if("transparent"===t)return t;const n=S(t),s=n[3];return`rgba(${n[0]}, ${n[1]}, ${n[2]}, ${i*s})`}(t,n+(s-n)*i)}function ai(t,i){const n=t%2600/2600;let s;for(const t of hi)if(n>=t.Dr&&n<=t.Or){s=t;break}c(void 0!==s,"Last price animation internal logic error");const e=(n-s.Dr)/(s.Or-s.Dr);return{Pr:li(i,e,s.Vr,s.zr),Rr:li(i,e,s.Er,s.Ir),ot:(r=e,h=s.Ar,l=s.Br,h+(l-h)*r)};var r,h,l}class oi{constructor(t){this.Ht=new ri,this.bt=!0,this.Lr=!0,this.Nr=performance.now(),this.Fr=this.Nr-1,this.Wr=t}jr(){this.Fr=this.Nr-1,this.gt()}Hr(){if(this.gt(),2===this.Wr.W().lastPriceAnimation){const t=performance.now(),i=this.Fr-t;if(i>0)return void(i<650&&(this.Fr+=2600));this.Nr=t,this.Fr=t+2600}}gt(){this.bt=!0}$r(){this.Lr=!0}Tt(){return 0!==this.Wr.W().lastPriceAnimation}Ur(){switch(this.Wr.W().lastPriceAnimation){case 0:return!1;case 1:return!0;case 2:return performance.now()<=this.Fr}}xt(){return this.bt?(this.St(),this.bt=!1,this.Lr=!1):this.Lr&&(this.qr(),this.Lr=!1),this.Ht}St(){this.Ht.it(null);const t=this.Wr.qt().kt(),i=t.Xs(),n=this.Wr.Pt();if(null===i||null===n)return;const s=this.Wr.Yr(!0);if(s.Xr||!i.Kr(s.se))return;const e={x:t.It(s.se),y:this.Wr.At().Ot(s.ct,n.Bt)},r=s.O,h=this.Wr.W().lineWidth,l=ai(this.Zr(),r);this.Ht.it({Cr:r,Tr:h,Pr:l.Pr,Rr:l.Rr,ot:l.ot,Xe:e})}qr(){const t=this.Ht.He();if(null!==t){const i=ai(this.Zr(),t.Cr);t.Pr=i.Pr,t.Rr=i.Rr,t.ot=i.ot}}Zr(){return this.Ur()?performance.now()-this.Nr:2599}}function _i(t,i){return St(Math.min(Math.max(t,12),30)*i)}function ui(t,i){switch(t){case"arrowDown":case"arrowUp":return _i(i,1);case"circle":return _i(i,.8);case"square":return _i(i,.7)}}function ci(t){return function(t){const i=Math.ceil(t);return i%2!=0?i-1:i}(_i(t,1))}function di(t){return Math.max(_i(t,.1),3)}function fi(t,i,n,s,e){const r=ui("square",n),h=(r-1)/2,l=t-h,a=i-h;return s>=l&&s<=l+r&&e>=a&&e<=a+r}function vi(t,i,n,s,e){const r=(ui("arrowUp",e)-1)/2,h=(St(e/2)-1)/2;i.beginPath(),t?(i.moveTo(n-r,s),i.lineTo(n,s-r),i.lineTo(n+r,s),i.lineTo(n+h,s),i.lineTo(n+h,s+r),i.lineTo(n-h,s+r),i.lineTo(n-h,s)):(i.moveTo(n-r,s),i.lineTo(n,s+r),i.lineTo(n+r,s),i.lineTo(n+h,s),i.lineTo(n+h,s-r),i.lineTo(n-h,s-r),i.lineTo(n-h,s)),i.fill()}function pi(t,i,n,s,e,r){return fi(i,n,s,e,r)}class mi extends L{constructor(){super(...arguments),this.Lt=null,this.lr=new Qt,this.j=-1,this.H="",this.Gr=""}it(t){this.Lt=t}ar(t,i){this.j===t&&this.H===i||(this.j=t,this.H=i,this.Gr=z(t,i),this.lr.ir())}br(t,i){if(null===this.Lt||null===this.Lt.nt)return null;for(let n=this.Lt.nt.from;n=t&&e<=t+n&&r>=i-h&&r<=i+h}(t.Jt.et,t.Jt.rt,t.Jt.$i,t.Jt.Et,i,n))||function(t,i,n){if(0===t.Ks)return!1;switch(t.th){case"arrowDown":case"arrowUp":return pi(0,t.et,t.rt,t.Ks,i,n);case"circle":return function(t,i,n,s,e){const r=2+ui("circle",n)/2,h=t-s,l=i-e;return Math.sqrt(h*h+l*l)<=r}(t.et,t.rt,t.Ks,i,n);case"square":return fi(t.et,t.rt,t.Ks,i,n)}}(t,i,n)}function gi(t,i,n,s,e,r,h,l,a){const o=T(n)?n:n.xe,_=T(n)?n:n.ge,u=T(n)?n:n.Me,c=T(i.size)?Math.max(i.size,0):1,d=ci(l.he())*c,f=d/2;switch(t.Ks=d,i.position){case"inBar":return t.rt=h.Ot(o,a),void(void 0!==t.Jt&&(t.Jt.rt=t.rt+f+r+.6*e));case"aboveBar":return t.rt=h.Ot(_,a)-f-s.ih,void 0!==t.Jt&&(t.Jt.rt=t.rt-f-.6*e,s.ih+=1.2*e),void(s.ih+=d+r);case"belowBar":return t.rt=h.Ot(u,a)+f+s.nh,void 0!==t.Jt&&(t.Jt.rt=t.rt+f+r+.6*e,s.nh+=1.2*e),void(s.nh+=d+r)}i.position}class Mi{constructor(t,i){this.bt=!0,this.sh=!0,this.eh=!0,this.rh=null,this.Ht=new mi,this.Wr=t,this.Ui=i,this.Lt={st:[],nt:null}}gt(t){this.bt=!0,this.eh=!0,"data"===t&&(this.sh=!0)}xt(t){if(!this.Wr.Tt())return null;this.bt&&this.hh();const i=this.Ui.W().layout;return this.Ht.ar(i.fontSize,i.fontFamily),this.Ht.it(this.Lt),this.Ht}lh(){if(this.eh){if(this.Wr.ah().length>0){const t=this.Ui.kt().he(),i=di(t),n=1.5*ci(t)+2*i;this.rh={above:n,below:n}}else this.rh=null;this.eh=!1}return this.rh}hh(){const t=this.Wr.At(),i=this.Ui.kt(),n=this.Wr.ah();this.sh&&(this.Lt.st=n.map((t=>({ut:t.time,et:0,rt:0,Ks:0,th:t.shape,O:t.color,Jr:t.Jr,wr:t.id,Jt:void 0}))),this.sh=!1);const s=this.Ui.W().layout;this.Lt.nt=null;const e=i.Xs();if(null===e)return;const r=this.Wr.Pt();if(null===r)return;if(0===this.Lt.st.length)return;let h=NaN;const l=di(i.he()),a={ih:l,nh:l};this.Lt.nt=Vt(this.Lt.st,e,!0);for(let e=this.Lt.nt.from;e0&&(_.Jt={Qr:o.text,et:0,rt:0,$i:0,Et:0});const u=this.Wr.oh(o.time);null!==u&&gi(_,o,u,a,s.fontSize,l,t,i,r.Bt)}this.bt=!1}}class xi extends si{constructor(t){super(t)}yr(){const t=this.Mr;t.Tt=!1;const i=this.Is.W();if(!i.priceLineVisible||!this.Is.Tt())return;const n=this.Is.Yr(0===i.priceLineSource);n.Xr||(t.Tt=!0,t.rt=n.ki,t.O=this.Is._h(n.O),t.ht=i.priceLineWidth,t.Wt=i.priceLineStyle)}}class Si extends Q{constructor(t){super(),this.$t=t}Ii(t,i,n){t.Tt=!1,i.Tt=!1;const s=this.$t;if(!s.Tt())return;const e=s.W(),r=e.lastValueVisible,h=""!==s.uh(),l=0===e.seriesLastValueMode,a=s.Yr(!1);if(a.Xr)return;r&&(t.Jt=this.dh(a,r,l),t.Tt=0!==t.Jt.length),(h||l)&&(i.Jt=this.fh(a,r,h,l),i.Tt=i.Jt.length>0);const o=s._h(a.O),_=y(o);n.t=_.t,n.ki=a.ki,i.Vt=s.qt().zt(a.ki/s.At().Et()),t.Vt=o,t.O=_.i,i.O=_.i}fh(t,i,n,s){let e="";const r=this.$t.uh();return n&&0!==r.length&&(e+=`${r} `),i&&s&&(e+=this.$t.At().ph()?t.mh:t.bh),e.trim()}dh(t,i,n){return i?n?this.$t.At().ph()?t.bh:t.mh:t.Jt:""}}function yi(t,i,n,s){const e=Number.isFinite(i),r=Number.isFinite(n);return e&&r?t(i,n):e||r?e?i:n:s}class ki{constructor(t,i){this.wh=t,this.gh=i}Mh(t){return null!==t&&(this.wh===t.wh&&this.gh===t.gh)}xh(){return new ki(this.wh,this.gh)}Sh(){return this.wh}yh(){return this.gh}kh(){return this.gh-this.wh}Fi(){return this.gh===this.wh||Number.isNaN(this.gh)||Number.isNaN(this.wh)}ts(t){return null===t?this:new ki(yi(Math.min,this.Sh(),t.Sh(),-1/0),yi(Math.max,this.yh(),t.yh(),1/0))}Ch(t){if(!T(t))return;if(0===this.gh-this.wh)return;const i=.5*(this.gh+this.wh);let n=this.gh-i,s=this.wh-i;n*=t,s*=t,this.gh=i+n,this.wh=i+s}Th(t){T(t)&&(this.gh+=t,this.wh+=t)}Ph(){return{minValue:this.wh,maxValue:this.gh}}static Rh(t){return null===t?null:new ki(t.minValue,t.maxValue)}}class Ci{constructor(t,i){this.Dh=t,this.Oh=i||null}Ah(){return this.Dh}Bh(){return this.Oh}Ph(){return null===this.Dh?null:{priceRange:this.Dh.Ph(),margins:this.Oh||void 0}}static Rh(t){return null===t?null:new Ci(ki.Rh(t.priceRange),t.margins)}}class Ti extends si{constructor(t,i){super(t),this.Vh=i}yr(){const t=this.Mr;t.Tt=!1;const i=this.Vh.W();if(!this.Is.Tt()||!i.lineVisible)return;const n=this.Vh.zh();null!==n&&(t.Tt=!0,t.rt=n,t.O=i.color,t.ht=i.lineWidth,t.Wt=i.lineStyle,t.wr=this.Vh.W().id)}}class Pi extends Q{constructor(t,i){super(),this.Wr=t,this.Vh=i}Ii(t,i,n){t.Tt=!1,i.Tt=!1;const s=this.Vh.W(),e=s.axisLabelVisible,r=""!==s.title,h=this.Wr;if(!e||!h.Tt())return;const l=this.Vh.zh();if(null===l)return;r&&(i.Jt=s.title,i.Tt=!0),i.Vt=h.qt().zt(l/h.At().Et()),t.Jt=this.Eh(s.price),t.Tt=!0;const a=y(s.axisLabelColor||s.color);n.t=a.t;const o=s.axisLabelTextColor||a.i;t.O=o,i.O=o,n.ki=l}Eh(t){const i=this.Wr.Pt();return null===i?"":this.Wr.At().Wi(t,i.Bt)}}class Ri{constructor(t,i){this.Wr=t,this.cn=i,this.Ih=new Ti(t,this),this._r=new Pi(t,this),this.Lh=new ii(this._r,t,t.qt())}Nh(t){C(this.cn,t),this.gt(),this.Wr.qt().Fh()}W(){return this.cn}Wh(){return this.Ih}jh(){return this.Lh}Hh(){return this._r}gt(){this.Ih.gt(),this._r.gt()}zh(){const t=this.Wr,i=t.At();if(t.qt().kt().Fi()||i.Fi())return null;const n=t.Pt();return null===n?null:i.Ot(this.cn.price,n.Bt)}}class Di extends et{constructor(t){super(),this.Ui=t}qt(){return this.Ui}}const Oi={Bar:(t,i,n,s)=>{var e;const r=i.upColor,h=i.downColor,l=f(t(n,s)),a=v(l.Bt[0])<=v(l.Bt[3]);return{ue:null!==(e=l.O)&&void 0!==e?e:a?r:h}},Candlestick:(t,i,n,s)=>{var e,r,h;const l=i.upColor,a=i.downColor,o=i.borderUpColor,_=i.borderDownColor,u=i.wickUpColor,c=i.wickDownColor,d=f(t(n,s)),p=v(d.Bt[0])<=v(d.Bt[3]);return{ue:null!==(e=d.O)&&void 0!==e?e:p?l:a,Le:null!==(r=d.Vt)&&void 0!==r?r:p?o:_,Ie:null!==(h=d.$h)&&void 0!==h?h:p?u:c}},Custom:(t,i,n,s)=>{var e;return{ue:null!==(e=f(t(n,s)).O)&&void 0!==e?e:i.color}},Area:(t,i,n,s)=>{var e,r,h,l;const a=f(t(n,s));return{ue:null!==(e=a._t)&&void 0!==e?e:i.lineColor,_t:null!==(r=a._t)&&void 0!==r?r:i.lineColor,Ts:null!==(h=a.Ts)&&void 0!==h?h:i.topColor,Ps:null!==(l=a.Ps)&&void 0!==l?l:i.bottomColor}},Baseline:(t,i,n,s)=>{var e,r,h,l,a,o;const _=f(t(n,s));return{ue:_.Bt[3]>=i.baseValue.price?i.topLineColor:i.bottomLineColor,Pe:null!==(e=_.Pe)&&void 0!==e?e:i.topLineColor,Re:null!==(r=_.Re)&&void 0!==r?r:i.bottomLineColor,Se:null!==(h=_.Se)&&void 0!==h?h:i.topFillColor1,ye:null!==(l=_.ye)&&void 0!==l?l:i.topFillColor2,ke:null!==(a=_.ke)&&void 0!==a?a:i.bottomFillColor1,Ce:null!==(o=_.Ce)&&void 0!==o?o:i.bottomFillColor2}},Line:(t,i,n,s)=>{var e,r;const h=f(t(n,s));return{ue:null!==(e=h.O)&&void 0!==e?e:i.color,_t:null!==(r=h.O)&&void 0!==r?r:i.color}},Histogram:(t,i,n,s)=>{var e;return{ue:null!==(e=f(t(n,s)).O)&&void 0!==e?e:i.color}}};class Ai{constructor(t){this.Uh=(t,i)=>void 0!==i?i.Bt:this.Wr.zn().qh(t),this.Wr=t,this.Yh=Oi[t.Xh()]}Hs(t,i){return this.Yh(this.Uh,this.Wr.W(),t,i)}}var Bi;!function(t){t[t.NearestLeft=-1]="NearestLeft",t[t.None=0]="None",t[t.NearestRight=1]="NearestRight"}(Bi||(Bi={}));const Vi=30;class zi{constructor(){this.Kh=[],this.Zh=new Map,this.Gh=new Map}Jh(){return this.Ks()>0?this.Kh[this.Kh.length-1]:null}Qh(){return this.Ks()>0?this.tl(0):null}Vn(){return this.Ks()>0?this.tl(this.Kh.length-1):null}Ks(){return this.Kh.length}Fi(){return 0===this.Ks()}Kr(t){return null!==this.il(t,0)}qh(t){return this.nl(t)}nl(t,i=0){const n=this.il(t,i);return null===n?null:Object.assign(Object.assign({},this.sl(n)),{se:this.tl(n)})}ie(){return this.Kh}el(t,i,n){if(this.Fi())return null;let s=null;for(const e of n){s=Ei(s,this.rl(t,i,e))}return s}it(t){this.Gh.clear(),this.Zh.clear(),this.Kh=t}tl(t){return this.Kh[t].se}sl(t){return this.Kh[t]}il(t,i){const n=this.hl(t);if(null===n&&0!==i)switch(i){case-1:return this.ll(t);case 1:return this.al(t);default:throw new TypeError("Unknown search mode")}return n}ll(t){let i=this.ol(t);return i>0&&(i-=1),i!==this.Kh.length&&this.tl(i)t.set.se>i))}ul(t,i,n){let s=null;for(let e=t;es.dl&&(s.dl=t)))}return s}rl(t,i,n){if(this.Fi())return null;let s=null;const e=f(this.Qh()),r=f(this.Vn()),h=Math.max(t,e),l=Math.min(i,r),a=Math.ceil(h/Vi)*Vi,o=Math.max(a,Math.floor(l/Vi)*Vi);{const t=this.ol(h),e=this._l(Math.min(l,a,i));s=Ei(s,this.ul(t,e,n))}let _=this.Zh.get(n);void 0===_&&(_=new Map,this.Zh.set(n,_));for(let t=Math.max(a+1,h);tnew Li(t)));return this.gl={vl:e,pl:r},r}tn(){var t,i,n,s;const e=null!==(n=null===(i=(t=this.kl).timeAxisViews)||void 0===i?void 0:i.call(t))&&void 0!==n?n:[];if((null===(s=this.Ml)||void 0===s?void 0:s.vl)===e)return this.Ml.pl;const r=this.Wr.qt().kt(),h=e.map((t=>new Fi(t,r)));return this.Ml={vl:e,pl:h},h}Rn(){var t,i,n,s;const e=null!==(n=null===(i=(t=this.kl).priceAxisViews)||void 0===i?void 0:i.call(t))&&void 0!==n?n:[];if((null===(s=this.xl)||void 0===s?void 0:s.vl)===e)return this.xl.pl;const r=this.Wr.At(),h=e.map((t=>new Wi(t,r)));return this.xl={vl:e,pl:h},h}Tl(){var t,i,n,s;const e=null!==(n=null===(i=(t=this.kl).priceAxisPaneViews)||void 0===i?void 0:i.call(t))&&void 0!==n?n:[];if((null===(s=this.Sl)||void 0===s?void 0:s.vl)===e)return this.Sl.pl;const r=e.map((t=>new Li(t)));return this.Sl={vl:e,pl:r},r}Pl(){var t,i,n,s;const e=null!==(n=null===(i=(t=this.kl).timeAxisPaneViews)||void 0===i?void 0:i.call(t))&&void 0!==n?n:[];if((null===(s=this.yl)||void 0===s?void 0:s.vl)===e)return this.yl.pl;const r=e.map((t=>new Li(t)));return this.yl={vl:e,pl:r},r}Rl(t,i){var n,s,e;return null!==(e=null===(s=(n=this.kl).autoscaleInfo)||void 0===s?void 0:s.call(n,t,i))&&void 0!==e?e:null}br(t,i){var n,s,e;return null!==(e=null===(s=(n=this.kl).hitTest)||void 0===s?void 0:s.call(n,t,i))&&void 0!==e?e:null}}function Hi(t,i,n,s){t.forEach((t=>{i(t).forEach((t=>{t.ml()===n&&s.push(t)}))}))}function $i(t){return t.Pn()}function Ui(t){return t.Tl()}function qi(t){return t.Pl()}class Yi extends Di{constructor(t,i,n,s,e){super(t),this.Lt=new zi,this.Ih=new xi(this),this.Dl=[],this.Ol=new ei(this),this.Al=null,this.Bl=null,this.Vl=[],this.zl=[],this.El=null,this.Il=[],this.cn=i,this.Ll=n;const r=new Si(this);this.hn=[r],this.Lh=new ii(r,this,t),"Area"!==n&&"Line"!==n&&"Baseline"!==n||(this.Al=new oi(this)),this.Nl(),this.Fl(e)}S(){null!==this.El&&clearTimeout(this.El)}_h(t){return this.cn.priceLineColor||t}Yr(t){const i={Xr:!0},n=this.At();if(this.qt().kt().Fi()||n.Fi()||this.Lt.Fi())return i;const s=this.qt().kt().Xs(),e=this.Pt();if(null===s||null===e)return i;let r,h;if(t){const t=this.Lt.Jh();if(null===t)return i;r=t,h=t.se}else{const t=this.Lt.nl(s.di(),-1);if(null===t)return i;if(r=this.Lt.qh(t.se),null===r)return i;h=t.se}const l=r.Bt[3],a=this.$s().Hs(h,{Bt:r}),o=n.Ot(l,e.Bt);return{Xr:!1,ct:l,Jt:n.Wi(l,e.Bt),mh:n.Wl(l),bh:n.jl(l,e.Bt),O:a.ue,ki:o,se:h}}$s(){return null!==this.Bl||(this.Bl=new Ai(this)),this.Bl}W(){return this.cn}Nh(t){const i=t.priceScaleId;void 0!==i&&i!==this.cn.priceScaleId&&this.qt().Hl(this,i),C(this.cn,t),void 0!==t.priceFormat&&(this.Nl(),this.qt().$l()),this.qt().Ul(this),this.qt().ql(),this.wn.gt("options")}it(t,i){this.Lt.it(t),this.Yl(),this.wn.gt("data"),this.dn.gt("data"),null!==this.Al&&(i&&i.Xl?this.Al.Hr():0===t.length&&this.Al.jr());const n=this.qt().cr(this);this.qt().Kl(n),this.qt().Ul(this),this.qt().ql(),this.qt().Fh()}Zl(t){this.Vl=t,this.Yl();const i=this.qt().cr(this);this.dn.gt("data"),this.qt().Kl(i),this.qt().Ul(this),this.qt().ql(),this.qt().Fh()}Gl(){return this.Vl}ah(){return this.zl}Jl(t){const i=new Ri(this,t);return this.Dl.push(i),this.qt().Ul(this),i}Ql(t){const i=this.Dl.indexOf(t);-1!==i&&this.Dl.splice(i,1),this.qt().Ul(this)}Xh(){return this.Ll}Pt(){const t=this.ta();return null===t?null:{Bt:t.Bt[3],ia:t.ut}}ta(){const t=this.qt().kt().Xs();if(null===t)return null;const i=t.Os();return this.Lt.nl(i,1)}zn(){return this.Lt}oh(t){const i=this.Lt.qh(t);return null===i?null:"Bar"===this.Ll||"Candlestick"===this.Ll||"Custom"===this.Ll?{we:i.Bt[0],ge:i.Bt[1],Me:i.Bt[2],xe:i.Bt[3]}:i.Bt[3]}na(t){const i=[];Hi(this.Il,$i,"top",i);const n=this.Al;return null!==n&&n.Tt()?(null===this.El&&n.Ur()&&(this.El=setTimeout((()=>{this.El=null,this.qt().sa()}),0)),n.$r(),i.push(n),i):i}Pn(){const t=[];this.ea()||t.push(this.Ol),t.push(this.wn,this.Ih,this.dn);const i=this.Dl.map((t=>t.Wh()));return t.push(...i),Hi(this.Il,$i,"normal",t),t}ra(){return this.ha($i,"bottom")}la(t){return this.ha(Ui,t)}aa(t){return this.ha(qi,t)}oa(t,i){return this.Il.map((n=>n.br(t,i))).filter((t=>null!==t))}Qi(t){return[this.Lh,...this.Dl.map((t=>t.jh()))]}Rn(t,i){if(i!==this.Xi&&!this.ea())return[];const n=[...this.hn];for(const t of this.Dl)n.push(t.Hh());return this.Il.forEach((t=>{n.push(...t.Rn())})),n}tn(){const t=[];return this.Il.forEach((i=>{t.push(...i.tn())})),t}Rl(t,i){if(void 0!==this.cn.autoscaleInfoProvider){const n=this.cn.autoscaleInfoProvider((()=>{const n=this._a(t,i);return null===n?null:n.Ph()}));return Ci.Rh(n)}return this._a(t,i)}ua(){return this.cn.priceFormat.minMove}ca(){return this.da}On(){var t;this.wn.gt(),this.dn.gt();for(const t of this.hn)t.gt();for(const t of this.Dl)t.gt();this.Ih.gt(),this.Ol.gt(),null===(t=this.Al)||void 0===t||t.gt(),this.Il.forEach((t=>t.On()))}At(){return f(super.At())}Ct(t){if(!(("Line"===this.Ll||"Area"===this.Ll||"Baseline"===this.Ll)&&this.cn.crosshairMarkerVisible))return null;const i=this.Lt.qh(t);if(null===i)return null;return{ct:i.Bt[3],ot:this.fa(),Vt:this.va(),Dt:this.pa(),Rt:this.ma(t)}}uh(){return this.cn.title}Tt(){return this.cn.visible}ba(t){this.Il.push(new ji(t,this))}wa(t){this.Il=this.Il.filter((i=>i.Cl()!==t))}ga(){if(this.wn instanceof Yt!=!1)return t=>this.wn.Fe(t)}Ma(){if(this.wn instanceof Yt!=!1)return t=>this.wn.We(t)}ea(){return!lt(this.At().xa())}_a(t,i){if(!P(t)||!P(i)||this.Lt.Fi())return null;const n="Line"===this.Ll||"Area"===this.Ll||"Baseline"===this.Ll||"Histogram"===this.Ll?[3]:[2,1],s=this.Lt.el(t,i,n);let e=null!==s?new ki(s.cl,s.dl):null;if("Histogram"===this.Xh()){const t=this.cn.base,i=new ki(t,t);e=null!==e?e.ts(i):i}let r=this.dn.lh();return this.Il.forEach((n=>{const s=n.Rl(t,i);if(null==s?void 0:s.priceRange){const t=new ki(s.priceRange.minValue,s.priceRange.maxValue);e=null!==e?e.ts(t):t}var h,l,a,o;(null==s?void 0:s.margins)&&(h=r,l=s.margins,r={above:Math.max(null!==(a=null==h?void 0:h.above)&&void 0!==a?a:0,l.above),below:Math.max(null!==(o=null==h?void 0:h.below)&&void 0!==o?o:0,l.below)})})),new Ci(e,r)}fa(){switch(this.Ll){case"Line":case"Area":case"Baseline":return this.cn.crosshairMarkerRadius}return 0}va(){switch(this.Ll){case"Line":case"Area":case"Baseline":{const t=this.cn.crosshairMarkerBorderColor;if(0!==t.length)return t}}return null}pa(){switch(this.Ll){case"Line":case"Area":case"Baseline":return this.cn.crosshairMarkerBorderWidth}return 0}ma(t){switch(this.Ll){case"Line":case"Area":case"Baseline":{const t=this.cn.crosshairMarkerBackgroundColor;if(0!==t.length)return t}}return this.$s().Hs(t).ue}Nl(){switch(this.cn.priceFormat.type){case"custom":this.da={format:this.cn.priceFormat.formatter};break;case"volume":this.da=new dt(this.cn.priceFormat.precision);break;case"percent":this.da=new ct(this.cn.priceFormat.precision);break;default:{const t=Math.pow(10,this.cn.priceFormat.precision);this.da=new ut(t,this.cn.priceFormat.minMove*t)}}null!==this.Xi&&this.Xi.Sa()}Yl(){const t=this.qt().kt();if(!t.ya()||this.Lt.Fi())return void(this.zl=[]);const i=f(this.Lt.Qh());this.zl=this.Vl.map(((n,s)=>{const e=f(t.ka(n.time,!0)),r=et instanceof Yi)).reduce(((t,s)=>{if(n.dr(s)||!s.Tt())return t;const e=s.At(),r=s.zn();if(e.Fi()||!r.Kr(i))return t;const h=r.qh(i);if(null===h)return t;const l=v(s.Pt());return t.concat([e.Ot(h.Bt[3],l.Bt)])}),[]);if(0===l.length)return s;l.sort(((t,i)=>Math.abs(t-h)-Math.abs(i-h)));const a=l[0];return s=e.pn(a,r),s}}class Ki extends H{constructor(){super(...arguments),this.Lt=null}it(t){this.Lt=t}Z({context:t,bitmapSize:i,horizontalPixelRatio:n,verticalPixelRatio:s}){if(null===this.Lt)return;const e=Math.max(1,Math.floor(n));t.lineWidth=e,function(t,i){t.save(),t.lineWidth%2&&t.translate(.5,.5),i(),t.restore()}(t,(()=>{const r=f(this.Lt);if(r.Pa){t.strokeStyle=r.Ra,_(t,r.Da),t.beginPath();for(const s of r.Oa){const r=Math.round(s.Aa*n);t.moveTo(r,-e),t.lineTo(r,i.height+e)}t.stroke()}if(r.Ba){t.strokeStyle=r.Va,_(t,r.za),t.beginPath();for(const n of r.Ea){const r=Math.round(n.Aa*s);t.moveTo(-e,r),t.lineTo(i.width+e,r)}t.stroke()}}))}}class Zi{constructor(t){this.Ht=new Ki,this.bt=!0,this.nn=t}gt(){this.bt=!0}xt(){if(this.bt){const t=this.nn.qt().W().grid,i={Ba:t.horzLines.visible,Pa:t.vertLines.visible,Va:t.horzLines.color,Ra:t.vertLines.color,za:t.horzLines.style,Da:t.vertLines.style,Ea:this.nn.vn().Ia(),Oa:(this.nn.qt().kt().Ia()||[]).map((t=>({Aa:t.coord})))};this.Ht.it(i),this.bt=!1}return this.Ht}}class Gi{constructor(t){this.wn=new Zi(t)}Wh(){return this.wn}}const Ji={La:4,Na:1e-4};function Qi(t,i){const n=100*(t-i)/i;return i<0?-n:n}function tn(t,i){const n=Qi(t.Sh(),i),s=Qi(t.yh(),i);return new ki(n,s)}function nn(t,i){const n=100*(t-i)/i+100;return i<0?-n:n}function sn(t,i){const n=nn(t.Sh(),i),s=nn(t.yh(),i);return new ki(n,s)}function en(t,i){const n=Math.abs(t);if(n<1e-15)return 0;const s=Math.log10(n+i.Na)+i.La;return t<0?-s:s}function rn(t,i){const n=Math.abs(t);if(n<1e-15)return 0;const s=Math.pow(10,n-i.La)-i.Na;return t<0?-s:s}function hn(t,i){if(null===t)return null;const n=en(t.Sh(),i),s=en(t.yh(),i);return new ki(n,s)}function ln(t,i){if(null===t)return null;const n=rn(t.Sh(),i),s=rn(t.yh(),i);return new ki(n,s)}function an(t){if(null===t)return Ji;const i=Math.abs(t.yh()-t.Sh());if(i>=1||i<1e-15)return Ji;const n=Math.ceil(Math.abs(Math.log10(i))),s=Ji.La+n;return{La:s,Na:1/Math.pow(10,s)}}class on{constructor(t,i){if(this.Fa=t,this.Wa=i,function(t){if(t<0)return!1;for(let i=t;i>1;i/=10)if(i%10!=0)return!1;return!0}(this.Fa))this.ja=[2,2.5,2];else{this.ja=[];for(let t=this.Fa;1!==t;){if(t%2==0)this.ja.push(2),t/=2;else{if(t%5!=0)throw new Error("unexpected base");this.ja.push(2,2.5),t/=5}if(this.ja.length>100)throw new Error("something wrong with base")}}}Ha(t,i,n){const s=0===this.Fa?0:1/this.Fa;let e=Math.pow(10,Math.max(0,Math.ceil(Math.log10(t-i)))),r=0,h=this.Wa[0];for(;;){const t=xt(e,s,1e-14)&&e>s+1e-14,i=xt(e,n*h,1e-14),l=xt(e,1,1e-14);if(!(t&&i&&l))break;e/=h,h=this.Wa[++r%this.Wa.length]}if(e<=s+1e-14&&(e=s),e=Math.max(1,e),this.ja.length>0&&(l=e,a=1,o=1e-14,Math.abs(l-a)s+1e-14;)e/=h,h=this.ja[++r%this.ja.length];var l,a,o;return e}}class _n{constructor(t,i,n,s){this.$a=[],this.Li=t,this.Fa=i,this.Ua=n,this.qa=s}Ha(t,i){if(t=o?1:-1;let d=null,f=0;for(let n=a-u;n>o;n-=_){const s=this.qa(n,i,!0);null!==d&&Math.abs(s-d)l||(ff(t.Zi())-f(i.Zi())))}var cn;!function(t){t[t.Normal=0]="Normal",t[t.Logarithmic=1]="Logarithmic",t[t.Percentage=2]="Percentage",t[t.IndexedTo100=3]="IndexedTo100"}(cn||(cn={}));const dn=new ct,fn=new ut(100,1);class vn{constructor(t,i,n,s){this.Qa=0,this.io=null,this.Dh=null,this.no=null,this.so={eo:!1,ro:null},this.ho=0,this.lo=0,this.ao=new k,this.oo=new k,this._o=[],this.uo=null,this.co=null,this.do=null,this.fo=null,this.da=fn,this.vo=an(null),this.po=t,this.cn=i,this.mo=n,this.bo=s,this.wo=new _n(this,100,this.Mo.bind(this),this.xo.bind(this))}xa(){return this.po}W(){return this.cn}Nh(t){if(C(this.cn,t),this.Sa(),void 0!==t.mode&&this.So({kr:t.mode}),void 0!==t.scaleMargins){const i=d(t.scaleMargins.top),n=d(t.scaleMargins.bottom);if(i<0||i>1)throw new Error(`Invalid top margin - expect value between 0 and 1, given=${i}`);if(n<0||n>1||i+n>1)throw new Error(`Invalid bottom margin - expect value between 0 and 1, given=${n}`);if(i+n>1)throw new Error(`Invalid margins - sum of margins must be less than 1, given=${i+n}`);this.yo(),this.co=null}}ko(){return this.cn.autoScale}Ja(){return 1===this.cn.mode}ph(){return 2===this.cn.mode}Co(){return 3===this.cn.mode}kr(){return{Wn:this.cn.autoScale,To:this.cn.invertScale,kr:this.cn.mode}}So(t){const i=this.kr();let n=null;void 0!==t.Wn&&(this.cn.autoScale=t.Wn),void 0!==t.kr&&(this.cn.mode=t.kr,2!==t.kr&&3!==t.kr||(this.cn.autoScale=!0),this.so.eo=!1),1===i.kr&&t.kr!==i.kr&&(!function(t,i){if(null===t)return!1;const n=rn(t.Sh(),i),s=rn(t.yh(),i);return isFinite(n)&&isFinite(s)}(this.Dh,this.vo)?this.cn.autoScale=!0:(n=ln(this.Dh,this.vo),null!==n&&this.Po(n))),1===t.kr&&t.kr!==i.kr&&(n=hn(this.Dh,this.vo),null!==n&&this.Po(n));const s=i.kr!==this.cn.mode;s&&(2===i.kr||this.ph())&&this.Sa(),s&&(3===i.kr||this.Co())&&this.Sa(),void 0!==t.To&&i.To!==t.To&&(this.cn.invertScale=t.To,this.Ro()),this.oo.m(i,this.kr())}Do(){return this.oo}P(){return this.mo.fontSize}Et(){return this.Qa}Oo(t){this.Qa!==t&&(this.Qa=t,this.yo(),this.co=null)}Ao(){if(this.io)return this.io;const t=this.Et()-this.Bo()-this.Vo();return this.io=t,t}Ah(){return this.zo(),this.Dh}Po(t,i){const n=this.Dh;(i||null===n&&null!==t||null!==n&&!n.Mh(t))&&(this.co=null,this.Dh=t)}Fi(){return this.zo(),0===this.Qa||!this.Dh||this.Dh.Fi()}Eo(t){return this.To()?t:this.Et()-1-t}Ot(t,i){return this.ph()?t=Qi(t,i):this.Co()&&(t=nn(t,i)),this.xo(t,i)}Qs(t,i,n){this.zo();const s=this.Vo(),e=f(this.Ah()),r=e.Sh(),h=e.yh(),l=this.Ao()-1,a=this.To(),o=l/(h-r),_=void 0===n?0:n.from,u=void 0===n?t.length:n.to,c=this.Io();for(let n=_;nt.On()))}Sa(){this.co=null;const t=this.Jo();let i=100;null!==t&&(i=Math.round(1/t.ua())),this.da=fn,this.ph()?(this.da=dn,i=100):this.Co()?(this.da=new ut(100,1),i=100):null!==t&&(this.da=t.ca()),this.wo=new _n(this,i,this.Mo.bind(this),this.xo.bind(this)),this.wo.Xa()}Wo(){this.uo=null}Jo(){return this._o[0]||null}Bo(){return this.To()?this.cn.scaleMargins.bottom*this.Et()+this.lo:this.cn.scaleMargins.top*this.Et()+this.ho}Vo(){return this.To()?this.cn.scaleMargins.top*this.Et()+this.ho:this.cn.scaleMargins.bottom*this.Et()+this.lo}zo(){this.so.eo||(this.so.eo=!0,this.i_())}yo(){this.io=null}xo(t,i){if(this.zo(),this.Fi())return 0;t=this.Ja()&&t?en(t,this.vo):t;const n=f(this.Ah()),s=this.Vo()+(this.Ao()-1)*(t-n.Sh())/n.kh();return this.Eo(s)}Mo(t,i){if(this.zo(),this.Fi())return 0;const n=this.Eo(t),s=f(this.Ah()),e=s.Sh()+s.kh()*((n-this.Vo())/(this.Ao()-1));return this.Ja()?rn(e,this.vo):e}Ro(){this.co=null,this.wo.Xa()}i_(){const t=this.so.ro;if(null===t)return;let i=null;const n=this.Qo();let s=0,e=0;for(const r of n){if(!r.Tt())continue;const n=r.Pt();if(null===n)continue;const h=r.Rl(t.Os(),t.di());let l=h&&h.Ah();if(null!==l){switch(this.cn.mode){case 1:l=hn(l,this.vo);break;case 2:l=tn(l,n.Bt);break;case 3:l=sn(l,n.Bt)}if(i=null===i?l:i.ts(f(l)),null!==h){const t=h.Bh();null!==t&&(s=Math.max(s,t.above),e=Math.max(s,t.below))}}}if(s===this.ho&&e===this.lo||(this.ho=s,this.lo=e,this.co=null,this.yo()),null!==i){if(i.Sh()===i.yh()){const t=this.Jo(),n=5*(null===t||this.ph()||this.Co()?1:t.ua());this.Ja()&&(i=ln(i,this.vo)),i=new ki(i.Sh()-n,i.yh()+n),this.Ja()&&(i=hn(i,this.vo))}if(this.Ja()){const t=ln(i,this.vo),n=an(t);if(r=n,h=this.vo,r.La!==h.La||r.Na!==h.Na){const s=null!==this.no?ln(this.no,this.vo):null;this.vo=n,i=hn(t,n),null!==s&&(this.no=hn(s,n))}}this.Po(i)}else null===this.Dh&&(this.Po(new ki(-.5,.5)),this.vo=an(null));var r,h;this.so.eo=!0}Io(){return this.ph()?Qi:this.Co()?nn:this.Ja()?t=>en(t,this.vo):null}n_(t,i,n){return void 0===i?(void 0===n&&(n=this.ca()),n.format(t)):i(t)}Eh(t,i){return this.n_(t,this.bo.priceFormatter,i)}Go(t,i){return this.n_(t,this.bo.percentageFormatter,i)}}class pn{constructor(t,i){this._o=[],this.s_=new Map,this.Qa=0,this.e_=0,this.r_=1e3,this.uo=null,this.h_=new k,this.wl=t,this.Ui=i,this.l_=new Gi(this);const n=i.W();this.a_=this.o_("left",n.leftPriceScale),this.__=this.o_("right",n.rightPriceScale),this.a_.Do().l(this.u_.bind(this,this.a_),this),this.__.Do().l(this.u_.bind(this,this.__),this),this.c_(n)}c_(t){if(t.leftPriceScale&&this.a_.Nh(t.leftPriceScale),t.rightPriceScale&&this.__.Nh(t.rightPriceScale),t.localization&&(this.a_.Sa(),this.__.Sa()),t.overlayPriceScales){const i=Array.from(this.s_.values());for(const n of i){const i=f(n[0].At());i.Nh(t.overlayPriceScales),t.localization&&i.Sa()}}}d_(t){switch(t){case"left":return this.a_;case"right":return this.__}return this.s_.has(t)?d(this.s_.get(t))[0].At():null}S(){this.qt().f_().p(this),this.a_.Do().p(this),this.__.Do().p(this),this._o.forEach((t=>{t.S&&t.S()})),this.h_.m()}v_(){return this.r_}p_(t){this.r_=t}qt(){return this.Ui}$i(){return this.e_}Et(){return this.Qa}m_(t){this.e_=t,this.b_()}Oo(t){this.Qa=t,this.a_.Oo(t),this.__.Oo(t),this._o.forEach((i=>{if(this.dr(i)){const n=i.At();null!==n&&n.Oo(t)}})),this.b_()}Ta(){return this._o}dr(t){const i=t.At();return null===i||this.a_!==i&&this.__!==i}Fo(t,i,n){const s=void 0!==n?n:this.g_().w_+1;this.M_(t,i,s)}jo(t){const i=this._o.indexOf(t);c(-1!==i,"removeDataSource: invalid data source"),this._o.splice(i,1);const n=f(t.At()).xa();if(this.s_.has(n)){const i=d(this.s_.get(n)),s=i.indexOf(t);-1!==s&&(i.splice(s,1),0===i.length&&this.s_.delete(n))}const s=t.At();s&&s.Ta().indexOf(t)>=0&&s.jo(t),null!==s&&(s.Wo(),this.x_(s)),this.uo=null}pr(t){return t===this.a_?"left":t===this.__?"right":"overlay"}S_(){return this.a_}y_(){return this.__}k_(t,i){t.Uo(i)}C_(t,i){t.qo(i),this.b_()}T_(t){t.Yo()}P_(t,i){t.Xo(i)}R_(t,i){t.Ko(i),this.b_()}D_(t){t.Zo()}b_(){this._o.forEach((t=>{t.On()}))}vn(){let t=null;return this.Ui.W().rightPriceScale.visible&&0!==this.__.Ta().length?t=this.__:this.Ui.W().leftPriceScale.visible&&0!==this.a_.Ta().length?t=this.a_:0!==this._o.length&&(t=this._o[0].At()),null===t&&(t=this.__),t}vr(){let t=null;return this.Ui.W().rightPriceScale.visible?t=this.__:this.Ui.W().leftPriceScale.visible&&(t=this.a_),t}x_(t){null!==t&&t.ko()&&this.O_(t)}A_(t){const i=this.wl.Xs();t.So({Wn:!0}),null!==i&&t.t_(i),this.b_()}B_(){this.O_(this.a_),this.O_(this.__)}V_(){this.x_(this.a_),this.x_(this.__),this._o.forEach((t=>{this.dr(t)&&this.x_(t.At())})),this.b_(),this.Ui.Fh()}No(){return null===this.uo&&(this.uo=un(this._o)),this.uo}z_(){return this.h_}E_(){return this.l_}O_(t){const i=t.Qo();if(i&&i.length>0&&!this.wl.Fi()){const i=this.wl.Xs();null!==i&&t.t_(i)}t.On()}g_(){const t=this.No();if(0===t.length)return{I_:0,w_:0};let i=0,n=0;for(let s=0;sn&&(n=e))}return{I_:i,w_:n}}M_(t,i,n){let s=this.d_(i);if(null===s&&(s=this.o_(i,this.Ui.W().overlayPriceScales)),this._o.push(t),!lt(i)){const n=this.s_.get(i)||[];n.push(t),this.s_.set(i,n)}s.Fo(t),t.Ji(s),t.Gi(n),this.x_(s),this.uo=null}u_(t,i,n){i.kr!==n.kr&&this.O_(t)}o_(t,i){const n=Object.assign({visible:!0,autoScale:!0},O(i)),s=new vn(t,n,this.Ui.W().layout,this.Ui.W().localization);return s.Oo(this.Et()),s}}class mn{constructor(t,i,n=50){this.Ke=0,this.Ze=1,this.Ge=1,this.Qe=new Map,this.Je=new Map,this.L_=t,this.N_=i,this.tr=n}F_(t){const i=t.time,n=this.N_.cacheKey(i),s=this.Qe.get(n);if(void 0!==s)return s.W_;if(this.Ke===this.tr){const t=this.Je.get(this.Ge);this.Je.delete(this.Ge),this.Qe.delete(d(t)),this.Ge++,this.Ke--}const e=this.L_(t);return this.Qe.set(n,{W_:e,er:this.Ze}),this.Je.set(this.Ze,n),this.Ke++,this.Ze++,e}}class bn{constructor(t,i){c(t<=i,"right should be >= left"),this.j_=t,this.H_=i}Os(){return this.j_}di(){return this.H_}U_(){return this.H_-this.j_+1}Kr(t){return this.j_<=t&&t<=this.H_}Mh(t){return this.j_===t.Os()&&this.H_===t.di()}}function wn(t,i){return null===t||null===i?t===i:t.Mh(i)}class gn{constructor(){this.q_=new Map,this.Qe=null,this.Y_=!1}X_(t){this.Y_=t,this.Qe=null}K_(t,i){this.Z_(i),this.Qe=null;for(let n=i;n{t<=n[0].index?i.push(s):n.splice(Dt(n,t,(i=>i.indexi-t))){if(!this.q_.get(n))continue;const s=i;i=[];const e=s.length;let r=0;const h=d(this.q_.get(n)),l=h.length;let a=1/0,o=-1/0;for(let n=0;n=t&&_-o>=t)i.push(l),o=_;else if(this.Y_)return s}for(;ri.weight?t:i}class Sn{constructor(t,i,n,s){this.e_=0,this.eu=null,this.ru=[],this.fo=null,this.do=null,this.hu=new gn,this.lu=new Map,this.au=Mn.su(),this.ou=!0,this._u=new k,this.uu=new k,this.cu=new k,this.du=null,this.fu=null,this.vu=[],this.cn=i,this.bo=n,this.pu=i.rightOffset,this.mu=i.barSpacing,this.Ui=t,this.N_=s,this.bu(),this.hu.X_(i.uniformDistribution)}W(){return this.cn}wu(t){C(this.bo,t),this.gu(),this.bu()}Nh(t,i){var n;C(this.cn,t),this.cn.fixLeftEdge&&this.Mu(),this.cn.fixRightEdge&&this.xu(),void 0!==t.barSpacing&&this.Ui.Gn(t.barSpacing),void 0!==t.rightOffset&&this.Ui.Jn(t.rightOffset),void 0!==t.minBarSpacing&&this.Ui.Gn(null!==(n=t.barSpacing)&&void 0!==n?n:this.mu),this.gu(),this.bu(),this.cu.m()}mn(t){var i,n;return null!==(n=null===(i=this.ru[t])||void 0===i?void 0:i.time)&&void 0!==n?n:null}qi(t){var i;return null!==(i=this.ru[t])&&void 0!==i?i:null}ka(t,i){if(this.ru.length<1)return null;if(this.N_.key(t)>this.N_.key(this.ru[this.ru.length-1].time))return i?this.ru.length-1:null;const n=Dt(this.ru,this.N_.key(t),((t,i)=>this.N_.key(t.time)0}Xs(){return this.Su(),this.au.iu()}yu(){return this.Su(),this.au.nu()}ku(){const t=this.Xs();if(null===t)return null;const i={from:t.Os(),to:t.di()};return this.Cu(i)}Cu(t){const i=Math.round(t.from),n=Math.round(t.to),s=f(this.Tu()),e=f(this.Pu());return{from:f(this.qi(Math.max(s,i))),to:f(this.qi(Math.min(e,n)))}}Ru(t){return{from:f(this.ka(t.from,!0)),to:f(this.ka(t.to,!0))}}$i(){return this.e_}m_(t){if(!isFinite(t)||t<=0)return;if(this.e_===t)return;const i=this.yu(),n=this.e_;if(this.e_=t,this.ou=!0,this.cn.lockVisibleTimeRangeOnResize&&0!==n){const i=this.mu*t/n;this.mu=i}if(this.cn.fixLeftEdge&&null!==i&&i.Os()<=0){const i=n-t;this.pu-=Math.round(i/this.mu)+1,this.ou=!0}this.Du(),this.Ou()}It(t){if(this.Fi()||!P(t))return 0;const i=this.Au()+this.pu-t;return this.e_-(i+.5)*this.mu-1}Js(t,i){const n=this.Au(),s=void 0===i?0:i.from,e=void 0===i?t.length:i.to;for(let i=s;ii/2&&!o?n.needAlignCoordinate=!1:n.needAlignCoordinate=_&&t.index<=l||u&&t.index>=a,c++}return this.vu.length=c,this.fu=this.vu,this.vu}Fu(){this.ou=!0,this.Gn(this.cn.barSpacing),this.Jn(this.cn.rightOffset)}Wu(t){this.ou=!0,this.eu=t,this.Ou(),this.Mu()}ju(t,i){const n=this.Vu(t),s=this.he(),e=s+i*(s/10);this.Gn(e),this.cn.rightBarStaysOnScroll||this.Jn(this.Iu()+(n-this.Vu(t)))}Uo(t){this.fo&&this.Zo(),null===this.do&&null===this.du&&(this.Fi()||(this.do=t,this.Hu()))}qo(t){if(null===this.du)return;const i=Mt(this.e_-t,0,this.e_),n=Mt(this.e_-f(this.do),0,this.e_);0!==i&&0!==n&&this.Gn(this.du.he*i/n)}Yo(){null!==this.do&&(this.do=null,this.$u())}Xo(t){null===this.fo&&null===this.du&&(this.Fi()||(this.fo=t,this.Hu()))}Ko(t){if(null===this.fo)return;const i=(this.fo-t)/this.he();this.pu=f(this.du).Iu+i,this.ou=!0,this.Ou()}Zo(){null!==this.fo&&(this.fo=null,this.$u())}Uu(){this.qu(this.cn.rightOffset)}qu(t,i=400){if(!isFinite(t))throw new RangeError("offset is required and must be finite number");if(!isFinite(i)||i<=0)throw new RangeError("animationDuration (optional) must be finite positive number");const n=this.pu,s=performance.now();this.Ui.Xn({Yu:t=>(t-s)/i>=1,Xu:e=>{const r=(e-s)/i;return r>=1?t:n+(t-n)*r}})}gt(t,i){this.ou=!0,this.ru=t,this.hu.K_(t,i),this.Ou()}Ku(){return this._u}Zu(){return this.uu}Gu(){return this.cu}Au(){return this.eu||0}Ju(t){const i=t.U_();this.Eu(this.e_/i),this.pu=t.di()-this.Au(),this.Ou(),this.ou=!0,this.Ui.zu(),this.Ui.Fh()}Qu(){const t=this.Tu(),i=this.Pu();null!==t&&null!==i&&this.Ju(new bn(t,i+this.cn.rightOffset))}tc(t){const i=new bn(t.from,t.to);this.Ju(i)}Yi(t){return void 0!==this.bo.timeFormatter?this.bo.timeFormatter(t.originalTime):this.N_.formatHorzItem(t.time)}Lu(){const{handleScroll:t,handleScale:i}=this.Ui.W();return!(t.horzTouchDrag||t.mouseWheel||t.pressedMouseMove||t.vertTouchDrag||i.axisDoubleClickReset.time||i.axisPressedMouseMove.time||i.mouseWheel||i.pinch)}Tu(){return 0===this.ru.length?null:0}Pu(){return 0===this.ru.length?null:this.ru.length-1}ic(t){return(this.e_-1-t)/this.mu}Vu(t){const i=this.ic(t),n=this.Au()+this.pu-i;return Math.round(1e6*n)/1e6}Eu(t){const i=this.mu;this.mu=t,this.Du(),i!==this.mu&&(this.ou=!0,this.nc())}Su(){if(!this.ou)return;if(this.ou=!1,this.Fi())return void this.sc(Mn.su());const t=this.Au(),i=this.e_/this.mu,n=this.pu+t,s=new bn(n-i+1,n);this.sc(new Mn(s))}Du(){const t=this.ec();if(this.mut&&(this.mu=t,this.ou=!0)}}ec(){return this.cn.fixLeftEdge&&this.cn.fixRightEdge&&0!==this.ru.length?this.e_/this.ru.length:this.cn.minBarSpacing}Ou(){const t=this.rc();this.pu>t&&(this.pu=t,this.ou=!0);const i=this.hc();null!==i&&this.puthis.lc(t)),this.N_),this.lu.set(t.weight,i)),i.F_(t)}lc(t){return this.N_.formatTickmark(t,this.bo)}sc(t){const i=this.au;this.au=t,wn(i.iu(),this.au.iu())||this._u.m(),wn(i.nu(),this.au.nu())||this.uu.m(),this.nc()}nc(){this.fu=null}gu(){this.nc(),this.lu.clear()}bu(){this.N_.updateFormatter(this.bo)}Mu(){if(!this.cn.fixLeftEdge)return;const t=this.Tu();if(null===t)return;const i=this.Xs();if(null===i)return;const n=i.Os()-t;if(n<0){const t=this.pu-n-1;this.Jn(t)}this.Du()}xu(){this.Ou(),this.Du()}}class yn extends L{constructor(t){super(),this.ac=new Map,this.Lt=t}Z(t){}J(t){if(!this.Lt.Tt)return;const{context:i,mediaSize:n}=t;let s=0;for(const t of this.Lt.oc){if(0===t.Jt.length)continue;i.font=t.R;const e=this._c(i,t.Jt);e>n.width?t.ju=n.width/e:t.ju=1,s+=t.uc*t.ju}let e=0;switch(this.Lt.cc){case"top":e=0;break;case"center":e=Math.max((n.height-s)/2,0);break;case"bottom":e=Math.max(n.height-s,0)}i.fillStyle=this.Lt.O;for(const t of this.Lt.oc){i.save();let s=0;switch(this.Lt.dc){case"left":i.textAlign="left",s=t.uc/2;break;case"center":i.textAlign="center",s=n.width/2;break;case"right":i.textAlign="right",s=n.width-1-t.uc/2}i.translate(s,e),i.textBaseline="top",i.font=t.R,i.scale(t.ju,t.ju),i.fillText(t.Jt,0,t.fc),i.restore(),e+=t.uc*t.ju}}_c(t,i){const n=this.vc(t.font);let s=n.get(i);return void 0===s&&(s=t.measureText(i).width,n.set(i,s)),s}vc(t){let i=this.ac.get(t);return void 0===i&&(i=new Map,this.ac.set(t,i)),i}}class kn{constructor(t){this.bt=!0,this.jt={Tt:!1,O:"",oc:[],cc:"center",dc:"center"},this.Ht=new yn(this.jt),this.$t=t}gt(){this.bt=!0}xt(){return this.bt&&(this.St(),this.bt=!1),this.Ht}St(){const t=this.$t.W(),i=this.jt;i.Tt=t.visible,i.Tt&&(i.O=t.color,i.dc=t.horzAlign,i.cc=t.vertAlign,i.oc=[{Jt:t.text,R:z(t.fontSize,t.fontFamily,t.fontStyle),uc:1.2*t.fontSize,fc:0,ju:0}])}}class Cn extends et{constructor(t,i){super(),this.cn=i,this.wn=new kn(this)}Rn(){return[]}Pn(){return[this.wn]}W(){return this.cn}On(){this.wn.gt()}}var Tn,Pn,Rn,Dn,On;!function(t){t[t.OnTouchEnd=0]="OnTouchEnd",t[t.OnNextTap=1]="OnNextTap"}(Tn||(Tn={}));class An{constructor(t,i,n){this.mc=[],this.bc=[],this.e_=0,this.wc=null,this.gc=new k,this.Mc=new k,this.xc=null,this.Sc=t,this.cn=i,this.N_=n,this.yc=new E(this),this.wl=new Sn(this,i.timeScale,this.cn.localization,n),this.wt=new ht(this,i.crosshair),this.kc=new Xi(i.crosshair),this.Cc=new Cn(this,i.watermark),this.Tc(),this.mc[0].p_(2e3),this.Pc=this.Rc(0),this.Dc=this.Rc(1)}$l(){this.Oc(at.es())}Fh(){this.Oc(at.ss())}sa(){this.Oc(new at(1))}Ul(t){const i=this.Ac(t);this.Oc(i)}Bc(){return this.wc}Vc(t){const i=this.wc;this.wc=t,null!==i&&this.Ul(i.zc),null!==t&&this.Ul(t.zc)}W(){return this.cn}Nh(t){C(this.cn,t),this.mc.forEach((i=>i.c_(t))),void 0!==t.timeScale&&this.wl.Nh(t.timeScale),void 0!==t.localization&&this.wl.wu(t.localization),(t.leftPriceScale||t.rightPriceScale)&&this.gc.m(),this.Pc=this.Rc(0),this.Dc=this.Rc(1),this.$l()}Ec(t,i){if("left"===t)return void this.Nh({leftPriceScale:i});if("right"===t)return void this.Nh({rightPriceScale:i});const n=this.Ic(t);null!==n&&(n.At.Nh(i),this.gc.m())}Ic(t){for(const i of this.mc){const n=i.d_(t);if(null!==n)return{Ut:i,At:n}}return null}kt(){return this.wl}Lc(){return this.mc}Nc(){return this.Cc}Fc(){return this.wt}Wc(){return this.Mc}jc(t,i){t.Oo(i),this.zu()}m_(t){this.e_=t,this.wl.m_(this.e_),this.mc.forEach((i=>i.m_(t))),this.zu()}Tc(t){const i=new pn(this.wl,this);void 0!==t?this.mc.splice(t,0,i):this.mc.push(i);const n=void 0===t?this.mc.length-1:t,s=at.es();return s.Nn(n,{Fn:0,Wn:!0}),this.Oc(s),i}k_(t,i,n){t.k_(i,n)}C_(t,i,n){t.C_(i,n),this.ql(),this.Oc(this.Hc(t,2))}T_(t,i){t.T_(i),this.Oc(this.Hc(t,2))}P_(t,i,n){i.ko()||t.P_(i,n)}R_(t,i,n){i.ko()||(t.R_(i,n),this.ql(),this.Oc(this.Hc(t,2)))}D_(t,i){i.ko()||(t.D_(i),this.Oc(this.Hc(t,2)))}A_(t,i){t.A_(i),this.Oc(this.Hc(t,2))}$c(t){this.wl.Uo(t)}Uc(t,i){const n=this.kt();if(n.Fi()||0===i)return;const s=n.$i();t=Math.max(1,Math.min(t,s)),n.ju(t,i),this.zu()}qc(t){this.Yc(0),this.Xc(t),this.Kc()}Zc(t){this.wl.qo(t),this.zu()}Gc(){this.wl.Yo(),this.Fh()}Yc(t){this.wl.Xo(t)}Xc(t){this.wl.Ko(t),this.zu()}Kc(){this.wl.Zo(),this.Fh()}Mt(){return this.bc}Jc(t,i,n,s,e){this.wt.gn(t,i);let r=NaN,h=this.wl.Bu(t);const l=this.wl.Xs();null!==l&&(h=Math.min(Math.max(l.Os(),h),l.di()));const a=s.vn(),o=a.Pt();null!==o&&(r=a.pn(i,o)),r=this.kc.Ca(r,h,s),this.wt.yn(h,r,s),this.sa(),e||this.Mc.m(this.wt.yt(),{x:t,y:i},n)}Qc(t,i,n){const s=n.vn(),e=s.Pt(),r=s.Ot(t,f(e)),h=this.wl.ka(i,!0),l=this.wl.It(f(h));this.Jc(l,r,null,n,!0)}td(t){this.Fc().Cn(),this.sa(),t||this.Mc.m(null,null,null)}ql(){const t=this.wt.Ut();if(null!==t){const i=this.wt.xn(),n=this.wt.Sn();this.Jc(i,n,null,t)}this.wt.On()}nd(t,i,n){const s=this.wl.mn(0);void 0!==i&&void 0!==n&&this.wl.gt(i,n);const e=this.wl.mn(0),r=this.wl.Au(),h=this.wl.Xs();if(null!==h&&null!==s&&null!==e){const i=h.Kr(r),l=this.N_.key(s)>this.N_.key(e),a=null!==t&&t>r&&!l,o=this.wl.W().allowShiftVisibleRangeOnWhitespaceReplacement,_=i&&(!(void 0===n)||o)&&this.wl.W().shiftVisibleRangeOnNewBar;if(a&&!_){const i=t-r;this.wl.Jn(this.wl.Iu()-i)}}this.wl.Wu(t)}Kl(t){null!==t&&t.V_()}cr(t){const i=this.mc.find((i=>i.No().includes(t)));return void 0===i?null:i}zu(){this.Cc.On(),this.mc.forEach((t=>t.V_())),this.ql()}S(){this.mc.forEach((t=>t.S())),this.mc.length=0,this.cn.localization.priceFormatter=void 0,this.cn.localization.percentageFormatter=void 0,this.cn.localization.timeFormatter=void 0}sd(){return this.yc}mr(){return this.yc.W()}f_(){return this.gc}ed(t,i,n){const s=this.mc[0],e=this.rd(i,t,s,n);return this.bc.push(e),1===this.bc.length?this.$l():this.Fh(),e}hd(t){const i=this.cr(t),n=this.bc.indexOf(t);c(-1!==n,"Series not found"),this.bc.splice(n,1),f(i).jo(t),t.S&&t.S()}Hl(t,i){const n=f(this.cr(t));n.jo(t);const s=this.Ic(i);if(null===s){const s=t.Zi();n.Fo(t,i,s)}else{const e=s.Ut===n?t.Zi():void 0;s.Ut.Fo(t,i,e)}}Qu(){const t=at.ss();t.$n(),this.Oc(t)}ld(t){const i=at.ss();i.Yn(t),this.Oc(i)}Zn(){const t=at.ss();t.Zn(),this.Oc(t)}Gn(t){const i=at.ss();i.Gn(t),this.Oc(i)}Jn(t){const i=at.ss();i.Jn(t),this.Oc(i)}Xn(t){const i=at.ss();i.Xn(t),this.Oc(i)}Un(){const t=at.ss();t.Un(),this.Oc(t)}ad(){return this.cn.rightPriceScale.visible?"right":"left"}od(){return this.Dc}q(){return this.Pc}zt(t){const i=this.Dc,n=this.Pc;if(i===n)return i;if(t=Math.max(0,Math.min(100,Math.round(100*t))),null===this.xc||this.xc.Ts!==n||this.xc.Ps!==i)this.xc={Ts:n,Ps:i,_d:new Map};else{const i=this.xc._d.get(t);if(void 0!==i)return i}const s=function(t,i,n){const[s,e,r,h]=S(t),[l,a,o,_]=S(i),u=[m(s+n*(l-s)),m(e+n*(a-e)),m(r+n*(o-r)),b(h+n*(_-h))];return`rgba(${u[0]}, ${u[1]}, ${u[2]}, ${u[3]})`}(n,i,t/100);return this.xc._d.set(t,s),s}Hc(t,i){const n=new at(i);if(null!==t){const s=this.mc.indexOf(t);n.Nn(s,{Fn:i})}return n}Ac(t,i){return void 0===i&&(i=2),this.Hc(this.cr(t),i)}Oc(t){this.Sc&&this.Sc(t),this.mc.forEach((t=>t.E_().Wh().gt()))}rd(t,i,n,s){const e=new Yi(this,t,i,n,s),r=void 0!==t.priceScaleId?t.priceScaleId:this.ad();return n.Fo(e,r),lt(r)||e.Nh(t),e}Rc(t){const i=this.cn.layout;return"gradient"===i.background.type?0===t?i.background.topColor:i.background.bottomColor:i.background.color}}function Bn(t){return!T(t)&&!R(t)}function Vn(t){return T(t)}!function(t){t[t.Disabled=0]="Disabled",t[t.Continuous=1]="Continuous",t[t.OnDataUpdate=2]="OnDataUpdate"}(Pn||(Pn={})),function(t){t[t.LastBar=0]="LastBar",t[t.LastVisible=1]="LastVisible"}(Rn||(Rn={})),function(t){t.Solid="solid",t.VerticalGradient="gradient"}(Dn||(Dn={})),function(t){t[t.Year=0]="Year",t[t.Month=1]="Month",t[t.DayOfMonth=2]="DayOfMonth",t[t.Time=3]="Time",t[t.TimeWithSeconds=4]="TimeWithSeconds"}(On||(On={}));const zn=t=>t.getUTCFullYear();function En(t,i,n){return i.replace(/yyyy/g,(t=>_t(zn(t),4))(t)).replace(/yy/g,(t=>_t(zn(t)%100,2))(t)).replace(/MMMM/g,((t,i)=>new Date(t.getUTCFullYear(),t.getUTCMonth(),1).toLocaleString(i,{month:"long"}))(t,n)).replace(/MMM/g,((t,i)=>new Date(t.getUTCFullYear(),t.getUTCMonth(),1).toLocaleString(i,{month:"short"}))(t,n)).replace(/MM/g,(t=>_t((t=>t.getUTCMonth()+1)(t),2))(t)).replace(/dd/g,(t=>_t((t=>t.getUTCDate())(t),2))(t))}class In{constructor(t="yyyy-MM-dd",i="default"){this.ud=t,this.dd=i}F_(t){return En(t,this.ud,this.dd)}}class Ln{constructor(t){this.fd=t||"%h:%m:%s"}F_(t){return this.fd.replace("%h",_t(t.getUTCHours(),2)).replace("%m",_t(t.getUTCMinutes(),2)).replace("%s",_t(t.getUTCSeconds(),2))}}const Nn={vd:"yyyy-MM-dd",pd:"%h:%m:%s",md:" ",bd:"default"};class Fn{constructor(t={}){const i=Object.assign(Object.assign({},Nn),t);this.wd=new In(i.vd,i.bd),this.gd=new Ln(i.pd),this.Md=i.md}F_(t){return`${this.wd.F_(t)}${this.Md}${this.gd.F_(t)}`}}function Wn(t){return 60*t*60*1e3}function jn(t){return 60*t*1e3}const Hn=[{xd:($n=1,1e3*$n),Sd:10},{xd:jn(1),Sd:20},{xd:jn(5),Sd:21},{xd:jn(30),Sd:22},{xd:Wn(1),Sd:30},{xd:Wn(3),Sd:31},{xd:Wn(6),Sd:32},{xd:Wn(12),Sd:33}];var $n;function Un(t,i){if(t.getUTCFullYear()!==i.getUTCFullYear())return 70;if(t.getUTCMonth()!==i.getUTCMonth())return 60;if(t.getUTCDate()!==i.getUTCDate())return 50;for(let n=Hn.length-1;n>=0;--n)if(Math.floor(i.getTime()/Hn[n].xd)!==Math.floor(t.getTime()/Hn[n].xd))return Hn[n].Sd;return 0}function qn(t){let i=t;if(R(t)&&(i=Xn(t)),!Bn(i))throw new Error("time must be of type BusinessDay");const n=new Date(Date.UTC(i.year,i.month-1,i.day,0,0,0,0));return{yd:Math.round(n.getTime()/1e3),kd:i}}function Yn(t){if(!Vn(t))throw new Error("time must be of type isUTCTimestamp");return{yd:t}}function Xn(t){const i=new Date(t);if(isNaN(i.getTime()))throw new Error(`Invalid date string=${t}, expected format=yyyy-mm-dd`);return{day:i.getUTCDate(),month:i.getUTCMonth()+1,year:i.getUTCFullYear()}}function Kn(t){R(t.time)&&(t.time=Xn(t.time))}class Zn{options(){return this.cn}setOptions(t){this.cn=t,this.updateFormatter(t.localization)}preprocessData(t){Array.isArray(t)?function(t){t.forEach(Kn)}(t):Kn(t)}createConverterToInternalObj(t){return f(function(t){return 0===t.length?null:Bn(t[0].time)||R(t[0].time)?qn:Yn}(t))}key(t){return"object"==typeof t&&"yd"in t?t.yd:this.key(this.convertHorzItemToInternal(t))}cacheKey(t){const i=t;return void 0===i.kd?new Date(1e3*i.yd).getTime():new Date(Date.UTC(i.kd.year,i.kd.month-1,i.kd.day)).getTime()}convertHorzItemToInternal(t){return Vn(i=t)?Yn(i):Bn(i)?qn(i):qn(Xn(i));var i}updateFormatter(t){if(!this.cn)return;const i=t.dateFormat;this.cn.timeScale.timeVisible?this.Cd=new Fn({vd:i,pd:this.cn.timeScale.secondsVisible?"%h:%m:%s":"%h:%m",md:"   ",bd:t.locale}):this.Cd=new In(i,t.locale)}formatHorzItem(t){const i=t;return this.Cd.F_(new Date(1e3*i.yd))}formatTickmark(t,i){const n=function(t,i,n){switch(t){case 0:case 10:return i?n?4:3:2;case 20:case 21:case 22:case 30:case 31:case 32:case 33:return i?3:2;case 50:return 2;case 60:return 1;case 70:return 0}}(t.weight,this.cn.timeScale.timeVisible,this.cn.timeScale.secondsVisible),s=this.cn.timeScale;if(void 0!==s.tickMarkFormatter){const e=s.tickMarkFormatter(t.originalTime,n,i.locale);if(null!==e)return e}return function(t,i,n){const s={};switch(i){case 0:s.year="numeric";break;case 1:s.month="short";break;case 2:s.day="numeric";break;case 3:s.hour12=!1,s.hour="2-digit",s.minute="2-digit";break;case 4:s.hour12=!1,s.hour="2-digit",s.minute="2-digit",s.second="2-digit"}const e=void 0===t.kd?new Date(1e3*t.yd):new Date(Date.UTC(t.kd.year,t.kd.month-1,t.kd.day));return new Date(e.getUTCFullYear(),e.getUTCMonth(),e.getUTCDate(),e.getUTCHours(),e.getUTCMinutes(),e.getUTCSeconds(),e.getUTCMilliseconds()).toLocaleString(n,s)}(t.time,n,i.locale)}maxTickMarkWeight(t){let i=t.reduce(xn,t[0]).weight;return i>30&&i<50&&(i=30),i}fillWeightsForPoints(t,i){!function(t,i=0){if(0===t.length)return;let n=0===i?null:t[i-1].time.yd,s=null!==n?new Date(1e3*n):null,e=0;for(let r=i;r1){const i=Math.ceil(e/(t.length-1)),n=new Date(1e3*(t[0].time.yd-i));t[0].timeWeight=Un(new Date(1e3*t[0].time.yd),n)}}(t,i)}static Td(t){return C({localization:{dateFormat:"dd MMM 'yy"}},null!=t?t:{})}}function Gn(t){var i=t.width,n=t.height;if(i<0)throw new Error("Negative width is not allowed for Size");if(n<0)throw new Error("Negative height is not allowed for Size");return{width:i,height:n}}function Jn(t,i){return t.width===i.width&&t.height===i.height}var Qn=function(){function t(t){var i=this;this._resolutionListener=function(){return i._onResolutionChanged()},this._resolutionMediaQueryList=null,this._observers=[],this._window=t,this._installResolutionListener()}return t.prototype.dispose=function(){this._uninstallResolutionListener(),this._window=null},Object.defineProperty(t.prototype,"value",{get:function(){return this._window.devicePixelRatio},enumerable:!1,configurable:!0}),t.prototype.subscribe=function(t){var i=this,n={next:t};return this._observers.push(n),{unsubscribe:function(){i._observers=i._observers.filter((function(t){return t!==n}))}}},t.prototype._installResolutionListener=function(){if(null!==this._resolutionMediaQueryList)throw new Error("Resolution listener is already installed");var t=this._window.devicePixelRatio;this._resolutionMediaQueryList=this._window.matchMedia("all and (resolution: ".concat(t,"dppx)")),this._resolutionMediaQueryList.addListener(this._resolutionListener)},t.prototype._uninstallResolutionListener=function(){null!==this._resolutionMediaQueryList&&(this._resolutionMediaQueryList.removeListener(this._resolutionListener),this._resolutionMediaQueryList=null)},t.prototype._reinstallResolutionListener=function(){this._uninstallResolutionListener(),this._installResolutionListener()},t.prototype._onResolutionChanged=function(){var t=this;this._observers.forEach((function(i){return i.next(t._window.devicePixelRatio)})),this._reinstallResolutionListener()},t}();var ts=function(){function t(t,i,n){var s;this._canvasElement=null,this._bitmapSizeChangedListeners=[],this._suggestedBitmapSize=null,this._suggestedBitmapSizeChangedListeners=[],this._devicePixelRatioObservable=null,this._canvasElementResizeObserver=null,this._canvasElement=t,this._canvasElementClientSize=Gn({width:this._canvasElement.clientWidth,height:this._canvasElement.clientHeight}),this._transformBitmapSize=null!=i?i:function(t){return t},this._allowResizeObserver=null===(s=null==n?void 0:n.allowResizeObserver)||void 0===s||s,this._chooseAndInitObserver()}return t.prototype.dispose=function(){var t,i;if(null===this._canvasElement)throw new Error("Object is disposed");null===(t=this._canvasElementResizeObserver)||void 0===t||t.disconnect(),this._canvasElementResizeObserver=null,null===(i=this._devicePixelRatioObservable)||void 0===i||i.dispose(),this._devicePixelRatioObservable=null,this._suggestedBitmapSizeChangedListeners.length=0,this._bitmapSizeChangedListeners.length=0,this._canvasElement=null},Object.defineProperty(t.prototype,"canvasElement",{get:function(){if(null===this._canvasElement)throw new Error("Object is disposed");return this._canvasElement},enumerable:!1,configurable:!0}),Object.defineProperty(t.prototype,"canvasElementClientSize",{get:function(){return this._canvasElementClientSize},enumerable:!1,configurable:!0}),Object.defineProperty(t.prototype,"bitmapSize",{get:function(){return Gn({width:this.canvasElement.width,height:this.canvasElement.height})},enumerable:!1,configurable:!0}),t.prototype.resizeCanvasElement=function(t){this._canvasElementClientSize=Gn(t),this.canvasElement.style.width="".concat(this._canvasElementClientSize.width,"px"),this.canvasElement.style.height="".concat(this._canvasElementClientSize.height,"px"),this._invalidateBitmapSize()},t.prototype.subscribeBitmapSizeChanged=function(t){this._bitmapSizeChangedListeners.push(t)},t.prototype.unsubscribeBitmapSizeChanged=function(t){this._bitmapSizeChangedListeners=this._bitmapSizeChangedListeners.filter((function(i){return i!==t}))},Object.defineProperty(t.prototype,"suggestedBitmapSize",{get:function(){return this._suggestedBitmapSize},enumerable:!1,configurable:!0}),t.prototype.subscribeSuggestedBitmapSizeChanged=function(t){this._suggestedBitmapSizeChangedListeners.push(t)},t.prototype.unsubscribeSuggestedBitmapSizeChanged=function(t){this._suggestedBitmapSizeChangedListeners=this._suggestedBitmapSizeChangedListeners.filter((function(i){return i!==t}))},t.prototype.applySuggestedBitmapSize=function(){if(null!==this._suggestedBitmapSize){var t=this._suggestedBitmapSize;this._suggestedBitmapSize=null,this._resizeBitmap(t),this._emitSuggestedBitmapSizeChanged(t,this._suggestedBitmapSize)}},t.prototype._resizeBitmap=function(t){var i=this.bitmapSize;Jn(i,t)||(this.canvasElement.width=t.width,this.canvasElement.height=t.height,this._emitBitmapSizeChanged(i,t))},t.prototype._emitBitmapSizeChanged=function(t,i){var n=this;this._bitmapSizeChangedListeners.forEach((function(s){return s.call(n,t,i)}))},t.prototype._suggestNewBitmapSize=function(t){var i=this._suggestedBitmapSize,n=Gn(this._transformBitmapSize(t,this._canvasElementClientSize)),s=Jn(this.bitmapSize,n)?null:n;null===i&&null===s||null!==i&&null!==s&&Jn(i,s)||(this._suggestedBitmapSize=s,this._emitSuggestedBitmapSizeChanged(i,s))},t.prototype._emitSuggestedBitmapSizeChanged=function(t,i){var n=this;this._suggestedBitmapSizeChangedListeners.forEach((function(s){return s.call(n,t,i)}))},t.prototype._chooseAndInitObserver=function(){var t=this;this._allowResizeObserver?new Promise((function(t){var i=new ResizeObserver((function(n){t(n.every((function(t){return"devicePixelContentBoxSize"in t}))),i.disconnect()}));i.observe(document.body,{box:"device-pixel-content-box"})})).catch((function(){return!1})).then((function(i){return i?t._initResizeObserver():t._initDevicePixelRatioObservable()})):this._initDevicePixelRatioObservable()},t.prototype._initDevicePixelRatioObservable=function(){var t=this;if(null!==this._canvasElement){var i=is(this._canvasElement);if(null===i)throw new Error("No window is associated with the canvas");this._devicePixelRatioObservable=function(t){return new Qn(t)}(i),this._devicePixelRatioObservable.subscribe((function(){return t._invalidateBitmapSize()})),this._invalidateBitmapSize()}},t.prototype._invalidateBitmapSize=function(){var t,i;if(null!==this._canvasElement){var n=is(this._canvasElement);if(null!==n){var s=null!==(i=null===(t=this._devicePixelRatioObservable)||void 0===t?void 0:t.value)&&void 0!==i?i:n.devicePixelRatio,e=this._canvasElement.getClientRects(),r=void 0!==e[0]?function(t,i){return Gn({width:Math.round(t.left*i+t.width*i)-Math.round(t.left*i),height:Math.round(t.top*i+t.height*i)-Math.round(t.top*i)})}(e[0],s):Gn({width:this._canvasElementClientSize.width*s,height:this._canvasElementClientSize.height*s});this._suggestNewBitmapSize(r)}}},t.prototype._initResizeObserver=function(){var t=this;null!==this._canvasElement&&(this._canvasElementResizeObserver=new ResizeObserver((function(i){var n=i.find((function(i){return i.target===t._canvasElement}));if(n&&n.devicePixelContentBoxSize&&n.devicePixelContentBoxSize[0]){var s=n.devicePixelContentBoxSize[0],e=Gn({width:s.inlineSize,height:s.blockSize});t._suggestNewBitmapSize(e)}})),this._canvasElementResizeObserver.observe(this._canvasElement,{box:"device-pixel-content-box"}))},t}();function is(t){return t.ownerDocument.defaultView}var ns=function(){function t(t,i,n){if(0===i.width||0===i.height)throw new TypeError("Rendering target could only be created on a media with positive width and height");if(this._mediaSize=i,0===n.width||0===n.height)throw new TypeError("Rendering target could only be created using a bitmap with positive integer width and height");this._bitmapSize=n,this._context=t}return t.prototype.useMediaCoordinateSpace=function(t){try{return this._context.save(),this._context.setTransform(1,0,0,1,0,0),this._context.scale(this._horizontalPixelRatio,this._verticalPixelRatio),t({context:this._context,mediaSize:this._mediaSize})}finally{this._context.restore()}},t.prototype.useBitmapCoordinateSpace=function(t){try{return this._context.save(),this._context.setTransform(1,0,0,1,0,0),t({context:this._context,mediaSize:this._mediaSize,bitmapSize:this._bitmapSize,horizontalPixelRatio:this._horizontalPixelRatio,verticalPixelRatio:this._verticalPixelRatio})}finally{this._context.restore()}},Object.defineProperty(t.prototype,"_horizontalPixelRatio",{get:function(){return this._bitmapSize.width/this._mediaSize.width},enumerable:!1,configurable:!0}),Object.defineProperty(t.prototype,"_verticalPixelRatio",{get:function(){return this._bitmapSize.height/this._mediaSize.height},enumerable:!1,configurable:!0}),t}();function ss(t,i){var n=t.canvasElementClientSize;if(0===n.width||0===n.height)return null;var s=t.bitmapSize;if(0===s.width||0===s.height)return null;var e=t.canvasElement.getContext("2d",i);return null===e?null:new ns(e,n,s)}const es="undefined"!=typeof window;function rs(){return!!es&&window.navigator.userAgent.toLowerCase().indexOf("firefox")>-1}function hs(){return!!es&&/iPhone|iPad|iPod/.test(window.navigator.platform)}function ls(t){return t+t%2}function as(t,i){return t.Pd-i.Pd}function os(t,i,n){const s=(t.Pd-i.Pd)/(t.ut-i.ut);return Math.sign(s)*Math.min(Math.abs(s),n)}class _s{constructor(t,i,n,s){this.Rd=null,this.Dd=null,this.Od=null,this.Ad=null,this.Bd=null,this.Vd=0,this.zd=0,this.Ed=t,this.Id=i,this.Ld=n,this.rs=s}Nd(t,i){if(null!==this.Rd){if(this.Rd.ut===i)return void(this.Rd.Pd=t);if(Math.abs(this.Rd.Pd-t)50)return;let n=0;const s=os(this.Rd,this.Dd,this.Id),e=as(this.Rd,this.Dd),r=[s],h=[e];if(n+=e,null!==this.Od){const t=os(this.Dd,this.Od,this.Id);if(Math.sign(t)===Math.sign(s)){const i=as(this.Dd,this.Od);if(r.push(t),h.push(i),n+=i,null!==this.Ad){const t=os(this.Od,this.Ad,this.Id);if(Math.sign(t)===Math.sign(s)){const i=as(this.Od,this.Ad);r.push(t),h.push(i),n+=i}}}}let l=0;for(let t=0;t({width:Math.max(t.width,i.width),height:Math.max(t.height,i.height)})});return s.resizeCanvasElement(i),s}function cs(t,i,n,s){t.G&&t.G(i,n,s)}function ds(t,i,n,s){t.K(i,n,s)}function fs(t,i,n,s){const e=t(n,s);for(const t of e){const n=t.xt();null!==n&&i(n)}}function vs(t){es&&void 0!==window.chrome&&t.addEventListener("mousedown",(t=>{if(1===t.button)return t.preventDefault(),!1}))}class ps{constructor(t,i,n){this.Wd=0,this.jd=null,this.Hd={et:Number.NEGATIVE_INFINITY,rt:Number.POSITIVE_INFINITY},this.$d=0,this.Ud=null,this.qd={et:Number.NEGATIVE_INFINITY,rt:Number.POSITIVE_INFINITY},this.Yd=null,this.Xd=!1,this.Kd=null,this.Zd=null,this.Gd=!1,this.Jd=!1,this.Qd=!1,this.tf=null,this.if=null,this.nf=null,this.sf=null,this.ef=null,this.rf=null,this.hf=null,this.lf=0,this.af=!1,this._f=!1,this.uf=!1,this.cf=0,this.df=null,this.ff=!hs(),this.vf=t=>{this.pf(t)},this.mf=t=>{if(this.bf(t)){const i=this.wf(t);if(++this.$d,this.Ud&&this.$d>1){const{gf:n}=this.Mf(ws(t),this.qd);n<30&&!this.Qd&&this.xf(i,this.yf.Sf),this.kf()}}else{const i=this.wf(t);if(++this.Wd,this.jd&&this.Wd>1){const{gf:n}=this.Mf(ws(t),this.Hd);n<5&&!this.Jd&&this.Cf(i,this.yf.Tf),this.Pf()}}},this.Rf=t,this.yf=i,this.cn=n,this.Df()}S(){null!==this.tf&&(this.tf(),this.tf=null),null!==this.if&&(this.if(),this.if=null),null!==this.sf&&(this.sf(),this.sf=null),null!==this.ef&&(this.ef(),this.ef=null),null!==this.rf&&(this.rf(),this.rf=null),null!==this.nf&&(this.nf(),this.nf=null),this.Of(),this.Pf()}Af(t){this.sf&&this.sf();const i=this.Bf.bind(this);if(this.sf=()=>{this.Rf.removeEventListener("mousemove",i)},this.Rf.addEventListener("mousemove",i),this.bf(t))return;const n=this.wf(t);this.Cf(n,this.yf.Vf),this.ff=!0}Pf(){null!==this.jd&&clearTimeout(this.jd),this.Wd=0,this.jd=null,this.Hd={et:Number.NEGATIVE_INFINITY,rt:Number.POSITIVE_INFINITY}}kf(){null!==this.Ud&&clearTimeout(this.Ud),this.$d=0,this.Ud=null,this.qd={et:Number.NEGATIVE_INFINITY,rt:Number.POSITIVE_INFINITY}}Bf(t){if(this.uf||null!==this.Zd)return;if(this.bf(t))return;const i=this.wf(t);this.Cf(i,this.yf.zf),this.ff=!0}Ef(t){const i=Ms(t.changedTouches,f(this.df));if(null===i)return;if(this.cf=gs(t),null!==this.hf)return;if(this._f)return;this.af=!0;const n=this.Mf(ws(i),f(this.Zd)),{If:s,Lf:e,gf:r}=n;if(this.Gd||!(r<5)){if(!this.Gd){const t=.5*s,i=e>=t&&!this.cn.Nf(),n=t>e&&!this.cn.Ff();i||n||(this._f=!0),this.Gd=!0,this.Qd=!0,this.Of(),this.kf()}if(!this._f){const n=this.wf(t,i);this.xf(n,this.yf.Wf),bs(t)}}}jf(t){if(0!==t.button)return;const i=this.Mf(ws(t),f(this.Kd)),{gf:n}=i;if(n>=5&&(this.Jd=!0,this.Pf()),this.Jd){const i=this.wf(t);this.Cf(i,this.yf.Hf)}}Mf(t,i){const n=Math.abs(i.et-t.et),s=Math.abs(i.rt-t.rt);return{If:n,Lf:s,gf:n+s}}$f(t){let i=Ms(t.changedTouches,f(this.df));if(null===i&&0===t.touches.length&&(i=t.changedTouches[0]),null===i)return;this.df=null,this.cf=gs(t),this.Of(),this.Zd=null,this.rf&&(this.rf(),this.rf=null);const n=this.wf(t,i);if(this.xf(n,this.yf.Uf),++this.$d,this.Ud&&this.$d>1){const{gf:t}=this.Mf(ws(i),this.qd);t<30&&!this.Qd&&this.xf(n,this.yf.Sf),this.kf()}else this.Qd||(this.xf(n,this.yf.qf),this.yf.qf&&bs(t));0===this.$d&&bs(t),0===t.touches.length&&this.Xd&&(this.Xd=!1,bs(t))}pf(t){if(0!==t.button)return;const i=this.wf(t);if(this.Kd=null,this.uf=!1,this.ef&&(this.ef(),this.ef=null),rs()){this.Rf.ownerDocument.documentElement.removeEventListener("mouseleave",this.vf)}if(!this.bf(t))if(this.Cf(i,this.yf.Yf),++this.Wd,this.jd&&this.Wd>1){const{gf:n}=this.Mf(ws(t),this.Hd);n<5&&!this.Jd&&this.Cf(i,this.yf.Tf),this.Pf()}else this.Jd||this.Cf(i,this.yf.Xf)}Of(){null!==this.Yd&&(clearTimeout(this.Yd),this.Yd=null)}Kf(t){if(null!==this.df)return;const i=t.changedTouches[0];this.df=i.identifier,this.cf=gs(t);const n=this.Rf.ownerDocument.documentElement;this.Qd=!1,this.Gd=!1,this._f=!1,this.Zd=ws(i),this.rf&&(this.rf(),this.rf=null);{const i=this.Ef.bind(this),s=this.$f.bind(this);this.rf=()=>{n.removeEventListener("touchmove",i),n.removeEventListener("touchend",s)},n.addEventListener("touchmove",i,{passive:!1}),n.addEventListener("touchend",s,{passive:!1}),this.Of(),this.Yd=setTimeout(this.Zf.bind(this,t),240)}const s=this.wf(t,i);this.xf(s,this.yf.Gf),this.Ud||(this.$d=0,this.Ud=setTimeout(this.kf.bind(this),500),this.qd=ws(i))}Jf(t){if(0!==t.button)return;const i=this.Rf.ownerDocument.documentElement;rs()&&i.addEventListener("mouseleave",this.vf),this.Jd=!1,this.Kd=ws(t),this.ef&&(this.ef(),this.ef=null);{const t=this.jf.bind(this),n=this.pf.bind(this);this.ef=()=>{i.removeEventListener("mousemove",t),i.removeEventListener("mouseup",n)},i.addEventListener("mousemove",t),i.addEventListener("mouseup",n)}if(this.uf=!0,this.bf(t))return;const n=this.wf(t);this.Cf(n,this.yf.Qf),this.jd||(this.Wd=0,this.jd=setTimeout(this.Pf.bind(this),500),this.Hd=ws(t))}Df(){this.Rf.addEventListener("mouseenter",this.Af.bind(this)),this.Rf.addEventListener("touchcancel",this.Of.bind(this));{const t=this.Rf.ownerDocument,i=t=>{this.yf.tv&&(t.composed&&this.Rf.contains(t.composedPath()[0])||t.target&&this.Rf.contains(t.target)||this.yf.tv())};this.if=()=>{t.removeEventListener("touchstart",i)},this.tf=()=>{t.removeEventListener("mousedown",i)},t.addEventListener("mousedown",i),t.addEventListener("touchstart",i,{passive:!0})}hs()&&(this.nf=()=>{this.Rf.removeEventListener("dblclick",this.mf)},this.Rf.addEventListener("dblclick",this.mf)),this.Rf.addEventListener("mouseleave",this.iv.bind(this)),this.Rf.addEventListener("touchstart",this.Kf.bind(this),{passive:!0}),vs(this.Rf),this.Rf.addEventListener("mousedown",this.Jf.bind(this)),this.nv(),this.Rf.addEventListener("touchmove",(()=>{}),{passive:!1})}nv(){void 0===this.yf.sv&&void 0===this.yf.ev&&void 0===this.yf.rv||(this.Rf.addEventListener("touchstart",(t=>this.hv(t.touches)),{passive:!0}),this.Rf.addEventListener("touchmove",(t=>{if(2===t.touches.length&&null!==this.hf&&void 0!==this.yf.ev){const i=ms(t.touches[0],t.touches[1])/this.lf;this.yf.ev(this.hf,i),bs(t)}}),{passive:!1}),this.Rf.addEventListener("touchend",(t=>{this.hv(t.touches)})))}hv(t){1===t.length&&(this.af=!1),2!==t.length||this.af||this.Xd?this.lv():this.av(t)}av(t){const i=this.Rf.getBoundingClientRect()||{left:0,top:0};this.hf={et:(t[0].clientX-i.left+(t[1].clientX-i.left))/2,rt:(t[0].clientY-i.top+(t[1].clientY-i.top))/2},this.lf=ms(t[0],t[1]),void 0!==this.yf.sv&&this.yf.sv(),this.Of()}lv(){null!==this.hf&&(this.hf=null,void 0!==this.yf.rv&&this.yf.rv())}iv(t){if(this.sf&&this.sf(),this.bf(t))return;if(!this.ff)return;const i=this.wf(t);this.Cf(i,this.yf.ov),this.ff=!hs()}Zf(t){const i=Ms(t.touches,f(this.df));if(null===i)return;const n=this.wf(t,i);this.xf(n,this.yf._v),this.Qd=!0,this.Xd=!0}bf(t){return t.sourceCapabilities&&void 0!==t.sourceCapabilities.firesTouchEvents?t.sourceCapabilities.firesTouchEvents:gs(t){"touchstart"!==t.type&&bs(t)}}}}function ms(t,i){const n=t.clientX-i.clientX,s=t.clientY-i.clientY;return Math.sqrt(n*n+s*s)}function bs(t){t.cancelable&&t.preventDefault()}function ws(t){return{et:t.pageX,rt:t.pageY}}function gs(t){return t.timeStamp||performance.now()}function Ms(t,i){for(let n=0;n{var s,e,r,h;return(null!==(e=null===(s=n.At())||void 0===s?void 0:s.xa())&&void 0!==e?e:"")!==i?[]:null!==(h=null===(r=n.la)||void 0===r?void 0:r.call(n,t))&&void 0!==h?h:[]}}class ks{constructor(t,i,n,s){this.Li=null,this.wv=null,this.gv=!1,this.Mv=new Qt(200),this.Gr=null,this.xv=0,this.Sv=!1,this.yv=()=>{this.Sv||this.nn.kv().qt().Fh()},this.Cv=()=>{this.Sv||this.nn.kv().qt().Fh()},this.nn=t,this.cn=i,this.mo=i.layout,this.yc=n,this.Tv="left"===s,this.Pv=ys("normal",s),this.Rv=ys("top",s),this.Dv=ys("bottom",s),this.Ov=document.createElement("div"),this.Ov.style.height="100%",this.Ov.style.overflow="hidden",this.Ov.style.width="25px",this.Ov.style.left="0",this.Ov.style.position="relative",this.Av=us(this.Ov,Gn({width:16,height:16})),this.Av.subscribeSuggestedBitmapSizeChanged(this.yv);const e=this.Av.canvasElement;e.style.position="absolute",e.style.zIndex="1",e.style.left="0",e.style.top="0",this.Bv=us(this.Ov,Gn({width:16,height:16})),this.Bv.subscribeSuggestedBitmapSizeChanged(this.Cv);const r=this.Bv.canvasElement;r.style.position="absolute",r.style.zIndex="2",r.style.left="0",r.style.top="0";const h={Qf:this.Vv.bind(this),Gf:this.Vv.bind(this),Hf:this.zv.bind(this),Wf:this.zv.bind(this),tv:this.Ev.bind(this),Yf:this.Iv.bind(this),Uf:this.Iv.bind(this),Tf:this.Lv.bind(this),Sf:this.Lv.bind(this),Vf:this.Nv.bind(this),ov:this.Fv.bind(this)};this.Wv=new ps(this.Bv.canvasElement,h,{Nf:()=>!this.cn.handleScroll.vertTouchDrag,Ff:()=>!0})}S(){this.Wv.S(),this.Bv.unsubscribeSuggestedBitmapSizeChanged(this.Cv),this.Bv.dispose(),this.Av.unsubscribeSuggestedBitmapSizeChanged(this.yv),this.Av.dispose(),null!==this.Li&&this.Li.$o().p(this),this.Li=null}jv(){return this.Ov}P(){return this.mo.fontSize}Hv(){const t=this.yc.W();return this.Gr!==t.R&&(this.Mv.ir(),this.Gr=t.R),t}$v(){if(null===this.Li)return 0;let t=0;const i=this.Hv(),n=f(this.Av.canvasElement.getContext("2d"));n.save();const s=this.Li.Ia();n.font=this.Uv(),s.length>0&&(t=Math.max(this.Mv.Si(n,s[0].Za),this.Mv.Si(n,s[s.length-1].Za)));const e=this.qv();for(let i=e.length;i--;){const s=this.Mv.Si(n,e[i].Jt());s>t&&(t=s)}const r=this.Li.Pt();if(null!==r&&null!==this.wv){const i=this.Li.pn(1,r),s=this.Li.pn(this.wv.height-2,r);t=Math.max(t,this.Mv.Si(n,this.Li.Wi(Math.floor(Math.min(i,s))+.11111111111111,r)),this.Mv.Si(n,this.Li.Wi(Math.ceil(Math.max(i,s))-.11111111111111,r)))}n.restore();const h=t||34;return ls(Math.ceil(i.C+i.T+i.V+i.I+5+h))}Yv(t){null!==this.wv&&Jn(this.wv,t)||(this.wv=t,this.Sv=!0,this.Av.resizeCanvasElement(t),this.Bv.resizeCanvasElement(t),this.Sv=!1,this.Ov.style.width=`${t.width}px`,this.Ov.style.height=`${t.height}px`)}Xv(){return f(this.wv).width}Ji(t){this.Li!==t&&(null!==this.Li&&this.Li.$o().p(this),this.Li=t,t.$o().l(this.ao.bind(this),this))}At(){return this.Li}ir(){const t=this.nn.Kv();this.nn.kv().qt().A_(t,f(this.At()))}Zv(t){if(null===this.wv)return;if(1!==t){this.Gv(),this.Av.applySuggestedBitmapSize();const t=ss(this.Av);null!==t&&(t.useBitmapCoordinateSpace((t=>{this.Jv(t),this.Ve(t)})),this.nn.Qv(t,this.Dv),this.tp(t),this.nn.Qv(t,this.Pv),this.ip(t))}this.Bv.applySuggestedBitmapSize();const i=ss(this.Bv);null!==i&&(i.useBitmapCoordinateSpace((({context:t,bitmapSize:i})=>{t.clearRect(0,0,i.width,i.height)})),this.np(i),this.nn.Qv(i,this.Rv))}sp(){return this.Av.bitmapSize}ep(t,i,n){const s=this.sp();s.width>0&&s.height>0&&t.drawImage(this.Av.canvasElement,i,n)}gt(){var t;null===(t=this.Li)||void 0===t||t.Ia()}Vv(t){if(null===this.Li||this.Li.Fi()||!this.cn.handleScale.axisPressedMouseMove.price)return;const i=this.nn.kv().qt(),n=this.nn.Kv();this.gv=!0,i.k_(n,this.Li,t.localY)}zv(t){if(null===this.Li||!this.cn.handleScale.axisPressedMouseMove.price)return;const i=this.nn.kv().qt(),n=this.nn.Kv(),s=this.Li;i.C_(n,s,t.localY)}Ev(){if(null===this.Li||!this.cn.handleScale.axisPressedMouseMove.price)return;const t=this.nn.kv().qt(),i=this.nn.Kv(),n=this.Li;this.gv&&(this.gv=!1,t.T_(i,n))}Iv(t){if(null===this.Li||!this.cn.handleScale.axisPressedMouseMove.price)return;const i=this.nn.kv().qt(),n=this.nn.Kv();this.gv=!1,i.T_(n,this.Li)}Lv(t){this.cn.handleScale.axisDoubleClickReset.price&&this.ir()}Nv(t){if(null===this.Li)return;!this.nn.kv().qt().W().handleScale.axisPressedMouseMove.price||this.Li.ph()||this.Li.Co()||this.rp(1)}Fv(t){this.rp(0)}qv(){const t=[],i=null===this.Li?void 0:this.Li;return(n=>{for(let s=0;s{t.fillStyle=n.borderColor;const l=Math.max(1,Math.floor(h)),a=Math.floor(.5*h),o=Math.round(s.T*r);t.beginPath();for(const n of i)t.rect(Math.floor(e*r),Math.round(n.Aa*h)-a,o,l);t.fill()})),t.useMediaCoordinateSpace((({context:t})=>{var r;t.font=this.Uv(),t.fillStyle=null!==(r=n.textColor)&&void 0!==r?r:this.mo.textColor,t.textAlign=this.Tv?"right":"left",t.textBaseline="middle";const h=this.Tv?Math.round(e-s.V):Math.round(e+s.T+s.V),l=i.map((i=>this.Mv.xi(t,i.Za)));for(let n=i.length;n--;){const s=i[n];t.fillText(s.Za,h,s.Aa+l[n])}}))}Gv(){if(null===this.wv||null===this.Li)return;let t=this.wv.height/2;const i=[],n=this.Li.No().slice(),s=this.nn.Kv(),e=this.Hv();this.Li===s.vr()&&this.nn.Kv().No().forEach((t=>{s.dr(t)&&n.push(t)}));const r=this.Li.Ta()[0],h=this.Li;n.forEach((n=>{const e=n.Rn(s,h);e.forEach((t=>{t.Bi(null),t.Vi()&&i.push(t)})),r===n&&e.length>0&&(t=e[0].ki())})),i.forEach((t=>t.Bi(t.ki())));this.Li.W().alignLabels&&this.hp(i,e,t)}hp(t,i,n){if(null===this.wv)return;const s=t.filter((t=>t.ki()<=n)),e=t.filter((t=>t.ki()>n));s.sort(((t,i)=>i.ki()-t.ki())),s.length&&e.length&&e.push(s[0]),e.sort(((t,i)=>t.ki()-i.ki()));for(const n of t){const t=Math.floor(n.Et(i)/2),s=n.ki();s>-t&&sthis.wv.height-t&&sl-r&&n.Bi(l-r)}for(let t=1;t{if(i.zi()){i.xt(f(this.Li)).K(t,n,this.Mv,s)}}))}np(t){if(null===this.wv||null===this.Li)return;const i=this.nn.kv().qt(),n=[],s=this.nn.Kv(),e=i.Fc().Rn(s,this.Li);e.length&&n.push(e);const r=this.Hv(),h=this.Tv?"right":"left";n.forEach((i=>{i.forEach((i=>{i.xt(f(this.Li)).K(t,r,this.Mv,h)}))}))}rp(t){this.Ov.style.cursor=1===t?"ns-resize":"default"}ao(){const t=this.$v();this.xv{this.Sv||null===this.wp||this.Ui().Fh()},this.Cv=()=>{this.Sv||null===this.wp||this.Ui().Fh()},this.gp=t,this.wp=i,this.wp.z_().l(this.Mp.bind(this),this,!0),this.xp=document.createElement("td"),this.xp.style.padding="0",this.xp.style.position="relative";const n=document.createElement("div");n.style.width="100%",n.style.height="100%",n.style.position="relative",n.style.overflow="hidden",this.Sp=document.createElement("td"),this.Sp.style.padding="0",this.yp=document.createElement("td"),this.yp.style.padding="0",this.xp.appendChild(n),this.Av=us(n,Gn({width:16,height:16})),this.Av.subscribeSuggestedBitmapSizeChanged(this.yv);const s=this.Av.canvasElement;s.style.position="absolute",s.style.zIndex="1",s.style.left="0",s.style.top="0",this.Bv=us(n,Gn({width:16,height:16})),this.Bv.subscribeSuggestedBitmapSizeChanged(this.Cv);const e=this.Bv.canvasElement;e.style.position="absolute",e.style.zIndex="2",e.style.left="0",e.style.top="0",this.kp=document.createElement("tr"),this.kp.appendChild(this.Sp),this.kp.appendChild(this.xp),this.kp.appendChild(this.yp),this.Cp(),this.Wv=new ps(this.Bv.canvasElement,this,{Nf:()=>null===this.vp&&!this.gp.W().handleScroll.vertTouchDrag,Ff:()=>null===this.vp&&!this.gp.W().handleScroll.horzTouchDrag})}S(){null!==this.lp&&this.lp.S(),null!==this.ap&&this.ap.S(),this.Bv.unsubscribeSuggestedBitmapSizeChanged(this.Cv),this.Bv.dispose(),this.Av.unsubscribeSuggestedBitmapSizeChanged(this.yv),this.Av.dispose(),null!==this.wp&&this.wp.z_().p(this),this.Wv.S()}Kv(){return f(this.wp)}Tp(t){null!==this.wp&&this.wp.z_().p(this),this.wp=t,null!==this.wp&&this.wp.z_().l(Ds.prototype.Mp.bind(this),this,!0),this.Cp()}kv(){return this.gp}jv(){return this.kp}Cp(){if(null!==this.wp&&(this.Pp(),0!==this.Ui().Mt().length)){if(null!==this.lp){const t=this.wp.S_();this.lp.Ji(f(t))}if(null!==this.ap){const t=this.wp.y_();this.ap.Ji(f(t))}}}Rp(){null!==this.lp&&this.lp.gt(),null!==this.ap&&this.ap.gt()}v_(){return null!==this.wp?this.wp.v_():0}p_(t){this.wp&&this.wp.p_(t)}Vf(t){if(!this.wp)return;this.Dp();const i=t.localX,n=t.localY;this.Op(i,n,t)}Qf(t){this.Dp(),this.Ap(),this.Op(t.localX,t.localY,t)}zf(t){var i;if(!this.wp)return;this.Dp();const n=t.localX,s=t.localY;this.Op(n,s,t);const e=this.br(n,s);this.gp.Bp(null!==(i=null==e?void 0:e.bv)&&void 0!==i?i:null),this.Ui().Vc(e&&{zc:e.zc,pv:e.pv})}Xf(t){null!==this.wp&&(this.Dp(),this.Vp(t))}Tf(t){null!==this.wp&&this.zp(this.cp,t)}Sf(t){this.Tf(t)}Hf(t){this.Dp(),this.Ep(t),this.Op(t.localX,t.localY,t)}Yf(t){null!==this.wp&&(this.Dp(),this.fp=!1,this.Ip(t))}qf(t){null!==this.wp&&this.Vp(t)}_v(t){if(this.fp=!0,null===this.vp){const i={x:t.localX,y:t.localY};this.Lp(i,i,t)}}ov(t){null!==this.wp&&(this.Dp(),this.wp.qt().Vc(null),this.Np())}Fp(){return this.up}Wp(){return this.cp}sv(){this.dp=1,this.Ui().Un()}ev(t,i){if(!this.gp.W().handleScale.pinch)return;const n=5*(i-this.dp);this.dp=i,this.Ui().Uc(t.et,n)}Gf(t){this.fp=!1,this.pp=null!==this.vp,this.Ap();const i=this.Ui().Fc();null!==this.vp&&i.Tt()&&(this.mp={x:i.Kt(),y:i.Zt()},this.vp={x:t.localX,y:t.localY})}Wf(t){if(null===this.wp)return;const i=t.localX,n=t.localY;if(null===this.vp)this.Ep(t);else{this.pp=!1;const s=f(this.mp),e=s.x+(i-this.vp.x),r=s.y+(n-this.vp.y);this.Op(e,r,t)}}Uf(t){0===this.kv().W().trackingMode.exitMode&&(this.pp=!0),this.jp(),this.Ip(t)}br(t,i){const n=this.wp;return null===n?null:function(t,i,n){const s=t.No(),e=function(t,i,n){var s,e;let r,h;for(const o of t){const t=null!==(e=null===(s=o.oa)||void 0===s?void 0:s.call(o,i,n))&&void 0!==e?e:[];for(const i of t)l=i.zOrder,(!(a=null==r?void 0:r.zOrder)||"top"===l&&"top"!==a||"normal"===l&&"bottom"===a)&&(r=i,h=o)}var l,a;return r&&h?{mv:r,zc:h}:null}(s,i,n);if("top"===(null==e?void 0:e.mv.zOrder))return xs(e);for(const r of s){if(e&&e.zc===r&&"bottom"!==e.mv.zOrder&&!e.mv.isBackground)return xs(e);const s=Ss(r.Pn(t),i,n);if(null!==s)return{zc:r,fv:s.fv,pv:s.pv};if(e&&e.zc===r&&"bottom"!==e.mv.zOrder&&e.mv.isBackground)return xs(e)}return(null==e?void 0:e.mv)?xs(e):null}(n,t,i)}Hp(t,i){f("left"===i?this.lp:this.ap).Yv(Gn({width:t,height:this.wv.height}))}$p(){return this.wv}Yv(t){Jn(this.wv,t)||(this.wv=t,this.Sv=!0,this.Av.resizeCanvasElement(t),this.Bv.resizeCanvasElement(t),this.Sv=!1,this.xp.style.width=t.width+"px",this.xp.style.height=t.height+"px")}Up(){const t=f(this.wp);t.x_(t.S_()),t.x_(t.y_());for(const i of t.Ta())if(t.dr(i)){const n=i.At();null!==n&&t.x_(n),i.On()}}sp(){return this.Av.bitmapSize}ep(t,i,n){const s=this.sp();s.width>0&&s.height>0&&t.drawImage(this.Av.canvasElement,i,n)}Zv(t){if(0===t)return;if(null===this.wp)return;if(t>1&&this.Up(),null!==this.lp&&this.lp.Zv(t),null!==this.ap&&this.ap.Zv(t),1!==t){this.Av.applySuggestedBitmapSize();const t=ss(this.Av);null!==t&&(t.useBitmapCoordinateSpace((t=>{this.Jv(t)})),this.wp&&(this.qp(t,Cs),this.Yp(t),this.Xp(t),this.qp(t,Ts),this.qp(t,Ps)))}this.Bv.applySuggestedBitmapSize();const i=ss(this.Bv);null!==i&&(i.useBitmapCoordinateSpace((({context:t,bitmapSize:i})=>{t.clearRect(0,0,i.width,i.height)})),this.Kp(i),this.qp(i,Rs))}Zp(){return this.lp}Gp(){return this.ap}Qv(t,i){this.qp(t,i)}Mp(){null!==this.wp&&this.wp.z_().p(this),this.wp=null}Vp(t){this.zp(this.up,t)}zp(t,i){const n=i.localX,s=i.localY;t.M()&&t.m(this.Ui().kt().Bu(n),{x:n,y:s},i)}Jv({context:t,bitmapSize:i}){const{width:n,height:s}=i,e=this.Ui(),r=e.q(),h=e.od();r===h?Y(t,0,0,n,s,h):G(t,0,0,n,s,r,h)}Yp(t){const i=f(this.wp).E_().Wh().xt();null!==i&&i.K(t,!1)}Xp(t){const i=this.Ui().Nc();this.Jp(t,Ts,cs,i),this.Jp(t,Ts,ds,i)}Kp(t){this.Jp(t,Ts,ds,this.Ui().Fc())}qp(t,i){const n=f(this.wp).No();for(const s of n)this.Jp(t,i,cs,s);for(const s of n)this.Jp(t,i,ds,s)}Jp(t,i,n,s){const e=f(this.wp),r=e.qt().Bc(),h=null!==r&&r.zc===s,l=null!==r&&h&&void 0!==r.pv?r.pv.gr:void 0;fs(i,(i=>n(i,t,h,l)),s,e)}Pp(){if(null===this.wp)return;const t=this.gp,i=this.wp.S_().W().visible,n=this.wp.y_().W().visible;i||null===this.lp||(this.Sp.removeChild(this.lp.jv()),this.lp.S(),this.lp=null),n||null===this.ap||(this.yp.removeChild(this.ap.jv()),this.ap.S(),this.ap=null);const s=t.qt().sd();i&&null===this.lp&&(this.lp=new ks(this,t.W(),s,"left"),this.Sp.appendChild(this.lp.jv())),n&&null===this.ap&&(this.ap=new ks(this,t.W(),s,"right"),this.yp.appendChild(this.ap.jv()))}Qp(t){return t.uv&&this.fp||null!==this.vp}tm(t){return Math.max(0,Math.min(t,this.wv.width-1))}im(t){return Math.max(0,Math.min(t,this.wv.height-1))}Op(t,i,n){this.Ui().Jc(this.tm(t),this.im(i),n,f(this.wp))}Np(){this.Ui().td()}jp(){this.pp&&(this.vp=null,this.Np())}Lp(t,i,n){this.vp=t,this.pp=!1,this.Op(i.x,i.y,n);const s=this.Ui().Fc();this.mp={x:s.Kt(),y:s.Zt()}}Ui(){return this.gp.qt()}Ip(t){if(!this._p)return;const i=this.Ui(),n=this.Kv();if(i.D_(n,n.vn()),this.op=null,this._p=!1,i.Kc(),null!==this.bp){const t=performance.now(),n=i.kt();this.bp.Dr(n.Iu(),t),this.bp.Yu(t)||i.Xn(this.bp)}}Dp(){this.vp=null}Ap(){if(!this.wp)return;if(this.Ui().Un(),document.activeElement!==document.body&&document.activeElement!==document.documentElement)f(document.activeElement).blur();else{const t=document.getSelection();null!==t&&t.removeAllRanges()}!this.wp.vn().Fi()&&this.Ui().kt().Fi()}Ep(t){if(null===this.wp)return;const i=this.Ui(),n=i.kt();if(n.Fi())return;const s=this.gp.W(),e=s.handleScroll,r=s.kineticScroll;if((!e.pressedMouseMove||t.uv)&&(!e.horzTouchDrag&&!e.vertTouchDrag||!t.uv))return;const h=this.wp.vn(),l=performance.now();if(null!==this.op||this.Qp(t)||(this.op={x:t.clientX,y:t.clientY,yd:l,nm:t.localX,sm:t.localY}),null!==this.op&&!this._p&&(this.op.x!==t.clientX||this.op.y!==t.clientY)){if(t.uv&&r.touch||!t.uv&&r.mouse){const t=n.he();this.bp=new _s(.2/t,7/t,.997,15/t),this.bp.Nd(n.Iu(),this.op.yd)}else this.bp=null;h.Fi()||i.P_(this.wp,h,t.localY),i.Yc(t.localX),this._p=!0}this._p&&(h.Fi()||i.R_(this.wp,h,t.localY),i.Xc(t.localX),null!==this.bp&&this.bp.Nd(n.Iu(),l))}}class Os{constructor(t,i,n,s,e){this.bt=!0,this.wv=Gn({width:0,height:0}),this.yv=()=>this.Zv(3),this.Tv="left"===t,this.yc=n.sd,this.cn=i,this.rm=s,this.hm=e,this.Ov=document.createElement("div"),this.Ov.style.width="25px",this.Ov.style.height="100%",this.Ov.style.overflow="hidden",this.Av=us(this.Ov,Gn({width:16,height:16})),this.Av.subscribeSuggestedBitmapSizeChanged(this.yv)}S(){this.Av.unsubscribeSuggestedBitmapSizeChanged(this.yv),this.Av.dispose()}jv(){return this.Ov}$p(){return this.wv}Yv(t){Jn(this.wv,t)||(this.wv=t,this.Av.resizeCanvasElement(t),this.Ov.style.width=`${t.width}px`,this.Ov.style.height=`${t.height}px`,this.bt=!0)}Zv(t){if(t<3&&!this.bt)return;if(0===this.wv.width||0===this.wv.height)return;this.bt=!1,this.Av.applySuggestedBitmapSize();const i=ss(this.Av);null!==i&&i.useBitmapCoordinateSpace((t=>{this.Jv(t),this.Ve(t)}))}sp(){return this.Av.bitmapSize}ep(t,i,n){const s=this.sp();s.width>0&&s.height>0&&t.drawImage(this.Av.canvasElement,i,n)}Ve({context:t,bitmapSize:i,horizontalPixelRatio:n,verticalPixelRatio:s}){if(!this.rm())return;t.fillStyle=this.cn.timeScale.borderColor;const e=Math.floor(this.yc.W().C*n),r=Math.floor(this.yc.W().C*s),h=this.Tv?i.width-e:0;t.fillRect(h,0,e,r)}Jv({context:t,bitmapSize:i}){Y(t,0,0,i.width,i.height,this.hm())}}function As(t){return i=>{var n,s;return null!==(s=null===(n=i.aa)||void 0===n?void 0:n.call(i,t))&&void 0!==s?s:[]}}const Bs=As("normal"),Vs=As("top"),zs=As("bottom");class Es{constructor(t,i){this.lm=null,this.am=null,this.k=null,this.om=!1,this.wv=Gn({width:0,height:0}),this._m=new k,this.Mv=new Qt(5),this.Sv=!1,this.yv=()=>{this.Sv||this.gp.qt().Fh()},this.Cv=()=>{this.Sv||this.gp.qt().Fh()},this.gp=t,this.N_=i,this.cn=t.W().layout,this.um=document.createElement("tr"),this.dm=document.createElement("td"),this.dm.style.padding="0",this.fm=document.createElement("td"),this.fm.style.padding="0",this.Ov=document.createElement("td"),this.Ov.style.height="25px",this.Ov.style.padding="0",this.vm=document.createElement("div"),this.vm.style.width="100%",this.vm.style.height="100%",this.vm.style.position="relative",this.vm.style.overflow="hidden",this.Ov.appendChild(this.vm),this.Av=us(this.vm,Gn({width:16,height:16})),this.Av.subscribeSuggestedBitmapSizeChanged(this.yv);const n=this.Av.canvasElement;n.style.position="absolute",n.style.zIndex="1",n.style.left="0",n.style.top="0",this.Bv=us(this.vm,Gn({width:16,height:16})),this.Bv.subscribeSuggestedBitmapSizeChanged(this.Cv);const s=this.Bv.canvasElement;s.style.position="absolute",s.style.zIndex="2",s.style.left="0",s.style.top="0",this.um.appendChild(this.dm),this.um.appendChild(this.Ov),this.um.appendChild(this.fm),this.pm(),this.gp.qt().f_().l(this.pm.bind(this),this),this.Wv=new ps(this.Bv.canvasElement,this,{Nf:()=>!0,Ff:()=>!this.gp.W().handleScroll.horzTouchDrag})}S(){this.Wv.S(),null!==this.lm&&this.lm.S(),null!==this.am&&this.am.S(),this.Bv.unsubscribeSuggestedBitmapSizeChanged(this.Cv),this.Bv.dispose(),this.Av.unsubscribeSuggestedBitmapSizeChanged(this.yv),this.Av.dispose()}jv(){return this.um}bm(){return this.lm}wm(){return this.am}Qf(t){if(this.om)return;this.om=!0;const i=this.gp.qt();!i.kt().Fi()&&this.gp.W().handleScale.axisPressedMouseMove.time&&i.$c(t.localX)}Gf(t){this.Qf(t)}tv(){const t=this.gp.qt();!t.kt().Fi()&&this.om&&(this.om=!1,this.gp.W().handleScale.axisPressedMouseMove.time&&t.Gc())}Hf(t){const i=this.gp.qt();!i.kt().Fi()&&this.gp.W().handleScale.axisPressedMouseMove.time&&i.Zc(t.localX)}Wf(t){this.Hf(t)}Yf(){this.om=!1;const t=this.gp.qt();t.kt().Fi()&&!this.gp.W().handleScale.axisPressedMouseMove.time||t.Gc()}Uf(){this.Yf()}Tf(){this.gp.W().handleScale.axisDoubleClickReset.time&&this.gp.qt().Zn()}Sf(){this.Tf()}Vf(){this.gp.qt().W().handleScale.axisPressedMouseMove.time&&this.rp(1)}ov(){this.rp(0)}$p(){return this.wv}gm(){return this._m}Mm(t,i,n){Jn(this.wv,t)||(this.wv=t,this.Sv=!0,this.Av.resizeCanvasElement(t),this.Bv.resizeCanvasElement(t),this.Sv=!1,this.Ov.style.width=`${t.width}px`,this.Ov.style.height=`${t.height}px`,this._m.m(t)),null!==this.lm&&this.lm.Yv(Gn({width:i,height:t.height})),null!==this.am&&this.am.Yv(Gn({width:n,height:t.height}))}xm(){const t=this.Sm();return Math.ceil(t.C+t.T+t.P+t.L+t.B+t.ym)}gt(){this.gp.qt().kt().Ia()}sp(){return this.Av.bitmapSize}ep(t,i,n){const s=this.sp();s.width>0&&s.height>0&&t.drawImage(this.Av.canvasElement,i,n)}Zv(t){if(0===t)return;if(1!==t){this.Av.applySuggestedBitmapSize();const i=ss(this.Av);null!==i&&(i.useBitmapCoordinateSpace((t=>{this.Jv(t),this.Ve(t),this.km(i,zs)})),this.tp(i),this.km(i,Bs)),null!==this.lm&&this.lm.Zv(t),null!==this.am&&this.am.Zv(t)}this.Bv.applySuggestedBitmapSize();const i=ss(this.Bv);null!==i&&(i.useBitmapCoordinateSpace((({context:t,bitmapSize:i})=>{t.clearRect(0,0,i.width,i.height)})),this.Cm([...this.gp.qt().Mt(),this.gp.qt().Fc()],i),this.km(i,Vs))}km(t,i){const n=this.gp.qt().Mt();for(const s of n)fs(i,(i=>cs(i,t,!1,void 0)),s,void 0);for(const s of n)fs(i,(i=>ds(i,t,!1,void 0)),s,void 0)}Jv({context:t,bitmapSize:i}){Y(t,0,0,i.width,i.height,this.gp.qt().od())}Ve({context:t,bitmapSize:i,verticalPixelRatio:n}){if(this.gp.W().timeScale.borderVisible){t.fillStyle=this.Tm();const s=Math.max(1,Math.floor(this.Sm().C*n));t.fillRect(0,0,i.width,s)}}tp(t){const i=this.gp.qt().kt(),n=i.Ia();if(!n||0===n.length)return;const s=this.N_.maxTickMarkWeight(n),e=this.Sm(),r=i.W();r.borderVisible&&r.ticksVisible&&t.useBitmapCoordinateSpace((({context:t,horizontalPixelRatio:i,verticalPixelRatio:s})=>{t.strokeStyle=this.Tm(),t.fillStyle=this.Tm();const r=Math.max(1,Math.floor(i)),h=Math.floor(.5*i);t.beginPath();const l=Math.round(e.T*s);for(let s=n.length;s--;){const e=Math.round(n[s].coord*i);t.rect(e-h,0,r,l)}t.fill()})),t.useMediaCoordinateSpace((({context:t})=>{const i=e.C+e.T+e.L+e.P/2;t.textAlign="center",t.textBaseline="middle",t.fillStyle=this.$(),t.font=this.Uv();for(const e of n)if(e.weight=s){const n=e.needAlignCoordinate?this.Pm(t,e.coord,e.label):e.coord;t.fillText(e.label,n,i)}}))}Pm(t,i,n){const s=this.Mv.Si(t,n),e=s/2,r=Math.floor(i-e)+.5;return r<0?i+=Math.abs(0-r):r+s>this.wv.width&&(i-=Math.abs(this.wv.width-(r+s))),i}Cm(t,i){const n=this.Sm();for(const s of t)for(const t of s.tn())t.xt().K(i,n)}Tm(){return this.gp.W().timeScale.borderColor}$(){return this.cn.textColor}j(){return this.cn.fontSize}Uv(){return z(this.j(),this.cn.fontFamily)}Rm(){return z(this.j(),this.cn.fontFamily,"bold")}Sm(){null===this.k&&(this.k={C:1,N:NaN,L:NaN,B:NaN,Hi:NaN,T:5,P:NaN,R:"",ji:new Qt,ym:0});const t=this.k,i=this.Uv();if(t.R!==i){const n=this.j();t.P=n,t.R=i,t.L=3*n/12,t.B=3*n/12,t.Hi=9*n/12,t.N=0,t.ym=4*n/12,t.ji.ir()}return this.k}rp(t){this.Ov.style.cursor=1===t?"ew-resize":"default"}pm(){const t=this.gp.qt(),i=t.W();i.leftPriceScale.visible||null===this.lm||(this.dm.removeChild(this.lm.jv()),this.lm.S(),this.lm=null),i.rightPriceScale.visible||null===this.am||(this.fm.removeChild(this.am.jv()),this.am.S(),this.am=null);const n={sd:this.gp.qt().sd()},s=()=>i.leftPriceScale.borderVisible&&t.kt().W().borderVisible,e=()=>t.od();i.leftPriceScale.visible&&null===this.lm&&(this.lm=new Os("left",i,n,s,e),this.dm.appendChild(this.lm.jv())),i.rightPriceScale.visible&&null===this.am&&(this.am=new Os("right",i,n,s,e),this.fm.appendChild(this.am.jv()))}}const Is=!!es&&!!navigator.userAgentData&&navigator.userAgentData.brands.some((t=>t.brand.includes("Chromium")))&&!!es&&((null===(Ls=null===navigator||void 0===navigator?void 0:navigator.userAgentData)||void 0===Ls?void 0:Ls.platform)?"Windows"===navigator.userAgentData.platform:navigator.userAgent.toLowerCase().indexOf("win")>=0);var Ls;class Ns{constructor(t,i,n){var s;this.Dm=[],this.Om=0,this.Qa=0,this.e_=0,this.Am=0,this.Bm=0,this.Vm=null,this.zm=!1,this.up=new k,this.cp=new k,this.Mc=new k,this.Em=null,this.Im=null,this.Lm=t,this.cn=i,this.N_=n,this.um=document.createElement("div"),this.um.classList.add("tv-lightweight-charts"),this.um.style.overflow="hidden",this.um.style.direction="ltr",this.um.style.width="100%",this.um.style.height="100%",(s=this.um).style.userSelect="none",s.style.webkitUserSelect="none",s.style.msUserSelect="none",s.style.MozUserSelect="none",s.style.webkitTapHighlightColor="transparent",this.Nm=document.createElement("table"),this.Nm.setAttribute("cellspacing","0"),this.um.appendChild(this.Nm),this.Fm=this.Wm.bind(this),Fs(this.cn)&&this.jm(!0),this.Ui=new An(this.Sc.bind(this),this.cn,n),this.qt().Wc().l(this.Hm.bind(this),this),this.$m=new Es(this,this.N_),this.Nm.appendChild(this.$m.jv());const e=i.autoSize&&this.Um();let r=this.cn.width,h=this.cn.height;if(e||0===r||0===h){const i=t.getBoundingClientRect();r=r||i.width,h=h||i.height}this.qm(r,h),this.Ym(),t.appendChild(this.um),this.Xm(),this.Ui.kt().Gu().l(this.Ui.$l.bind(this.Ui),this),this.Ui.f_().l(this.Ui.$l.bind(this.Ui),this)}qt(){return this.Ui}W(){return this.cn}Km(){return this.Dm}Zm(){return this.$m}S(){this.jm(!1),0!==this.Om&&window.cancelAnimationFrame(this.Om),this.Ui.Wc().p(this),this.Ui.kt().Gu().p(this),this.Ui.f_().p(this),this.Ui.S();for(const t of this.Dm)this.Nm.removeChild(t.jv()),t.Fp().p(this),t.Wp().p(this),t.S();this.Dm=[],f(this.$m).S(),null!==this.um.parentElement&&this.um.parentElement.removeChild(this.um),this.Mc.S(),this.up.S(),this.cp.S(),this.Gm()}qm(t,i,n=!1){if(this.Qa===i&&this.e_===t)return;const s=function(t){const i=Math.floor(t.width),n=Math.floor(t.height);return Gn({width:i-i%2,height:n-n%2})}(Gn({width:t,height:i}));this.Qa=s.height,this.e_=s.width;const e=this.Qa+"px",r=this.e_+"px";f(this.um).style.height=e,f(this.um).style.width=r,this.Nm.style.height=e,this.Nm.style.width=r,n?this.Jm(at.es(),performance.now()):this.Ui.$l()}Zv(t){void 0===t&&(t=at.es());for(let i=0;i{let s=0;for(let e=0;e{f("left"===i?this.$m.bm():this.$m.wm()).ep(f(t),n,s)};if(this.cn.timeScale.visible){const i=this.$m.sp();if(null!==t){let e=0;this.sb()&&(r("left",e,n),e=f(s.Zp()).sp().width),this.$m.ep(t,e,n),e+=i.width,this.eb()&&r("right",e,n)}n+=i.height}return Gn({width:i,height:n})}ob(){let t=0,i=0,n=0;for(const s of this.Dm)this.sb()&&(i=Math.max(i,f(s.Zp()).$v(),this.cn.leftPriceScale.minimumWidth)),this.eb()&&(n=Math.max(n,f(s.Gp()).$v(),this.cn.rightPriceScale.minimumWidth)),t+=s.v_();i=ls(i),n=ls(n);const s=this.e_,e=this.Qa,r=Math.max(s-i-n,0),h=this.cn.timeScale.visible;let l=h?Math.max(this.$m.xm(),this.cn.timeScale.minimumHeight):0;var a;l=(a=l)+a%2;const o=0+l,_=e{t.Rp()})),3===(null===(n=this.Vm)||void 0===n?void 0:n.jn())&&(this.Vm.ts(t),this.ub(),this.cb(this.Vm),this.fb(this.Vm,i),t=this.Vm,this.Vm=null)),this.Zv(t)}fb(t,i){for(const n of t.Qn())this.ns(n,i)}cb(t){const i=this.Ui.Lc();for(let n=0;n{if(this.zm=!1,this.Om=0,null!==this.Vm){const i=this.Vm;this.Vm=null,this.Jm(i,t);for(const n of i.Qn())if(5===n.qn&&!n.Bt.Yu(t)){this.qt().Xn(n.Bt);break}}})))}ub(){this.Ym()}Ym(){const t=this.Ui.Lc(),i=t.length,n=this.Dm.length;for(let t=i;t{const n=i.zn().nl(t);null!==n&&e.set(i,n)}))}let r;if(null!==t){const i=null===(s=this.Ui.kt().qi(t))||void 0===s?void 0:s.originalTime;void 0!==i&&(r=i)}const h=this.qt().Bc(),l=null!==h&&h.zc instanceof Yi?h.zc:void 0,a=null!==h&&void 0!==h.pv?h.pv.wr:void 0;return{bb:r,se:null!=t?t:void 0,wb:null!=i?i:void 0,gb:l,Mb:e,xb:a,Sb:null!=n?n:void 0}}vb(t,i,n){this.up.m((()=>this.mb(t,i,n)))}pb(t,i,n){this.cp.m((()=>this.mb(t,i,n)))}Hm(t,i,n){this.Mc.m((()=>this.mb(t,i,n)))}Xm(){const t=this.cn.timeScale.visible?"":"none";this.$m.jv().style.display=t}sb(){return this.Dm[0].Kv().S_().W().visible}eb(){return this.Dm[0].Kv().y_().W().visible}Um(){return"ResizeObserver"in window&&(this.Em=new ResizeObserver((t=>{const i=t.find((t=>t.target===this.Lm));i&&this.qm(i.contentRect.width,i.contentRect.height)})),this.Em.observe(this.Lm,{box:"border-box"}),!0)}Gm(){null!==this.Em&&this.Em.disconnect(),this.Em=null}}function Fs(t){return Boolean(t.handleScroll.mouseWheel||t.handleScale.mouseWheel)}function Ws(t,i){var n={};for(var s in t)Object.prototype.hasOwnProperty.call(t,s)&&i.indexOf(s)<0&&(n[s]=t[s]);if(null!=t&&"function"==typeof Object.getOwnPropertySymbols){var e=0;for(s=Object.getOwnPropertySymbols(t);efunction(t,i){return i?i(t):void 0===(n=t).open&&void 0===n.value;var n}(s,h)?Ks({ut:i,se:n,bb:e},s):Ks(t(i,n,s,e,r),s)}function Gs(t){return{Candlestick:Zs(qs),Bar:Zs(Us),Area:Zs(Hs),Baseline:Zs($s),Histogram:Zs(js),Line:Zs(js),Custom:Zs(Ys)}[t]}function Js(t){return{se:0,kb:new Map,ia:t}}function Qs(t,i){if(void 0!==t&&0!==t.length)return{Cb:i.key(t[0].ut),Tb:i.key(t[t.length-1].ut)}}function te(t){let i;return t.forEach((t=>{void 0===i&&(i=t.bb)})),d(i)}class ie{constructor(t){this.Pb=new Map,this.Rb=new Map,this.Db=new Map,this.Ob=[],this.N_=t}S(){this.Pb.clear(),this.Rb.clear(),this.Db.clear(),this.Ob=[]}Ab(t,i){let n=0!==this.Pb.size,s=!1;const e=this.Rb.get(t);if(void 0!==e)if(1===this.Rb.size)n=!1,s=!0,this.Pb.clear();else for(const i of this.Ob)i.pointData.kb.delete(t)&&(s=!0);let r=[];if(0!==i.length){const n=i.map((t=>t.time)),e=this.N_.createConverterToInternalObj(i),h=Gs(t.Xh()),l=t.ga(),a=t.Ma();r=i.map(((i,r)=>{const o=e(i.time),_=this.N_.key(o);let u=this.Pb.get(_);void 0===u&&(u=Js(o),this.Pb.set(_,u),s=!0);const c=h(o,u.se,i,n[r],l,a);return u.kb.set(t,c),c}))}n&&this.Bb(),this.Vb(t,r);let h=-1;if(s){const t=[];this.Pb.forEach((i=>{t.push({timeWeight:0,time:i.ia,pointData:i,originalTime:te(i.kb)})})),t.sort(((t,i)=>this.N_.key(t.time)-this.N_.key(i.time))),h=this.zb(t)}return this.Eb(t,h,function(t,i,n){const s=Qs(t,n),e=Qs(i,n);if(void 0!==s&&void 0!==e)return{Xl:s.Tb>=e.Tb&&s.Cb>=e.Cb}}(this.Rb.get(t),e,this.N_))}hd(t){return this.Ab(t,[])}Ib(t,i){const n=i;!function(t){void 0===t.bb&&(t.bb=t.time)}(n),this.N_.preprocessData(i);const s=this.N_.createConverterToInternalObj([i])(i.time),e=this.Db.get(t);if(void 0!==e&&this.N_.key(s)this.N_.key(t.time)this.N_.key(s.ut)?Xs(i)&&n.push(i):Xs(i)?n[n.length-1]=i:n.splice(-1,1),this.Db.set(t,i.ut)}Vb(t,i){0!==i.length?(this.Rb.set(t,i.filter(Xs)),this.Db.set(t,i[i.length-1].ut)):(this.Rb.delete(t),this.Db.delete(t))}Bb(){for(const t of this.Ob)0===t.pointData.kb.size&&this.Pb.delete(this.N_.key(t.time))}zb(t){let i=-1;for(let n=0;n{0!==i.length&&(t=Math.max(t,i[i.length-1].se))})),t}Eb(t,i,n){const s={Fb:new Map,kt:{Au:this.Nb()}};if(-1!==i)this.Rb.forEach(((i,e)=>{s.Fb.set(e,{He:i,Wb:e===t?n:void 0})})),this.Rb.has(t)||s.Fb.set(t,{He:[],Wb:n}),s.kt.jb=this.Ob,s.kt.Hb=i;else{const i=this.Rb.get(t);s.Fb.set(t,{He:i||[],Wb:n})}return s}}function ne(t,i){t.se=i,t.kb.forEach((t=>{t.se=i}))}function se(t){const i={value:t.Bt[3],time:t.bb};return void 0!==t.yb&&(i.customValues=t.yb),i}function ee(t){const i=se(t);return void 0!==t.O&&(i.color=t.O),i}function re(t){const i=se(t);return void 0!==t._t&&(i.lineColor=t._t),void 0!==t.Ts&&(i.topColor=t.Ts),void 0!==t.Ps&&(i.bottomColor=t.Ps),i}function he(t){const i=se(t);return void 0!==t.Pe&&(i.topLineColor=t.Pe),void 0!==t.Re&&(i.bottomLineColor=t.Re),void 0!==t.Se&&(i.topFillColor1=t.Se),void 0!==t.ye&&(i.topFillColor2=t.ye),void 0!==t.ke&&(i.bottomFillColor1=t.ke),void 0!==t.Ce&&(i.bottomFillColor2=t.Ce),i}function le(t){const i={open:t.Bt[0],high:t.Bt[1],low:t.Bt[2],close:t.Bt[3],time:t.bb};return void 0!==t.yb&&(i.customValues=t.yb),i}function ae(t){const i=le(t);return void 0!==t.O&&(i.color=t.O),i}function oe(t){const i=le(t),{O:n,Vt:s,$h:e}=t;return void 0!==n&&(i.color=n),void 0!==s&&(i.borderColor=s),void 0!==e&&(i.wickColor=e),i}function _e(t){return{Area:re,Line:ee,Baseline:he,Histogram:ee,Bar:ae,Candlestick:oe,Custom:ue}[t]}function ue(t){const i=t.bb;return Object.assign(Object.assign({},t.He),{time:i})}const ce={vertLine:{color:"#9598A1",width:1,style:3,visible:!0,labelVisible:!0,labelBackgroundColor:"#131722"},horzLine:{color:"#9598A1",width:1,style:3,visible:!0,labelVisible:!0,labelBackgroundColor:"#131722"},mode:1},de={vertLines:{color:"#D6DCDE",style:0,visible:!0},horzLines:{color:"#D6DCDE",style:0,visible:!0}},fe={background:{type:"solid",color:"#FFFFFF"},textColor:"#191919",fontSize:12,fontFamily:V},ve={autoScale:!0,mode:0,invertScale:!1,alignLabels:!0,borderVisible:!0,borderColor:"#2B2B43",entireTextOnly:!1,visible:!1,ticksVisible:!1,scaleMargins:{bottom:.1,top:.2},minimumWidth:0},pe={rightOffset:0,barSpacing:6,minBarSpacing:.5,fixLeftEdge:!1,fixRightEdge:!1,lockVisibleTimeRangeOnResize:!1,rightBarStaysOnScroll:!1,borderVisible:!0,borderColor:"#2B2B43",visible:!0,timeVisible:!1,secondsVisible:!0,shiftVisibleRangeOnNewBar:!0,allowShiftVisibleRangeOnWhitespaceReplacement:!1,ticksVisible:!1,uniformDistribution:!1,minimumHeight:0},me={color:"rgba(0, 0, 0, 0)",visible:!1,fontSize:48,fontFamily:V,fontStyle:"",text:"",horzAlign:"center",vertAlign:"center"};function be(){return{width:0,height:0,autoSize:!1,layout:fe,crosshair:ce,grid:de,overlayPriceScales:Object.assign({},ve),leftPriceScale:Object.assign(Object.assign({},ve),{visible:!1}),rightPriceScale:Object.assign(Object.assign({},ve),{visible:!0}),timeScale:pe,watermark:me,localization:{locale:es?navigator.language:"",dateFormat:"dd MMM 'yy"},handleScroll:{mouseWheel:!0,pressedMouseMove:!0,horzTouchDrag:!0,vertTouchDrag:!0},handleScale:{axisPressedMouseMove:{time:!0,price:!0},axisDoubleClickReset:{time:!0,price:!0},mouseWheel:!0,pinch:!0},kineticScroll:{mouse:!1,touch:!0},trackingMode:{exitMode:1}}}class we{constructor(t,i){this.$b=t,this.Ub=i}applyOptions(t){this.$b.qt().Ec(this.Ub,t)}options(){return this.Li().W()}width(){return lt(this.Ub)?this.$b.nb(this.Ub):0}Li(){return f(this.$b.qt().Ic(this.Ub)).At}}function ge(t,i,n){const s=Ws(t,["time","originalTime"]),e=Object.assign({time:i},s);return void 0!==n&&(e.originalTime=n),e}const Me={color:"#FF0000",price:0,lineStyle:2,lineWidth:1,lineVisible:!0,axisLabelVisible:!0,title:"",axisLabelColor:"",axisLabelTextColor:""};class xe{constructor(t){this.Vh=t}applyOptions(t){this.Vh.Nh(t)}options(){return this.Vh.W()}qb(){return this.Vh}}class Se{constructor(t,i,n,s,e){this.Yb=new k,this.Is=t,this.Xb=i,this.Kb=n,this.N_=e,this.Zb=s}S(){this.Yb.S()}priceFormatter(){return this.Is.ca()}priceToCoordinate(t){const i=this.Is.Pt();return null===i?null:this.Is.At().Ot(t,i.Bt)}coordinateToPrice(t){const i=this.Is.Pt();return null===i?null:this.Is.At().pn(t,i.Bt)}barsInLogicalRange(t){if(null===t)return null;const i=new Mn(new bn(t.from,t.to)).iu(),n=this.Is.zn();if(n.Fi())return null;const s=n.nl(i.Os(),1),e=n.nl(i.di(),-1),r=f(n.Qh()),h=f(n.Vn());if(null!==s&&null!==e&&s.se>e.se)return{barsBefore:t.from-r,barsAfter:h-t.to};const l={barsBefore:null===s||s.se===r?t.from-r:s.se-r,barsAfter:null===e||e.se===h?h-t.to:h-e.se};return null!==s&&null!==e&&(l.from=s.bb,l.to=e.bb),l}setData(t){this.N_,this.Is.Xh(),this.Xb.Gb(this.Is,t),this.Jb("full")}update(t){this.Is.Xh(),this.Xb.Qb(this.Is,t),this.Jb("update")}dataByIndex(t,i){const n=this.Is.zn().nl(t,i);if(null===n)return null;return _e(this.seriesType())(n)}data(){const t=_e(this.seriesType());return this.Is.zn().ie().map((i=>t(i)))}subscribeDataChanged(t){this.Yb.l(t)}unsubscribeDataChanged(t){this.Yb.v(t)}setMarkers(t){this.N_;const i=t.map((t=>ge(t,this.N_.convertHorzItemToInternal(t.time),t.time)));this.Is.Zl(i)}markers(){return this.Is.Gl().map((t=>ge(t,t.originalTime,void 0)))}applyOptions(t){this.Is.Nh(t)}options(){return O(this.Is.W())}priceScale(){return this.Kb.priceScale(this.Is.At().xa())}createPriceLine(t){const i=C(O(Me),t),n=this.Is.Jl(i);return new xe(n)}removePriceLine(t){this.Is.Ql(t.qb())}seriesType(){return this.Is.Xh()}attachPrimitive(t){this.Is.ba(t),t.attached&&t.attached({chart:this.Zb,series:this,requestUpdate:()=>this.Is.qt().$l()})}detachPrimitive(t){this.Is.wa(t),t.detached&&t.detached()}Jb(t){this.Yb.M()&&this.Yb.m(t)}}class ye{constructor(t,i,n){this.tw=new k,this.uu=new k,this._m=new k,this.Ui=t,this.wl=t.kt(),this.$m=i,this.wl.Ku().l(this.iw.bind(this)),this.wl.Zu().l(this.nw.bind(this)),this.$m.gm().l(this.sw.bind(this)),this.N_=n}S(){this.wl.Ku().p(this),this.wl.Zu().p(this),this.$m.gm().p(this),this.tw.S(),this.uu.S(),this._m.S()}scrollPosition(){return this.wl.Iu()}scrollToPosition(t,i){i?this.wl.qu(t,1e3):this.Ui.Jn(t)}scrollToRealTime(){this.wl.Uu()}getVisibleRange(){const t=this.wl.ku();return null===t?null:{from:t.from.originalTime,to:t.to.originalTime}}setVisibleRange(t){const i={from:this.N_.convertHorzItemToInternal(t.from),to:this.N_.convertHorzItemToInternal(t.to)},n=this.wl.Ru(i);this.Ui.ld(n)}getVisibleLogicalRange(){const t=this.wl.yu();return null===t?null:{from:t.Os(),to:t.di()}}setVisibleLogicalRange(t){c(t.from<=t.to,"The from index cannot be after the to index."),this.Ui.ld(t)}resetTimeScale(){this.Ui.Zn()}fitContent(){this.Ui.Qu()}logicalToCoordinate(t){const i=this.Ui.kt();return i.Fi()?null:i.It(t)}coordinateToLogical(t){return this.wl.Fi()?null:this.wl.Bu(t)}timeToCoordinate(t){const i=this.N_.convertHorzItemToInternal(t),n=this.wl.ka(i,!1);return null===n?null:this.wl.It(n)}coordinateToTime(t){const i=this.Ui.kt(),n=i.Bu(t),s=i.qi(n);return null===s?null:s.originalTime}width(){return this.$m.$p().width}height(){return this.$m.$p().height}subscribeVisibleTimeRangeChange(t){this.tw.l(t)}unsubscribeVisibleTimeRangeChange(t){this.tw.v(t)}subscribeVisibleLogicalRangeChange(t){this.uu.l(t)}unsubscribeVisibleLogicalRangeChange(t){this.uu.v(t)}subscribeSizeChange(t){this._m.l(t)}unsubscribeSizeChange(t){this._m.v(t)}applyOptions(t){this.wl.Nh(t)}options(){return Object.assign(Object.assign({},O(this.wl.W())),{barSpacing:this.wl.he()})}iw(){this.tw.M()&&this.tw.m(this.getVisibleRange())}nw(){this.uu.M()&&this.uu.m(this.getVisibleLogicalRange())}sw(t){this._m.m(t.width,t.height)}}function ke(t){if(void 0===t||"custom"===t.type)return;const i=t;void 0!==i.minMove&&void 0===i.precision&&(i.precision=function(t){if(t>=1)return 0;let i=0;for(;i<8;i++){const n=Math.round(t);if(Math.abs(n-t)<1e-8)return i;t*=10}return i}(i.minMove))}function Ce(t){return function(t){if(D(t.handleScale)){const i=t.handleScale;t.handleScale={axisDoubleClickReset:{time:i,price:i},axisPressedMouseMove:{time:i,price:i},mouseWheel:i,pinch:i}}else if(void 0!==t.handleScale){const{axisPressedMouseMove:i,axisDoubleClickReset:n}=t.handleScale;D(i)&&(t.handleScale.axisPressedMouseMove={time:i,price:i}),D(n)&&(t.handleScale.axisDoubleClickReset={time:n,price:n})}const i=t.handleScroll;D(i)&&(t.handleScroll={horzTouchDrag:i,vertTouchDrag:i,mouseWheel:i,pressedMouseMove:i})}(t),t}class Te{constructor(t,i,n){this.ew=new Map,this.rw=new Map,this.hw=new k,this.lw=new k,this.aw=new k,this.ow=new ie(i);const s=void 0===n?O(be()):C(O(be()),Ce(n));this.N_=i,this.$b=new Ns(t,s,i),this.$b.Fp().l((t=>{this.hw.M()&&this.hw.m(this._w(t()))}),this),this.$b.Wp().l((t=>{this.lw.M()&&this.lw.m(this._w(t()))}),this),this.$b.Wc().l((t=>{this.aw.M()&&this.aw.m(this._w(t()))}),this);const e=this.$b.qt();this.uw=new ye(e,this.$b.Zm(),this.N_)}remove(){this.$b.Fp().p(this),this.$b.Wp().p(this),this.$b.Wc().p(this),this.uw.S(),this.$b.S(),this.ew.clear(),this.rw.clear(),this.hw.S(),this.lw.S(),this.aw.S(),this.ow.S()}resize(t,i,n){this.autoSizeActive()||this.$b.qm(t,i,n)}addCustomSeries(t,i){const n=v(t),s=Object.assign(Object.assign({},h),n.defaultOptions());return this.cw("Custom",s,i,n)}addAreaSeries(t){return this.cw("Area",s,t)}addBaselineSeries(t){return this.cw("Baseline",e,t)}addBarSeries(t){return this.cw("Bar",i,t)}addCandlestickSeries(i={}){return function(t){void 0!==t.borderColor&&(t.borderUpColor=t.borderColor,t.borderDownColor=t.borderColor),void 0!==t.wickColor&&(t.wickUpColor=t.wickColor,t.wickDownColor=t.wickColor)}(i),this.cw("Candlestick",t,i)}addHistogramSeries(t){return this.cw("Histogram",r,t)}addLineSeries(t){return this.cw("Line",n,t)}removeSeries(t){const i=d(this.ew.get(t)),n=this.ow.hd(i);this.$b.qt().hd(i),this.dw(n),this.ew.delete(t),this.rw.delete(i)}Gb(t,i){this.dw(this.ow.Ab(t,i))}Qb(t,i){this.dw(this.ow.Ib(t,i))}subscribeClick(t){this.hw.l(t)}unsubscribeClick(t){this.hw.v(t)}subscribeCrosshairMove(t){this.aw.l(t)}unsubscribeCrosshairMove(t){this.aw.v(t)}subscribeDblClick(t){this.lw.l(t)}unsubscribeDblClick(t){this.lw.v(t)}priceScale(t){return new we(this.$b,t)}timeScale(){return this.uw}applyOptions(t){this.$b.Nh(Ce(t))}options(){return this.$b.W()}takeScreenshot(){return this.$b.tb()}autoSizeActive(){return this.$b.rb()}chartElement(){return this.$b.hb()}paneSize(){const t=this.$b.ab();return{height:t.height,width:t.width}}setCrosshairPosition(t,i,n){const s=this.ew.get(n);if(void 0===s)return;const e=this.$b.qt().cr(s);null!==e&&this.$b.qt().Qc(t,i,e)}clearCrosshairPosition(){this.$b.qt().td(!0)}cw(t,i,n={},s){ke(n.priceFormat);const e=C(O(l),O(i),n),r=this.$b.qt().ed(t,e,s),h=new Se(r,this,this,this,this.N_);return this.ew.set(h,r),this.rw.set(r,h),h}dw(t){const i=this.$b.qt();i.nd(t.kt.Au,t.kt.jb,t.kt.Hb),t.Fb.forEach(((t,i)=>i.it(t.He,t.Wb))),i.zu()}fw(t){return d(this.rw.get(t))}_w(t){const i=new Map;t.Mb.forEach(((t,n)=>{const s=n.Xh(),e=_e(s)(t);if("Custom"!==s)c(function(t){return void 0!==t.open||void 0!==t.value}(e));else{const t=n.Ma();c(!t||!1===t(e))}i.set(this.fw(n),e)}));const n=void 0===t.gb?void 0:this.fw(t.gb);return{time:t.bb,logical:t.se,point:t.wb,hoveredSeries:n,hoveredObjectId:t.xb,seriesData:i,sourceEvent:t.Sb}}}function Pe(t,i,n){let s;if(R(t)){const i=document.getElementById(t);c(null!==i,`Cannot find element in DOM with id=${t}`),s=i}else s=t;const e=new Te(s,i,n);return i.setOptions(e.options()),e}const Re=Object.assign(Object.assign({},l),h);var De=Object.freeze({__proto__:null,get ColorType(){return Dn},get CrosshairMode(){return rt},get LastPriceAnimationMode(){return Pn},get LineStyle(){return o},get LineType(){return a},get MismatchDirection(){return Bi},get PriceLineSource(){return Rn},get PriceScaleMode(){return cn},get TickMarkType(){return On},get TrackingModeExitMode(){return Tn},createChart:function(t,i){return Pe(t,new Zn,Zn.Td(i))},createChartEx:Pe,customSeriesDefaultOptions:Re,isBusinessDay:Bn,isUTCTimestamp:Vn,version:function(){return"4.1.1"}});window.LightweightCharts=De}();
diff --git a/src/aleph/vm/orchestrator/views/templates/index.html b/src/aleph/vm/orchestrator/views/templates/index.html
index e1bea0876..49a3822f2 100644
--- a/src/aleph/vm/orchestrator/views/templates/index.html
+++ b/src/aleph/vm/orchestrator/views/templates/index.html
@@ -4,7 +4,7 @@
     
     Aleph.im Compute Node
     
-    
+    
 
 
     
    

From 5013d5a94d9c809b5627c190dc1d8f6be8a4b28f Mon Sep 17 00:00:00 2001
From: Bonjour Internet 
Date: Fri, 17 Nov 2023 11:29:11 +0100
Subject: [PATCH 551/990] add: license

---
 src/aleph/vm/orchestrator/views/static/main.css      | 5 +++++
 src/aleph/vm/orchestrator/views/templates/index.html | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/src/aleph/vm/orchestrator/views/static/main.css b/src/aleph/vm/orchestrator/views/static/main.css
index e4791d163..9e1812a38 100644
--- a/src/aleph/vm/orchestrator/views/static/main.css
+++ b/src/aleph/vm/orchestrator/views/static/main.css
@@ -92,4 +92,9 @@ progress {
 
 #chart-wrapper{
     display: none;
+}
+
+footer{
+    font-size: 70%;
+    opacity: .75;
 }
\ No newline at end of file
diff --git a/src/aleph/vm/orchestrator/views/templates/index.html b/src/aleph/vm/orchestrator/views/templates/index.html
index 49a3822f2..bbb117354 100644
--- a/src/aleph/vm/orchestrator/views/templates/index.html
+++ b/src/aleph/vm/orchestrator/views/templates/index.html
@@ -103,10 +103,15 @@ 
    Version
    
                 
    
 
                 
    
+
+                
             
         
    
     
 
+
     
     
     
 
-
\ No newline at end of file
+

From 493616d014bf2ebec96c3a717e3a2bd3f1fe2d32 Mon Sep 17 00:00:00 2001
From: Mike Hukiewitz <70762838+MHHukiewitz@users.noreply.github.com>
Date: Wed, 22 Nov 2023 16:04:53 +0100
Subject: [PATCH 572/990] Fix minor issues in #458 (#485)

* Change HTTP response to 403 where applicable

* Clarify is_token_still_valid()

* Fix typo
---
 .../vm/orchestrator/views/authentication.py      |  8 ++++----
 src/aleph/vm/orchestrator/views/operator.py      | 16 ++++++++--------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index d572f7f8d..532ce49ee 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -18,12 +18,12 @@
 
 def is_token_still_valid(timestamp):
     """
-    Checks if a token has exprired based on its timestamp
+    Checks if a token has expired based on its expiry timestamp
     """
     current_datetime = datetime.now(tz=timezone.utc)
-    target_datetime = datetime.fromisoformat(timestamp)
+    expiry_datetime = datetime.fromisoformat(timestamp)
 
-    return target_datetime > current_datetime
+    return expiry_datetime > current_datetime
 
 
 def verify_wallet_signature(signature, message, address):
@@ -205,7 +205,7 @@ async def authenticate_jwk(request: web.Request) -> str:
     return verify_signed_operation(signed_operation, signed_pubkey)
 
 
-async def authenicate_websocket_message(message) -> str:
+async def authenticate_websocket_message(message) -> str:
     """Authenticate a websocket message since JS cannot configure headers on WebSockets."""
     signed_pubkey = SignedPubKeyHeader.parse_obj(message["X-SignedPubKey"])
     signed_operation = SignedOperation.parse_obj(message["X-SignedOperation"])
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index f5bf8e881..75b9a024b 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -12,7 +12,7 @@
 from aleph.vm.models import VmExecution
 from aleph.vm.orchestrator.run import create_vm_execution
 from aleph.vm.orchestrator.views.authentication import (
-    authenicate_websocket_message,
+    authenticate_websocket_message,
     require_jwk_authentication,
 )
 from aleph.vm.pool import VmPool
@@ -71,12 +71,12 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
             # Authentication
             first_message = await ws.receive_json()
             credentials = first_message["auth"]
-            authenticated_sender = await authenicate_websocket_message(credentials)
+            authenticated_sender = await authenticate_websocket_message(credentials)
 
             if not is_sender_authorized(authenticated_sender, execution.message):
                 logger.debug(f"Denied request to access logs by {authenticated_sender} on {vm_hash}")
                 await ws.send_json({"status": "failed", "reason": "unauthorized sender"})
-                return web.Response(status=401, body="Unauthorized sender")
+                return web.Response(status=403, body="Unauthorized sender")
             else:
                 logger.debug(f"Accepted request to access logs by {authenticated_sender} on {vm_hash}")
 
@@ -118,7 +118,7 @@ async def operate_expire(request: web.Request, authenticated_sender: str) -> web
     execution = get_execution_or_404(vm_hash, pool=pool)
 
     if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=401, body="Unauthorized sender")
+        return web.Response(status=403, body="Unauthorized sender")
 
     logger.info(f"Expiring in {timeout} seconds: {execution.vm_hash}")
     await execution.expire(timeout=timeout)
@@ -138,10 +138,10 @@ async def operate_stop(request: web.Request, authenticated_sender: str) -> web.R
     execution = get_execution_or_404(vm_hash, pool=pool)
 
     if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=401, body="Unauthorized sender")
+        return web.Response(status=403, body="Unauthorized sender")
 
     if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=401, body="Unauthorized sender")
+        return web.Response(status=403, body="Unauthorized sender")
 
     if execution.is_running:
         logger.info(f"Stopping {execution.vm_hash}")
@@ -162,7 +162,7 @@ async def operate_reboot(request: web.Request, authenticated_sender: str) -> web
     execution = get_execution_or_404(vm_hash, pool=pool)
 
     if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=401, body="Unauthorized sender")
+        return web.Response(status=403, body="Unauthorized sender")
 
     if execution.is_running:
         logger.info(f"Rebooting {execution.vm_hash}")
@@ -184,7 +184,7 @@ async def operate_erase(request: web.Request, authenticated_sender: str) -> web.
     execution = get_execution_or_404(vm_hash, pool=pool)
 
     if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=401, body="Unauthorized sender")
+        return web.Response(status=403, body="Unauthorized sender")
 
     logger.info(f"Erasing {execution.vm_hash}")
 

From f0382b6e03891e1f43ce2c7999e7afd03340889f Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 21 Nov 2023 19:29:16 +0100
Subject: [PATCH 573/990] Problem: nftables chain initialization failed if no
 nat table present

If the nat table was not present, the code already tried to create it but the command execution order was not correct and it tried to add a chain to it before creating it

Solution: The chain creation command was attempted immediatly while the other command were put in a patch, put that command in a batch too
---
 src/aleph/vm/network/firewall.py | 42 ++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/src/aleph/vm/network/firewall.py b/src/aleph/vm/network/firewall.py
index eb76597ea..fb91d7288 100644
--- a/src/aleph/vm/network/firewall.py
+++ b/src/aleph/vm/network/firewall.py
@@ -130,10 +130,12 @@ def initialize_nftables() -> None:
             raise NotImplementedError(msg)
         base_chains[hook] = chains.pop()["chain"]
 
-    add_chain(
-        "ip",
-        base_chains["postrouting"]["table"],
-        f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-nat",
+    commands.append(
+        _make_add_chain_command(
+            "ip",
+            base_chains["postrouting"]["table"],
+            f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-nat",
+        )
     )
     commands.append(
         {
@@ -148,10 +150,12 @@ def initialize_nftables() -> None:
         }
     )
 
-    add_chain(
-        "ip",
-        base_chains["forward"]["table"],
-        f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-filter",
+    commands.append(
+        _make_add_chain_command(
+            "ip",
+            base_chains["forward"]["table"],
+            f"{settings.NFTABLES_CHAIN_PREFIX}-supervisor-filter",
+        )
     )
     commands.append(
         {
@@ -200,18 +204,20 @@ def teardown_nftables() -> None:
 def add_chain(family: str, table: str, name: str) -> int:
     """Helper function to quickly create a new chain in the nftables ruleset
     Returns the exit code from executing the nftables commands"""
-    commands = [
-        {
-            "add": {
-                "chain": {
-                    "family": family,
-                    "table": table,
-                    "name": name,
-                }
+    commands = [_make_add_chain_command(family, table, name)]
+    return execute_json_nft_commands(commands)
+
+
+def _make_add_chain_command(family: str, table: str, name: str) -> dict:
+    return {
+        "add": {
+            "chain": {
+                "family": family,
+                "table": table,
+                "name": name,
             }
         }
-    ]
-    return execute_json_nft_commands(commands)
+    }
 
 
 def remove_chain(name: str) -> int:

From d712985a6565f44c183caf217b3509af117e680b Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 24 Nov 2023 12:31:04 +0100
Subject: [PATCH 574/990] Problem: Stopped Instance could not be restarted
 (#486)

Problem: Stopped Instance could not be restarted

A VM instance that was started and then stopped (either via allocation, /control/stop or either) could never be restarted

Solution: The problem was that the stopped Execution keep being returned by create_vm, since it was still in the pool

We now remove the Execution from the pool once they are stopped, so we
no longer return it.

After discussion with Hugo it was decided to standardize the callback
mechanism using asyncio.Event:

1. a new Execution.stop_event was added.
2. VMPool create a task which wait on this event to remove the VM from the pool when it is stopped
---
 src/aleph/vm/models.py                      |  3 ++
 src/aleph/vm/orchestrator/views/operator.py |  2 +-
 src/aleph/vm/pool.py                        | 33 ++++++++++++++-------
 3 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index a4e3d4d8d..0fc5fd03e 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -72,6 +72,7 @@ class VmExecution:
     concurrent_runs: int
     runs_done_event: asyncio.Event
     stop_pending_lock: asyncio.Lock
+    stop_event: asyncio.Event
     expire_task: Optional[asyncio.Task] = None
     update_task: Optional[asyncio.Task] = None
 
@@ -112,6 +113,7 @@ def __init__(
         self.ready_event = asyncio.Event()
         self.concurrent_runs = 0
         self.runs_done_event = asyncio.Event()
+        self.stop_event = asyncio.Event()  #  triggered when the VM is stopped
         self.preparation_pending_lock = asyncio.Lock()
         self.stop_pending_lock = asyncio.Lock()
         self.snapshot_manager = snapshot_manager
@@ -237,6 +239,7 @@ async def stop(self):
 
             if isinstance(self.message, InstanceContent):
                 await self.snapshot_manager.stop_for(self.vm_hash)
+            self.stop_event.set()
 
     def start_watching_for_updates(self, pubsub: PubSub):
         if not self.update_task:
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 75b9a024b..1992ab029 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -167,7 +167,7 @@ async def operate_reboot(request: web.Request, authenticated_sender: str) -> web
     if execution.is_running:
         logger.info(f"Rebooting {execution.vm_hash}")
         await pool.stop_vm(vm_hash)
-        pool.forget_vm(vm_hash)
+
         await create_vm_execution(vm_hash=vm_hash, pool=pool)
         return web.Response(status=200, body=f"Rebooted VM with ref {vm_hash}")
     else:
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 0eb18e592..404d36bf2 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -83,20 +83,31 @@ async def create_a_vm(
             )
             self.executions[vm_hash] = execution
 
-        await execution.prepare()
-        vm_id = self.get_unique_vm_id()
+        try:
+            await execution.prepare()
+            vm_id = self.get_unique_vm_id()
 
-        if self.network:
-            vm_type = VmType.from_message_content(message)
-            tap_interface = await self.network.create_tap(vm_id, vm_hash, vm_type)
-        else:
-            tap_interface = None
+            if self.network:
+                vm_type = VmType.from_message_content(message)
+                tap_interface = await self.network.create_tap(vm_id, vm_hash, vm_type)
+            else:
+                tap_interface = None
+
+            await execution.create(vm_id=vm_id, tap_interface=tap_interface)
+
+            # Start VM snapshots automatically
+            if isinstance(message, InstanceContent):
+                await self.snapshot_manager.start_for(vm=execution.vm)
+        except Exception:
+            # ensure the VM is removed from the pool on creation error
+            self.forget_vm(vm_hash)
+            raise
 
-        await execution.create(vm_id=vm_id, tap_interface=tap_interface)
+        async def forget_on_stop(stop_event: asyncio.Event):
+            await stop_event.wait()
+            self.forget_vm(vm_hash)
 
-        # Start VM snapshots automatically
-        if isinstance(message, InstanceContent):
-            await self.snapshot_manager.start_for(vm=execution.vm)
+        asyncio.create_task(forget_on_stop(stop_event=execution.stop_event))
 
         return execution
 

From d8a4077acb438267dde5341cde5ed3531a9b6e91 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 24 Nov 2023 13:53:12 +0100
Subject: [PATCH 575/990] Make JAILER_BASE_DIR dependend on execution root
 (#481)

Make JAILER_BASE_DIR dependend on execution root

it was hardcoded before on /var/lib/aleph so it wasn't respecting the proper user settings
and was failling if the user didn't have the correct permission for this folder and was producing warning

Pass the settings to MicroVM instead of importing it so this module can stay separate
---
 src/aleph/vm/conf.py                               |  3 +++
 src/aleph/vm/controllers/__main__.py               |  1 +
 src/aleph/vm/controllers/firecracker/executable.py |  1 +
 src/aleph/vm/hypervisors/firecracker/microvm.py    | 12 +++++++-----
 4 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 2044368e2..33d09c3ba 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -195,6 +195,7 @@ class Settings(BaseSettings):
     PERSISTENT_VOLUMES_DIR: Path = Field(
         None, description="Persistent volumes location. Default to EXECUTION_ROOT/volumes/persistent/"
     )
+    JAILER_BASE_DIR: Path = Field(None)
 
     MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000  # 10 MB
     MAX_DATA_ARCHIVE_SIZE = 10_000_000  # 10 MB
@@ -347,6 +348,8 @@ def __init__(
             self.PERSISTENT_VOLUMES_DIR = self.EXECUTION_ROOT / "volumes" / "persistent"
         if not self.EXECUTION_LOG_DIRECTORY:
             self.EXECUTION_LOG_DIRECTORY = self.EXECUTION_ROOT / "executions"
+        if not self.JAILER_BASE_DIR:
+            self.JAILER_BASE_DIR = self.EXECUTION_ROOT / "jailer"
 
     class Config:
         env_prefix = "ALEPH_VM_"
diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index ed8d9db5c..d3071bcd3 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -72,6 +72,7 @@ async def run_instance(config: Configuration):
     execution = MicroVM(
         vm_id=config.vm_id,
         firecracker_bin_path=config.vm_configuration.firecracker_bin_path,
+        jailer_base_directory=config.settings.JAILER_BASE_DIR,
         use_jailer=config.vm_configuration.use_jailer,
         jailer_bin_path=config.vm_configuration.jailer_bin_path,
         init_timeout=config.vm_configuration.init_timeout,
diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index 46055a1cd..e72a84877 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -174,6 +174,7 @@ def __init__(
         self.fvm = MicroVM(
             vm_id=self.vm_id,
             firecracker_bin_path=settings.FIRECRACKER_PATH,
+            jailer_base_directory=settings.JAILER_BASE_DIR,
             use_jailer=settings.USE_JAILER,
             jailer_bin_path=settings.JAILER_PATH,
             init_timeout=settings.INIT_TIMEOUT,
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 5b30655da..e4be60e38 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -22,7 +22,6 @@
 logger = logging.getLogger(__name__)
 
 VSOCK_PATH = "/tmp/v.sock"
-JAILER_BASE_DIRECTORY = "/var/lib/aleph/vm/jailer"
 DEVICE_BASE_DIRECTORY = "/dev/mapper"
 
 
@@ -95,9 +94,9 @@ def __str__(self):
         return f"vm-{self.vm_id}"
 
     @property
-    def namespace_path(self):
+    def namespace_path(self) -> str:
         firecracker_bin_name = os.path.basename(self.firecracker_bin_path)
-        return f"{JAILER_BASE_DIRECTORY}/{firecracker_bin_name}/{self.vm_id}"
+        return str(self.jailer_base_directory / firecracker_bin_name / str(self.vm_id))
 
     @property
     def jailer_path(self):
@@ -121,12 +120,14 @@ def __init__(
         self,
         vm_id: int,
         firecracker_bin_path: Path,
+        jailer_base_directory: Path,
         use_jailer: bool = True,
         jailer_bin_path: Optional[Path] = None,
         init_timeout: float = 5.0,
     ):
         self.vm_id = vm_id
         self.use_jailer = use_jailer
+        self.jailer_base_directory = jailer_base_directory
         self.firecracker_bin_path = firecracker_bin_path
         self.jailer_bin_path = jailer_bin_path
         self.drives = []
@@ -237,7 +238,7 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces
             "--gid",
             gid,
             "--chroot-base-dir",
-            JAILER_BASE_DIRECTORY,
+            str(self.jailer_base_directory),
             "--",
             "--config-file",
             "/tmp/" + str(self.config_file_path.name),
@@ -492,7 +493,8 @@ async def teardown(self):
         logger.debug("Removing files")
         if self.config_file_path:
             self.config_file_path.unlink(missing_ok=True)
-        system(f"rm -fr {self.namespace_path}")
+        if Path(self.namespace_path).exists():
+            system(f"rm -fr {self.namespace_path}")
 
     def __del__(self):
         try:

From 73915f1057db7152a9be022a9b0b1ba2f689c14a Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 27 Nov 2023 11:14:39 +0100
Subject: [PATCH 576/990] Problem: Exception when stopping multiple VM via
 allocation endpoint

RuntimeError: dictionary changed size during iteration

Solution:
make a copy of the executions list before altering it

To reproduce.
----

You will need to set ALEPH_VM_ALLOCATION_TOKEN_HASH= and pass the
corresponding signatures:

Launch both debian VMs
```http
POST http://localhost:4020/control/allocations
Content-Type: application/json
X-Auth-Signature: test
Accept: application/json

{"persistent_vms": [], "instances": ["67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8", "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af"]}
```
Stop all instances
```http
POST http://localhost:4020/control/allocations
Content-Type: application/json
X-Auth-Signature: test
Accept: application/json

{"persistent_vms": [], "instances": []}
```

```pyhon3-traceback
Traceback (most recent call last):
  File "/home/ubuntu/.virtualenvs/aleph-vm/lib/python3.10/site-packages/aiohttp/web_protocol.py", line 433, in _handle_request
    resp = await request_handler(request)
  File "/home/ubuntu/.virtualenvs/aleph-vm/lib/python3.10/site-packages/aiohttp/web_app.py", line 504, in _handle
    resp = await handler(request)
  File "/home/ubuntu/.virtualenvs/aleph-vm/lib/python3.10/site-packages/aiohttp/web_middlewares.py", line 117, in impl
    return await handler(request)
  File "/home/ubuntu/remote-aleph/src/aleph/vm/orchestrator/supervisor.py", line 52, in server_version_middleware
    resp: web.StreamResponse = await handler(request)
  File "/home/ubuntu/remote-aleph/src/aleph/vm/orchestrator/views/__init__.py", line 254, in update_allocations
    for execution in pool.get_persistent_executions():
  File "/home/ubuntu/remote-aleph/src/aleph/vm/pool.py", line 178, in get_persistent_executions
    for _vm_hash, execution in self.executions.items():
RuntimeError: dictionary changed size during iteration
```
---
 src/aleph/vm/orchestrator/views/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 6f51c4faf..813efa202 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -251,8 +251,9 @@ async def update_allocations(request: web.Request):
 
     # First free resources from persistent programs and instances that are not scheduled anymore.
     allocations = allocation.persistent_vms | allocation.instances
-    for execution in pool.get_persistent_executions():
-        if execution.vm_hash not in allocations:
+    # Make a copy since the pool is modified
+    for execution in list(pool.executions.values()):
+        if execution.vm_hash not in allocations and execution.is_running:
             vm_type = "instance" if execution.is_instance else "persistent program"
             logger.info("Stopping %s %s", vm_type, execution.vm_hash)
             await execution.stop()

From 8a6990cbc277d892da32199a613226e312d7bc01 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 27 Nov 2023 16:19:53 +0100
Subject: [PATCH 577/990] Allow accessing then logs endpoint using the
 allocation key auth method (#489)

* Allow control key for log endpoint

Allow accessing then logs endpoint using the allocation key

which is mainly useful when debugging

* Refactor auth code in a separate function
---
 src/aleph/vm/orchestrator/views/operator.py | 32 +++++++++++++--------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 1992ab029..d008dd0f4 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -11,6 +11,7 @@
 
 from aleph.vm.models import VmExecution
 from aleph.vm.orchestrator.run import create_vm_execution
+from aleph.vm.orchestrator.views import authenticate_api_request
 from aleph.vm.orchestrator.views.authentication import (
     authenticate_websocket_message,
     require_jwk_authentication,
@@ -68,18 +69,7 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
         await ws.prepare(request)
 
         try:
-            # Authentication
-            first_message = await ws.receive_json()
-            credentials = first_message["auth"]
-            authenticated_sender = await authenticate_websocket_message(credentials)
-
-            if not is_sender_authorized(authenticated_sender, execution.message):
-                logger.debug(f"Denied request to access logs by {authenticated_sender} on {vm_hash}")
-                await ws.send_json({"status": "failed", "reason": "unauthorized sender"})
-                return web.Response(status=403, body="Unauthorized sender")
-            else:
-                logger.debug(f"Accepted request to access logs by {authenticated_sender} on {vm_hash}")
-
+            await authenticate_for_vm_or_403(execution, request, vm_hash, ws)
             await ws.send_json({"status": "connected"})
 
             # Limit the number of queues per VM
@@ -101,6 +91,24 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
         queue.empty()
 
 
+async def authenticate_for_vm_or_403(execution, request, vm_hash, ws):
+    """Allow authentication via HEADER or via websocket"""
+    if authenticate_api_request(request):
+        logger.debug(f"Accepted request to access logs via the allocatioan api key on {vm_hash}")
+        return True
+
+    first_message = await ws.receive_json()
+    credentials = first_message["auth"]
+    authenticated_sender = await authenticate_websocket_message(credentials)
+    if is_sender_authorized(authenticated_sender, execution.message):
+        logger.debug(f"Accepted request to access logs by {authenticated_sender} on {vm_hash}")
+        return True
+
+    logger.debug(f"Denied request to access logs by {authenticated_sender} on {vm_hash}")
+    await ws.send_json({"status": "failed", "reason": "unauthorized sender"})
+    raise web.HTTPForbidden(body="Unauthorized sender")
+
+
 @require_jwk_authentication
 async def operate_expire(request: web.Request, authenticated_sender: str) -> web.Response:
     """Stop the virtual machine, smoothly if possible.

From fd2b102a3c6083ba72fb8fcdb4b638d4fe47de6a Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 29 Nov 2023 17:21:58 +0100
Subject: [PATCH 578/990] Fix: Concurrent creation resulted in no
 `execution.vm`

This caused an error while the processes calling this do check for the execution to become ready.
---
 src/aleph/vm/orchestrator/run.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index f3b02f60f..53cdd3157 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -81,10 +81,6 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool) -> VmExecution:
         pool.forget_vm(vm_hash=vm_hash)
         raise HTTPInternalServerError(reason="Host did not respond to ping") from error
 
-    if not execution.vm:
-        msg = "The VM has not been created"
-        raise ValueError(msg)
-
     return execution
 
 

From ef8ae0bd08c22d0fc0affad4648cd94b0576e205 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 5 Dec 2023 11:30:09 +0100
Subject: [PATCH 579/990] QEmu support continuation (#490)

continuation of #487

Allow launching VM Instance via QEMU instead of firecracker.

This works by adding a new Controller for Qemu alongside  AlephFirecrackerProgram and AlephFirecrackerInstance and launch it if the message == Instance + hypervisor == qemu.

I'm opening this so we can get the review and discussion started but from discussion I understand it won't be for the next release since we are focusing on bugfixes for now. I can clean it up the git history afterward if needed. Please play with it plenty to find any problem I might have missed.

There is a corresponding PR in aleph-message: https://github.com/aleph-im/aleph-message/pull/78


## how to test
See this pretty complete readme on how to test it https://github.com/aleph-im/aleph-vm/blob/qemu_support/src/aleph/vm/controllers/qemu/QEMU.md

Necessary change in aleph-message were released in 0.4.1

## Modification to the code

I had to make a few change outside the Qemu controller itself to provide compatibility between all controller:

- New abstract class: AlephControllerInterface which define the shared interface between Firecracker and Qemu controllers for sharing and typing.
- Add field `support_snapshot` on controller so the controler can declare support to the SnapShotManager without the different guessing from the method we had till now.
- a Mixin to manage the cloud init config, I intended to have it used between all the controllers that need it  but at the moment I had to tweak the cloud init configuration so it's not done yet
- `get_log_queue` and `unregister_queue` so the operator can register to the Log queues without knowing the internal logic of the VM (which is different since Qemu don't use MicroVM


Refer to QEMU.md for a list of supported feature at the moment.
IMHO the main thing missing is automated testing.
moment I had to tweak the cloud init configuration so it's not done yet
    get_log_queue and unregister_queue so the operator can register to the Log queues without knowing the internal logic of the VM (which is different since Qemu don't use MicroVM

Refer to QEMU.md for a list of supported feature at the moment.
IMHO the main thing missing is automated testing.
---
 .../workflows/test-new-runtime-examples.yml   |   6 +
 .github/workflows/test-on-droplets-matrix.yml |   6 +
 docker/vm_supervisor-dev.dockerfile           |   2 +-
 examples/qemu_message_from_aleph.json         |  66 ++++
 examples/volumes/Dockerfile                   |   2 +-
 packaging/Makefile                            |   2 +-
 packaging/aleph-vm/DEBIAN/control             |   2 +-
 pyproject.toml                                |   3 +-
 src/aleph/vm/conf.py                          |  14 +
 .../vm/controllers/firecracker/executable.py  |  40 +-
 .../vm/controllers/firecracker/instance.py    |  11 +-
 .../vm/controllers/firecracker/program.py     |   9 +-
 .../firecracker/snapshot_manager.py           |   2 +-
 src/aleph/vm/controllers/interface.py         |  91 +++++
 src/aleph/vm/controllers/qemu/QEMU.md         | 158 ++++++++
 src/aleph/vm/controllers/qemu/__init__.py     |   0
 src/aleph/vm/controllers/qemu/cloudinit.py    | 148 ++++++++
 src/aleph/vm/controllers/qemu/instance.py     | 357 ++++++++++++++++++
 .../vm/hypervisors/firecracker/microvm.py     |   4 +
 src/aleph/vm/models.py                        |  66 +++-
 src/aleph/vm/orchestrator/README.md           |   2 +-
 src/aleph/vm/orchestrator/cli.py              |   1 +
 src/aleph/vm/orchestrator/views/operator.py   |  14 +-
 src/aleph/vm/pool.py                          |   3 +-
 24 files changed, 941 insertions(+), 68 deletions(-)
 create mode 100644 examples/qemu_message_from_aleph.json
 create mode 100644 src/aleph/vm/controllers/interface.py
 create mode 100644 src/aleph/vm/controllers/qemu/QEMU.md
 create mode 100644 src/aleph/vm/controllers/qemu/__init__.py
 create mode 100644 src/aleph/vm/controllers/qemu/cloudinit.py
 create mode 100644 src/aleph/vm/controllers/qemu/instance.py

diff --git a/.github/workflows/test-new-runtime-examples.yml b/.github/workflows/test-new-runtime-examples.yml
index cd0df6978..a8e62d0be 100644
--- a/.github/workflows/test-new-runtime-examples.yml
+++ b/.github/workflows/test-new-runtime-examples.yml
@@ -96,6 +96,12 @@ jobs:
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
 
+      - name: Export aleph logs
+        if: always()
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          ssh root@${DROPLET_IPV4} "journalctl -u aleph-vm-supervisor"
+
       - name: Cleanup
         if: always()
         run: |
diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index 5d8abf0b1..a457495c4 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -142,6 +142,12 @@ jobs:
             -d '{"persistent_vms": [], "instances": ["${{ matrix.check_vm.item_hash }}"]}' \
             "http://${DROPLET_IPV4}:4020/control/allocations"
 
+      - name: Export aleph logs
+        if: always()
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          ssh root@${DROPLET_IPV4} "journalctl -u aleph-vm-supervisor"
+
       - name: Cleanup
         if: always()
         run: |
diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index a840385ab..402a1e014 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -19,7 +19,7 @@ RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://s3.amazonaws.com/spec.ccf
 RUN ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker
 RUN ln /opt/firecracker/release-*/jailer-v* /opt/firecracker/jailer
 
-RUN pip3 install typing-extensions 'aleph-message==0.4.0'
+RUN pip3 install typing-extensions 'aleph-message==0.4.1'
 
 RUN mkdir -p /var/lib/aleph/vm/jailer
 
diff --git a/examples/qemu_message_from_aleph.json b/examples/qemu_message_from_aleph.json
new file mode 100644
index 000000000..65220c198
--- /dev/null
+++ b/examples/qemu_message_from_aleph.json
@@ -0,0 +1,66 @@
+{
+    "chain": "ETH",
+    "item_hash": "fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-hash",
+    "sender": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba",
+    "type": "INSTANCE",
+    "channel": "Fun-dApps",
+    "confirmed": true,
+    "content": {
+        "address": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba",
+        "allow_amend": false,
+        "variables": {
+            "VM_CUSTOM_NUMBER": "32"
+        },
+        "environment": {
+            "reproducible": true,
+            "internet": true,
+            "aleph_api": true,
+            "shared_cache": true,
+            "hypervisor": "qemu"
+        },
+        "resources": {
+            "vcpus": 1,
+            "memory": 512,
+            "seconds": 30
+        },
+        "rootfs": {
+            "parent": {
+                "ref": "549ec451d9b099cad112d4aaa2c00ac40fb6729a92ff252ff22eef0b5c3cb613",
+                "use_latest": false
+            },
+            "persistence": "host",
+            "size_mib": 5000
+        },
+        "authorized_keys": [
+            "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDj95BHGUx0/z2G/tTrEi8o49i70xvjcEUdSs3j4A33jE7pAphrfRVbuFMgFubcm8n9r5ftd/H8SjjTL4hY9YvWV5ZuMf92GUga3n4wgevvPlBszYZCy/idxFl0vtHYC1CcK9v4tVb9onhDt8FOJkf2m6PmDyvC+6tl6LwoerXTeeiKr5VnTB4KOBkammtFmix3d1X1SZd/cxdwZIHcQ7BNsqBm2w/YzVba6Z4ZnFUelBkQtMQqNs2aV51O1pFFqtZp2mM71D5d8vn9pOtqJ5QmY5IW6NypcyqKJZg5o6QguK5rdXLkc7AWro27BiaHIENl3w0wazp9EDO9zPAGJ6lz olivier@lanius"
+        ],
+        "volumes": [
+            {
+                "mount": "/opt/venv",
+                "ref": "5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51",
+                "use_latest": false
+            },
+            {
+                "comment": "Working data persisted on the VM supervisor, not available on other nodes",
+                "mount": "/var/lib/example",
+                "name": "data",
+                "persistence": "host",
+                "size_mib": 5
+            }
+        ],
+        "replaces": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba",
+        "time": 1619017773.8950517
+    },
+    "item_content": "{\"address\":\"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\",\"allow_amend\":false,\"variables\":{\"VM_CUSTOM_NUMBER\":\"32\"},\"environment\":{\"reproducible\":true,\"internet\":true,\"aleph_api\":true,\"shared_cache\":true},\"resources\":{\"vcpus\":1,\"memory\":128,\"seconds\":30},\"rootfs\":{\"parent\":{\"ref\":\"549ec451d9b099cad112d4aaa2c00ac40fb6729a92ff252ff22eef0b5c3cb613\",\"use_latest\":true},\"persistence\":\"host\",\"size_mib\":20000},\"cloud_config\":{\"password\":\"password\",\"chpasswd\":{\"expire\":\"False\"}},\"volumes\":[{\"mount\":\"/opt/venv\",\"ref\":\"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\",\"use_latest\":false},{\"comment\":\"Working data persisted on the VM supervisor, not available on other nodes\",\"mount\":\"/var/lib/example\",\"name\":\"data\",\"persistence\":\"host\",\"size_mib\":5}],\"replaces\":\"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\",\"time\":1619017773.8950517}",
+    "item_type": "inline",
+    "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b",
+    "size": 749,
+    "time": 1619017773.8950577,
+    "confirmations": [
+        {
+            "chain": "ETH",
+            "height": 12284734,
+            "hash": "0x67f2f3cde5e94e70615c92629c70d22dc959a118f46e9411b29659c2fce87cdc"
+        }
+    ]
+}
diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile
index 6b85c1fff..96dcb6b73 100644
--- a/examples/volumes/Dockerfile
+++ b/examples/volumes/Dockerfile
@@ -6,6 +6,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 RUN python3 -m venv /opt/venv
-RUN /opt/venv/bin/pip install 'aleph-message==0.4.0'
+RUN /opt/venv/bin/pip install 'aleph-message==0.4.1'
 
 CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs
diff --git a/packaging/Makefile b/packaging/Makefile
index 1808cf047..cc28a391c 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.0' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.1' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo
diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index e9947992c..ab9e37366 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils
 Section: aleph-im
 Priority: Extra
diff --git a/pyproject.toml b/pyproject.toml
index e225f6714..3886cc7fc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,7 +30,7 @@ dependencies = [
   "alembic~=1.7.6",
   "setproctitle~=1.3.3",
   "pyyaml~=6.0.1",
-  "aleph-message~=0.4.0",
+  "aleph-message~=0.4.1",
   "jwskate~=0.8.0",
   "eth-account~=0.9.0",
   "sentry-sdk~=1.31.0",
@@ -42,6 +42,7 @@ dependencies = [
   "msgpack~=1.0.7",
   "packaging~=23.2",
   "jsonschema==4.19.1",
+  "qmp==0.0.1"
 ]
 
 [project.urls]
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 33d09c3ba..581eb8f2e 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -213,6 +213,8 @@ class Settings(BaseSettings):
     # hashlib.sha256(b"secret-token").hexdigest()
     ALLOCATION_TOKEN_HASH = "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda"
 
+    ENABLE_QEMU_SUPPORT: bool = Field(default=False)
+
     # Tests on programs
 
     FAKE_DATA_PROGRAM: Optional[Path] = None
@@ -292,6 +294,18 @@ def check(self):
         if self.USE_NDP_PROXY:
             assert is_command_available("ndppd"), "Command `ndppd` not found, run `apt install ndppd`"
 
+        # Necessary for cloud-init customisation of instance
+        assert is_command_available(
+            "cloud-localds"
+        ), "Command `cloud-localds` not found, run `apt install cloud-image-utils`"
+
+        if settings.ENABLE_QEMU_SUPPORT:
+            # Qemu support
+            assert is_command_available("qemu-img"), "Command `qemu-img` not found, run `apt install qemu-utils`"
+            assert is_command_available(
+                "qemu-system-x86_64"
+            ), "Command `qemu-system-x86_64` not found, run `apt install qemu-system-x86`"
+
     def setup(self):
         os.makedirs(self.MESSAGE_CACHE, exist_ok=True)
         os.makedirs(self.CODE_CACHE, exist_ok=True)
diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index e72a84877..fcb8a6d96 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -4,7 +4,6 @@
 
 import asyncio
 import logging
-import subprocess
 from dataclasses import dataclass, field
 from multiprocessing import Process, set_start_method
 from os.path import exists, isfile
@@ -17,6 +16,7 @@
 
 from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.snapshots import CompressedDiskVolumeSnapshot
+from aleph.vm.controllers.interface import AlephVmControllerInterface
 from aleph.vm.guest_api.__main__ import run_guest_api
 from aleph.vm.hypervisors.firecracker.microvm import FirecrackerConfig, MicroVM
 from aleph.vm.network.firewall import teardown_nftables_for_vm
@@ -137,7 +137,7 @@ class VmInitNotConnectedError(Exception):
 ConfigurationType = TypeVar("ConfigurationType")
 
 
-class AlephFirecrackerExecutable(Generic[ConfigurationType]):
+class AlephFirecrackerExecutable(Generic[ConfigurationType], AlephVmControllerInterface):
     vm_id: int
     vm_hash: ItemHash
     resources: AlephFirecrackerResources
@@ -150,6 +150,7 @@ class AlephFirecrackerExecutable(Generic[ConfigurationType]):
     guest_api_process: Optional[Process] = None
     is_instance: bool
     _firecracker_config: Optional[FirecrackerConfig] = None
+    support_snapshot: bool
 
     def __init__(
         self,
@@ -186,26 +187,6 @@ def __init__(
         self.guest_api_process = None
         self._firecracker_config = None
 
-    def get_vm_ip(self) -> Optional[str]:
-        if self.tap_interface:
-            return self.tap_interface.guest_ip.with_prefixlen
-        return None
-
-    def get_vm_route(self) -> Optional[str]:
-        if self.tap_interface:
-            return str(self.tap_interface.host_ip).split("/", 1)[0]
-        return None
-
-    def get_vm_ipv6(self) -> Optional[str]:
-        if self.tap_interface:
-            return self.tap_interface.guest_ipv6.with_prefixlen
-        return None
-
-    def get_vm_ipv6_gateway(self) -> Optional[str]:
-        if self.tap_interface:
-            return str(self.tap_interface.host_ipv6.ip)
-        return None
-
     def to_dict(self):
         """Dict representation of the virtual machine. Used to record resource usage and for JSON serialization."""
         if self.fvm.proc and psutil:
@@ -301,3 +282,18 @@ async def teardown(self):
 
     async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
         raise NotImplementedError()
+
+    async def get_log_queue(self) -> asyncio.Queue:
+        queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
+        # Limit the number of queues per VM
+
+        if len(self.fvm.log_queues) > 20:
+            logger.warning("Too many log queues, dropping the oldest one")
+            self.fvm.log_queues.pop(0)
+        self.fvm.log_queues.append(queue)
+        return queue
+
+    async def unregister_queue(self, queue: asyncio.Queue):
+        if queue in self.fvm.log_queues:
+            self.fvm.log_queues.remove(queue)
+        queue.empty()
diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index 485f5e6a4..fd44325f9 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -58,6 +58,7 @@ class AlephFirecrackerInstance(AlephFirecrackerExecutable):
     resources: AlephInstanceResources
     latest_snapshot: Optional[DiskVolumeSnapshot]
     is_instance = True
+    support_snapshot = False
 
     def __init__(
         self,
@@ -118,7 +119,7 @@ async def wait_for_init(self) -> None:
         """Wait for the init process of the instance to be ready."""
         assert self.enable_networking and self.tap_interface, f"Network not enabled for VM {self.vm_id}"
 
-        ip = self.get_vm_ip()
+        ip = self.get_ip()
         if not ip:
             msg = "Host IP not available"
             raise ValueError(msg)
@@ -188,10 +189,10 @@ def _create_network_file(self) -> bytes:
 
         assert self.enable_networking and self.tap_interface, f"Network not enabled for VM {self.vm_id}"
 
-        ip = self.get_vm_ip()
-        route = self.get_vm_route()
-        ipv6 = self.get_vm_ipv6()
-        ipv6_gateway = self.get_vm_ipv6_gateway()
+        ip = self.get_ip()
+        route = self.get_ip_route()
+        ipv6 = self.get_ipv6()
+        ipv6_gateway = self.get_ipv6_gateway()
 
         network = {
             "ethernets": {
diff --git a/src/aleph/vm/controllers/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
index c98e817d5..0cacf4123 100644
--- a/src/aleph/vm/controllers/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -258,6 +258,7 @@ class AlephFirecrackerProgram(AlephFirecrackerExecutable[ProgramVmConfiguration]
     vm_configuration: ProgramVmConfiguration | None
     resources: AlephProgramResources
     is_instance = False
+    support_snapshot = False
 
     def __init__(
         self,
@@ -342,14 +343,14 @@ async def _setup_configuration(
         machine to send this configuration. Other modes may use Cloud-init, ..."""
         reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
 
-        ip = self.get_vm_ip()
+        ip = self.get_ip()
         if ip:
             # The ip and route should not contain the network mask in order to maintain
             # compatibility with the existing runtimes.
             ip = ip.split("/", 1)[0]
-        route = self.get_vm_route()
-        ipv6 = self.get_vm_ipv6()
-        ipv6_gateway = self.get_vm_ipv6_gateway()
+        route = self.get_ip_route()
+        ipv6 = self.get_ipv6()
+        ipv6_gateway = self.get_ipv6_gateway()
 
         if not settings.DNS_NAMESERVERS:
             msg = "Invalid configuration: DNS nameservers missing"
diff --git a/src/aleph/vm/controllers/firecracker/snapshot_manager.py b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
index e3fd42032..2a42774e0 100644
--- a/src/aleph/vm/controllers/firecracker/snapshot_manager.py
+++ b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
@@ -96,7 +96,7 @@ def run_snapshots(self) -> None:
         job_thread.start()
 
     async def start_for(self, vm: AlephFirecrackerExecutable, frequency: Optional[int] = None) -> None:
-        if not vm.is_instance:
+        if not vm.support_snapshot:
             msg = "Snapshots are not implemented for programs."
             raise NotImplementedError(msg)
 
diff --git a/src/aleph/vm/controllers/interface.py b/src/aleph/vm/controllers/interface.py
new file mode 100644
index 000000000..32caf84a0
--- /dev/null
+++ b/src/aleph/vm/controllers/interface.py
@@ -0,0 +1,91 @@
+import asyncio
+import logging
+from abc import ABC
+from asyncio.subprocess import Process
+from typing import Any, Coroutine, Optional
+
+from aleph_message.models import ItemHash
+from aleph_message.models.execution.environment import MachineResources
+
+from aleph.vm.controllers.firecracker.snapshots import CompressedDiskVolumeSnapshot
+from aleph.vm.network.interfaces import TapInterface
+
+logger = logging.getLogger(__name__)
+
+
+class AlephVmControllerInterface(ABC):
+    vm_id: int
+    """id in the VMPool, attributed at execution"""
+    vm_hash: ItemHash
+    """identifier for the VM definition, linked to an Aleph Message"""
+    resources: Any
+    """local resource for the machine"""
+    enable_console: bool
+    enable_networking: bool
+    """enable networking for this VM"""
+    hardware_resources: MachineResources
+    support_snapshot: bool
+    """Does this controller support snapshotting"""
+    guest_api_process: Optional[Process] = None
+    tap_interface: Optional[TapInterface] = None
+    """Network interface used for this VM"""
+
+    def get_ip(self) -> Optional[str]:
+        if self.tap_interface:
+            return self.tap_interface.guest_ip.with_prefixlen
+        return None
+
+    def get_ip_route(self) -> Optional[str]:
+        if self.tap_interface:
+            return str(self.tap_interface.host_ip).split("/", 1)[0]
+        return None
+
+    def get_ipv6(self) -> Optional[str]:
+        if self.tap_interface:
+            return self.tap_interface.guest_ipv6.with_prefixlen
+        return None
+
+    def get_ipv6_gateway(self) -> Optional[str]:
+        if self.tap_interface:
+            return str(self.tap_interface.host_ipv6.ip)
+        return None
+
+    def to_dict(self):
+        """Dict representation of the virtual machine. Used to record resource usage and for JSON serialization."""
+        raise NotImplementedError()
+
+    async def setup(self):
+        """Configuration done before the VM process is started"""
+        raise NotImplementedError()
+
+    async def start(self):
+        """Start the VM process"""
+        raise NotImplementedError()
+
+    async def wait_for_init(self) -> None:
+        """Wait for the init process of the virtual machine to be ready.
+        May be empty."""
+        pass
+
+    async def configure(self) -> None:
+        """Configuration done after the VM process is started"""
+        raise NotImplementedError()
+
+    async def start_guest_api(self):
+        raise NotImplementedError()
+
+    async def stop_guest_api(self):
+        raise NotImplementedError()
+
+    async def teardown(self) -> Coroutine:
+        raise NotImplementedError()
+
+    async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
+        """Must be implement if self.support_snapshot is True"""
+        raise NotImplementedError()
+
+    async def get_log_queue(self) -> asyncio.Queue:
+        raise NotImplementedError()
+
+    async def unregister_queue(self, queue: asyncio.Queue):
+        raise NotImplementedError()
diff --git a/src/aleph/vm/controllers/qemu/QEMU.md b/src/aleph/vm/controllers/qemu/QEMU.md
new file mode 100644
index 000000000..e79a6c8b6
--- /dev/null
+++ b/src/aleph/vm/controllers/qemu/QEMU.md
@@ -0,0 +1,158 @@
+# Qemu support
+
+## Requirements
+Commands : qemu, cloud-ds, qemu-img
+
+These are installable via 
+`apt install cloud-image-utils qemu-utils qemu-system-x86`
+
+This branch depends on the version 0.4.1 of `aleph-message` that add the `hypervisor` field. The easiest way is to install tha version using `pip install -e .`
+
+To create a local venv use the `--system-site-packages` option so it can acess nftables 
+
+## To test launching a VM instance
+
+Launch aleph.vm.orchestrator with the following environment variables
+
+
+```environ
+ALEPH_VM_FAKE_INSTANCE_BASE=/home/olivier/Projects/qemu-quickstart/jammy-server-cloudimg-amd64.img
+ALEPH_VM_FAKE_INSTANCE_MESSAGE=/home/olivier/Projects/aleph/aleph-vm/examples/qemu_message_from_aleph.json
+ALEPH_VM_USE_FAKE_INSTANCE_BASE=1
+# set test as the allocation password
+ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
+
+```
+
+Where `ALEPH_VM_FAKE_INSTANCE_BASE` is the path to the base disk image. You can get the Ubuntu one via:
+`wget https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img`
+
+You can use any base VM image supporting cloud-init. cloud-init support is mandatory because it is used to set up the network.  
+
+
+To only launch the VM instance, use the parameter:
+`--run-fake-instance`
+
+You can then try to connect via ssh to it's ip. Wait a minute or so for it to set up properly with the network
+
+Or launching the whole supervisor server (no params), then launch the VM via http
+
+```http request
+### Start fake VM
+POST http://localhost:4020/control/allocations
+Content-Type: application/json
+X-Auth-Signature: test
+Accept: application/json
+
+{"persistent_vms": [], "instances": ["decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca"]}
+```
+
+After a minutes or two you should be able to SSH into the VM. Check in the log for the VM ip. 
+If you used an Ubuntu image the username should be ubuntu
+
+You can then stop the VM using
+```http request
+### Stop the VM
+POST http://localhost:4020/control/machine/decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca/stop
+Accept: application/json
+```
+(you will need to comment @require_jwk_authentication)
+
+# Connecting to the VM via your own ssh key
+In local development, if you want to connect via ssh to the VM and you don't have your
+ key already included in you base image or inside the aleph message, you can configure it in the following way.
+
+First set your key in the environment variable ALEPH_VM_DEVELOPER_SSH_KEYS in the json format. You can add it directly in the `.env` file
+```env
+ALEPH_VM_DEVELOPER_SSH_KEYS=["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDj95BHGUx0/z2G/tTrEi8o49i70xvjcEUdSs3j4A33jE7pAphrfRVbuFMgFubcm8n9r5ftd/H8SjjTL4hY9YvWV5ZuMf92GUga3n4wgevvPlBszYZCy/idxFl0vtHYC1CcK9v4tVb9onhDt8FOJkf2m6PmDyvC+6tl6LwoerXTeeiKr5VnTB4KOBkammtFmix3d1X1SZd/cxdwZIHcQ7BNsqBm2w/YzVba6Z4ZnFUelBkQtMQqNs2aV51O1pFFqtZp2mM71D5d8vn9pOtqJ5QmY5IW6NypcyqKJZg5o6QguK5rdXLkc7AWro27BiaHIENl3w0wazp9EDO9zPAGJ6lz olivier@lanius"]
+```
+
+Then pass the `--developer-ssh-keys` as an argument when starting the supervisor.
+
+Cloud init support for settings the ssh key in the VM image is required, this is the same mechanism and settings as for firecracker program, of course this is not for production use.
+
+## Using the CentOS distro for your VM
+Qemu support has also been tested with CentOS 7
+
+To test it locally
+1. Download the CentOS cloud image distro:
+`curl -LO -C - http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2.xz`
+2. It is in a compressed format, so you will  need to uncompress it
+```unxz CentOS-7-x86_64-GenericCloud.qcow2.xz```
+3. Set the fake instance base to point to the file `CentOS-7-x86_64-GenericCloud.qcow2
+(either via --fake-instance base parameter or the  ALEPH_VM_FAKE_INSTANCE_BASE environment)
+4. Launch it as per instruction aboce
+5. To ssh use the user: `centos` 
+
+## Using the Debian distro for your VM
+Debian QEMU Support has been tested with Debian 12 bookworm. Download the image from https://cloud.debian.org/images/cloud/
+
+Use the AMD64 `genericcloud` image. The `generic` should work too but `genericcloud` is smaller as it doesn't contain unnecessary hardware drivers.
+
+e.g `wget https://cloud.debian.org/images/cloud/bookworm/20231013-1532/debian-12-genericcloud-amd64-20231013-1532.qcow2`
+
+See instruction above for the rest. The default user is `root`
+
+# Check the log via Websocket
+You can stream the logs from the VM using, the following python example script. 
+Caveat: This requires to temporarly disable auth on this endpoint, you need the print system log settings to be active `ALEPH_VM_PRINT_SYSTEM_LOGS=1`. The system only stream new log content from the VM not the old one.
+```python
+import json
+import sys
+
+import asyncio
+import aiohttp
+
+
+def on_message(content):
+    try:
+        msg = json.loads(content)
+        fd = sys.stderr if msg["type"] == "stderr" else sys.stdout
+        print("<", msg["message"], file=fd, end="")
+    except:
+        print("unable to parse", content)
+
+
+async def tail_websocket(url):
+    async with aiohttp.ClientSession() as session:
+        async with session.ws_connect(url) as ws:
+            print(f"connected to {url}")
+            async for msg in ws:
+                if msg.type == aiohttp.WSMsgType.TEXT:
+                    on_message(msg.data)
+                elif msg.type == aiohttp.WSMsgType.CLOSED:
+                    print("closed")
+                    break
+                elif msg.type == aiohttp.WSMsgType.ERROR:
+                    print("Error", msg)
+                    break
+
+
+vm_hash = "decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca"
+url = f"ws://localhost:4020/control/machine/{vm_hash}/logs"
+loop = asyncio.get_event_loop()
+loop.run_until_complete(tail_websocket(url))
+```
+
+
+# TODO
+- [x] Launch
+- [x] Message format
+- [x] Network
+- [x] Cloud init support
+- [x] Download ressource
+- [ ] snapshot
+- [ ] Multi volume
+- [x] fix logs
+- [ ] Testing
+- [x] Support raw format for base image
+- [x] More testing with different Distro:
+  - [x] Centos
+  - [x] Debian
+  - [x] Alpine (do not support centos no cloud)
+- [ ] Document for user how to build their own images
+- [x] Allow ssh developer key
+- [ ] Automated testing in CI
+- [x] Output the whole serial console in logs
+- [x] Test code for websocket logs
+- [ ] Multi Layer Qcow image?
diff --git a/src/aleph/vm/controllers/qemu/__init__.py b/src/aleph/vm/controllers/qemu/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/aleph/vm/controllers/qemu/cloudinit.py b/src/aleph/vm/controllers/qemu/cloudinit.py
new file mode 100644
index 000000000..7a12461ef
--- /dev/null
+++ b/src/aleph/vm/controllers/qemu/cloudinit.py
@@ -0,0 +1,148 @@
+"""Generate a cloud-init ISO image for the VM configuration.
+
+This module automates the creation of a cloud-init ISO image, which is utilized for configuring the
+Virtual Machine. The configurations included in this process are the hostname, SSH keys, and network settings.
+
+The generated ISO image, created using the `cloud-localds` command, is intended to be mounted as a CD-ROM inside the
+VM. Upon booting, the VM's cloud-init service detects this CD-ROM and applies the configurations based on the data it
+contains.
+
+Refer to the cloud-init documentation, in particular the NoCloud datasource which is the method we are using.
+https://cloudinit.readthedocs.io/en/latest/reference/datasources/nocloud.html
+
+See also the cloud-localds  man page (1)
+"""
+import base64
+import json
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import Union
+
+import yaml
+from aleph_message.models import ItemHash
+
+from aleph.vm.conf import settings
+from aleph.vm.controllers.interface import AlephVmControllerInterface
+from aleph.vm.hypervisors.firecracker.config import Drive
+from aleph.vm.utils import is_command_available, run_in_subprocess
+
+
+def get_hostname_from_hash(vm_hash: ItemHash) -> str:
+    item_hash_binary: bytes = base64.b16decode(vm_hash.encode().upper())
+    return base64.b32encode(item_hash_binary).decode().strip("=").lower()
+
+
+def encode_user_data(hostname, ssh_authorized_keys) -> bytes:
+    """Creates user data configuration file for cloud-init tool"""
+    config: dict[str, Union[str, bool, list[str]]] = {
+        "hostname": hostname,
+        "disable_root": False,
+        "ssh_pwauth": False,
+        "ssh_authorized_keys": ssh_authorized_keys,
+        "resize_rootfs": True,
+    }
+    cloud_config_header = "#cloud-config\n"
+    config_output = yaml.safe_dump(config, default_flow_style=False, sort_keys=False)
+    content = (cloud_config_header + config_output).encode()
+    return content
+
+
+def create_metadata_file(hostname, vm_id) -> bytes:
+    """Creates metadata configuration file for cloud-init tool"""
+    metadata = {
+        "instance-id": f"iid-instance-{vm_id}",
+        "local-hostname": hostname,
+    }
+    return json.dumps(metadata).encode()
+
+
+def create_network_file(ip, ipv6, ipv6_gateway, nameservers, route) -> bytes:
+    """Creates network configuration file for cloud-init tool"""
+    network = {
+        "ethernets": {
+            "eth0": {
+                # Match the config to the `virtio` driver since the network interface name is not constant across distro
+                "match": {"driver": "virtio_net"},
+                "addresses": [ip, ipv6],
+                "gateway4": route,
+                "gateway6": ipv6_gateway,
+                "nameservers": {
+                    "addresses": nameservers,
+                },
+                # there is a bug in Centos 7 where it will try DHCP if the key is present, even if set to false
+                # https://stackoverflow.com/questions/59757022/set-static-ip-using-cloud-init-on-centos-7-with-terraform-kvm
+                # Thus theses are commented for now
+                # "dhcp4": False,
+                # "dhcp6": False,
+            },
+        },
+        "version": 2,
+    }
+    return yaml.safe_dump(network, default_flow_style=False, sort_keys=False).encode()
+
+
+async def create_cloud_init_drive_image(
+    disk_image_path, hostname, vm_id, ip, ipv6, ipv6_gateway, nameservers, route, ssh_authorized_keys
+):
+    with (
+        NamedTemporaryFile() as user_data_config_file,
+        NamedTemporaryFile() as network_config_file,
+        NamedTemporaryFile() as metadata_config_file,
+    ):
+        user_data = encode_user_data(hostname, ssh_authorized_keys)
+        user_data_config_file.write(user_data)
+        user_data_config_file.flush()
+        network_config = create_network_file(ip, ipv6, ipv6_gateway, nameservers, route)
+        network_config_file.write(network_config)
+        network_config_file.flush()
+
+        metadata_config = create_metadata_file(hostname, vm_id)
+        metadata_config_file.write(metadata_config)
+        metadata_config_file.flush()
+
+        await run_in_subprocess(
+            [
+                "cloud-localds",
+                f"--network-config={network_config_file.name}",
+                str(disk_image_path),
+                user_data_config_file.name,
+                metadata_config_file.name,
+            ]
+        )
+
+
+class CloudInitMixin(AlephVmControllerInterface):
+    async def _create_cloud_init_drive(self) -> Drive:
+        """Creates the cloud-init volume to configure and set up the VM"""
+        ssh_authorized_keys = self.resources.message_content.authorized_keys or []
+        if settings.USE_DEVELOPER_SSH_KEYS:
+            ssh_authorized_keys += settings.DEVELOPER_SSH_KEYS
+        ip = self.get_ip()
+        route = self.get_ip_route()
+        ipv6 = self.get_ipv6()
+        ipv6_gateway = self.get_ipv6_gateway()
+        vm_id = self.vm_id
+        nameservers = settings.DNS_NAMESERVERS
+        hostname = get_hostname_from_hash(self.vm_hash)
+
+        disk_image_path: Path = settings.EXECUTION_ROOT / f"cloud-init-{self.vm_hash}.img"
+        assert is_command_available("cloud-localds")
+
+        await create_cloud_init_drive_image(
+            disk_image_path,
+            hostname,
+            vm_id,
+            ip,
+            ipv6,
+            ipv6_gateway,
+            nameservers,
+            route,
+            ssh_authorized_keys,
+        )
+
+        return Drive(
+            drive_id="Fake",
+            path_on_host=disk_image_path,
+            is_root_device=False,
+            is_read_only=True,
+        )
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
new file mode 100644
index 000000000..592a01f44
--- /dev/null
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -0,0 +1,357 @@
+import asyncio
+import json
+import logging
+import shutil
+import sys
+from asyncio import Task
+from asyncio.subprocess import Process
+from typing import Generic, Optional, TypeVar, Union
+
+import psutil
+import qmp
+from aleph_message.models import ItemHash
+from aleph_message.models.execution.environment import MachineResources
+from aleph_message.models.execution.instance import RootfsVolume
+from aleph_message.models.execution.volume import PersistentVolume, VolumePersistence
+
+from aleph.vm.conf import settings
+from aleph.vm.controllers.firecracker.executable import (
+    AlephFirecrackerResources,
+    VmSetupError,
+)
+from aleph.vm.controllers.interface import AlephVmControllerInterface
+from aleph.vm.controllers.qemu.cloudinit import CloudInitMixin
+from aleph.vm.network.firewall import teardown_nftables_for_vm
+from aleph.vm.network.interfaces import TapInterface
+from aleph.vm.storage import get_rootfs_base_path
+from aleph.vm.utils import HostNotFoundError, ping, run_in_subprocess
+
+logger = logging.getLogger(__name__)
+
+
+class AlephQemuResources(AlephFirecrackerResources):
+    async def download_all(self):
+        volume = self.message_content.rootfs
+        parent_image_path = await get_rootfs_base_path(volume.parent.ref)
+        self.rootfs_path = await self.make_writable_volume(parent_image_path, volume)
+        return
+
+    async def make_writable_volume(self, parent_image_path, volume: Union[PersistentVolume, RootfsVolume]):
+        """Create a new qcow2 image file based on the passed one, that we give to the VM to write onto"""
+        qemu_img_path = shutil.which("qemu-img")
+        volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
+
+        # detect the image format
+        out_json = await run_in_subprocess([qemu_img_path, "info", str(parent_image_path), "--output=json"])
+        out = json.loads(out_json)
+        parent_format = out.get("format", None)
+        if parent_format is None:
+            raise VmSetupError(f"Failed to detect format for {volume}: {out_json}")
+        if parent_format not in ("qcow2", "raw"):
+            raise VmSetupError(f"Format {parent_format} for {volume} unhandled by QEMU hypervisor")
+
+        dest_path = settings.PERSISTENT_VOLUMES_DIR / self.namespace / f"{volume_name}.qcow2"
+        # Do not override if user asked for host persistance.
+        if dest_path.exists() and volume.persistence == VolumePersistence.host:
+            return dest_path
+
+        dest_path.parent.mkdir(parents=True, exist_ok=True)
+        size_in_bytes = int(volume.size_mib * 1024 * 1024)
+
+        await run_in_subprocess(
+            [
+                qemu_img_path,
+                "create",
+                "-f",  # Format
+                "qcow2",
+                "-F",
+                parent_format,
+                "-b",
+                str(parent_image_path),
+                str(dest_path),
+                str(size_in_bytes),
+            ]
+        )
+        return dest_path
+
+
+ConfigurationType = TypeVar("ConfigurationType")
+
+
+class AlephQemuInstance(Generic[ConfigurationType], CloudInitMixin, AlephVmControllerInterface):
+    vm_id: int
+    vm_hash: ItemHash
+    resources: AlephQemuResources
+    enable_console: bool
+    enable_networking: bool
+    hardware_resources: MachineResources
+    tap_interface: Optional[TapInterface] = None
+    vm_configuration: Optional[ConfigurationType]
+    is_instance: bool
+    qemu_process: Optional[Process]
+    support_snapshot = False
+    qmp_socket_path = None
+
+    def __repr__(self):
+        return f""
+
+    def __str__(self):
+        return f"vm-{self.vm_id}"
+
+    def __init__(
+        self,
+        vm_id: int,
+        vm_hash: ItemHash,
+        resources: AlephQemuResources,
+        enable_networking: bool = False,
+        enable_console: Optional[bool] = None,
+        hardware_resources: MachineResources = MachineResources(),
+        tap_interface: Optional[TapInterface] = None,
+    ):
+        self.vm_id = vm_id
+        self.vm_hash = vm_hash
+        self.resources = resources
+        if enable_console is None:
+            enable_console = settings.PRINT_SYSTEM_LOGS
+        self.enable_console = enable_console
+        self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING
+        self.hardware_resources = hardware_resources
+        self.tap_interface = tap_interface
+
+    def to_dict(self):
+        """Dict representation of the virtual machine. Used to record resource usage and for JSON serialization."""
+        if self.qemu_process and psutil:
+            # The firecracker process is still running and process information can be obtained from `psutil`.
+            try:
+                p = psutil.Process(self.qemu_process.pid)
+                pid_info = {
+                    "status": p.status(),
+                    "create_time": p.create_time(),
+                    "cpu_times": p.cpu_times(),
+                    "cpu_percent": p.cpu_percent(),
+                    "memory_info": p.memory_info(),
+                    "io_counters": p.io_counters(),
+                    "open_files": p.open_files(),
+                    "connections": p.connections(),
+                    "num_threads": p.num_threads(),
+                    "num_ctx_switches": p.num_ctx_switches(),
+                }
+            except psutil.NoSuchProcess:
+                logger.warning("Cannot read process metrics (process %s not found)", self.qemu_process)
+                pid_info = None
+        else:
+            pid_info = None
+
+        return {
+            "process": pid_info,
+            **self.__dict__,
+        }
+
+    async def setup(self):
+        pass
+
+    async def start(self):
+        logger.debug(f"Starting Qemu: {self} ")
+        # Based on the command
+        #  qemu-system-x86_64 -enable-kvm -m 2048 -net nic,model=virtio
+        # -net tap,ifname=tap0,script=no,downscript=no -drive file=alpine.qcow2,media=disk,if=virtio -nographic
+
+        qemu_path = shutil.which("qemu-system-x86_64")
+        image_path = self.resources.rootfs_path
+        vcpu_count = self.hardware_resources.vcpus
+        mem_size_mib = self.hardware_resources.memory
+        mem_size_mb = int(mem_size_mib / 1024 / 1024 * 1000 * 1000)
+        # hardware_resources.published ports -> not implemented at the moment
+        # hardware_resources.seconds -> only for microvm
+
+        monitor_socket_path = settings.EXECUTION_ROOT / (str(self.vm_id) + "-monitor.socket")
+        self.qmp_socket_path = qmp_socket_path = settings.EXECUTION_ROOT / (str(self.vm_id) + "-qmp.socket")
+
+        args = [
+            qemu_path,
+            "-enable-kvm",
+            "-nodefaults",
+            "-m",
+            str(mem_size_mb),
+            "-smp",
+            str(vcpu_count),
+            # Disable floppy
+            "-fda",
+            "",
+            # "-snapshot",  # Do not save anything to disk
+            "-drive",
+            f"file={image_path},media=disk,if=virtio",
+            # To debug you can pass gtk or curses instead
+            "-display",
+            "none",
+            "--no-reboot",  # Rebooting from inside the VM shuts down the machine
+            # Listen for commands on this socket
+            "-monitor",
+            f"unix:{monitor_socket_path},server,nowait",
+            # Listen for commands on this socket (QMP protocol in json). Supervisor use it to send shutdown or start
+            # command
+            "-qmp",
+            f"unix:{qmp_socket_path},server,nowait",
+            # Tell to put the output to std fd, so we can include them in the log
+            "-serial",
+            "stdio",
+            # Uncomment for debug
+            # "-serial", "telnet:localhost:4321,server,nowait",
+        ]
+        if self.tap_interface:
+            interface_name = self.tap_interface.device_name
+            # script=no, downscript=no tell qemu not to try to set up the network itself
+            args += ["-net", "nic,model=virtio", "-net", f"tap,ifname={interface_name},script=no,downscript=no"]
+
+        cloud_init_drive = await self._create_cloud_init_drive()
+        if cloud_init_drive:
+            args += ["-cdrom", f"{cloud_init_drive.path_on_host}"]
+
+        try:
+            print(*args)
+            self.qemu_process = proc = await asyncio.create_subprocess_exec(
+                *args,
+                stdin=asyncio.subprocess.DEVNULL,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+
+            logger.debug(f"setup done {self}, {proc}")
+
+            async def handle_termination(proc: Process):
+                await proc.wait()
+                logger.info(f"{self} Process terminated with {proc.returncode} : {str(args)}")
+
+            loop = asyncio.get_running_loop()
+            loop.create_task(handle_termination(proc))
+        except Exception:
+            # Stop the VM and clear network interfaces in case any error prevented the start of the virtual machine.
+            logger.error("VM startup failed, cleaning up network")
+            if self.enable_networking:
+                teardown_nftables_for_vm(self.vm_id)
+            if self.tap_interface:
+                await self.tap_interface.delete()
+            raise
+
+        if self.enable_console:
+            self.process_logs()
+
+        await self.wait_for_init()
+        logger.debug(f"started qemu vm {self} on {self.get_ip()}")
+
+    async def wait_for_init(self) -> None:
+        """Wait for the init process of the instance to be ready."""
+        assert self.enable_networking and self.tap_interface, f"Network not enabled for VM {self.vm_id}"
+
+        ip = self.get_ip()
+        if not ip:
+            msg = "Host IP not available"
+            raise ValueError(msg)
+
+        ip = ip.split("/", 1)[0]
+
+        attempts = 30
+        timeout_seconds = 2.0
+
+        for attempt in range(attempts):
+            try:
+                await ping(ip, packets=1, timeout=timeout_seconds)
+                return
+            except HostNotFoundError:
+                if attempt < (attempts - 1):
+                    continue
+                else:
+                    raise
+
+    async def configure(self):
+        """Nothing to configure, we do the configuration via cloud init"""
+        pass
+
+    async def start_guest_api(self):
+        pass
+
+    async def stop_guest_api(self):
+        pass
+
+    stdout_task: Optional[Task] = None
+    stderr_task: Optional[Task] = None
+    log_queues: list[asyncio.Queue] = []
+
+    async def teardown(self):
+        if self.stdout_task:
+            self.stdout_task.cancel()
+        if self.stderr_task:
+            self.stderr_task.cancel()
+
+        self._shutdown()
+
+        if self.enable_networking:
+            teardown_nftables_for_vm(self.vm_id)
+            if self.tap_interface:
+                await self.tap_interface.delete()
+        await self.stop_guest_api()
+
+    async def _process_stdout(self):
+        while not self.qemu_process:
+            await asyncio.sleep(0.01)  # Todo: Use signal here
+        while True:
+            line = await self.qemu_process.stdout.readline()
+            if not line:  # FD is closed nothing more will come
+                print(self, "EOF")
+                return
+            for queue in self.log_queues:
+                await queue.put(("stdout", line))
+            print(self, line.decode().strip())
+
+    async def _process_stderr(self):
+        while not self.qemu_process:
+            await asyncio.sleep(0.01)  # Todo: Use signal here
+        while True:
+            line = await self.qemu_process.stderr.readline()
+            if not line:  # FD is closed nothing more will come
+                print(self, "EOF")
+                return
+            for queue in self.log_queues:
+                await queue.put(("stderr", line))
+            print(self, line.decode().strip(), file=sys.stderr)
+
+    def process_logs(self) -> tuple[Task, Task]:
+        """Start two tasks to process the stdout and stderr
+
+        It will stream their content to queues registered on self.log_queues
+        It will also print them"""
+
+        loop = asyncio.get_running_loop()
+        self.stdout_task = loop.create_task(self._process_stdout())
+        self.stderr_task = loop.create_task(self._process_stderr())
+        return self.stdout_task, self.stderr_task
+
+    def _get_qmpclient(self) -> Optional[qmp.QEMUMonitorProtocol]:
+        if not self.qmp_socket_path:
+            return None
+        client = qmp.QEMUMonitorProtocol(str(self.qmp_socket_path))
+        client.connect()
+        return client
+
+    def _shutdown(self):
+        client = self._get_qmpclient()
+        if client:
+            resp = client.command("system_powerdown")
+            if not resp == {}:
+                logger.warning("unexpected answer from VM", resp)
+            client.close()
+            self.qmp_socket_path = None
+
+    async def get_log_queue(self) -> asyncio.Queue:
+        queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
+        # Limit the number of queues per VM
+        if len(self.log_queues) > 20:
+            logger.warning("Too many log queues, dropping the oldest one")
+            self.log_queues.pop(0)
+        self.log_queues.append(queue)
+        return queue
+
+    async def unregister_queue(self, queue: asyncio.Queue):
+        if queue in self.log_queues:
+            self.log_queues.remove(queue)
+        queue.empty()
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index e4be60e38..5085cd618 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -50,6 +50,10 @@ def system(command):
 
 
 async def setfacl():
+    """Give current user permission to access /dev/kvm via acl"""
+    if os.access("/dev/kvm", os.R_OK | os.W_OK):
+        return
+
     user = getuid()
     cmd = f"sudo setfacl -m u:{user}:rw /dev/kvm"
     proc = await asyncio.create_subprocess_shell(cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 0fc5fd03e..cb9f8b84d 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -10,8 +10,10 @@
     ExecutableContent,
     InstanceContent,
     ItemHash,
+    MessageType,
     ProgramContent,
 )
+from aleph_message.models.execution.environment import HypervisorType
 
 from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.executable import AlephFirecrackerExecutable
@@ -21,6 +23,8 @@
     AlephFirecrackerResources,
     AlephProgramResources,
 )
+from aleph.vm.controllers.interface import AlephVmControllerInterface
+from aleph.vm.controllers.qemu.instance import AlephQemuInstance, AlephQemuResources
 from aleph.vm.network.interfaces import TapInterface
 from aleph.vm.orchestrator.metrics import (
     ExecutionRecord,
@@ -34,7 +38,6 @@
 if TYPE_CHECKING:
     from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -64,7 +67,7 @@ class VmExecution:
     original: ExecutableContent
     message: ExecutableContent
     resources: Optional[AlephFirecrackerResources] = None
-    vm: Optional[AlephFirecrackerExecutable] = None
+    vm: Optional[Union[AlephFirecrackerExecutable, AlephQemuInstance]] = None
 
     times: VmExecutionTimes
 
@@ -90,6 +93,11 @@ def is_program(self):
     def is_instance(self):
         return isinstance(self.message, InstanceContent)
 
+    @property
+    def hypervisor(self):
+        # default to firecracker for retro compat
+        return self.message.environment.hypervisor or HypervisorType.firecracker
+
     @property
     def becomes_ready(self):
         return self.ready_event.wait
@@ -135,24 +143,29 @@ async def prepare(self):
                 return
 
             self.times.preparing_at = datetime.now(tz=timezone.utc)
+            resources = None
             if self.is_program:
                 resources = AlephProgramResources(self.message, namespace=self.vm_hash)
             elif self.is_instance:
-                resources = AlephInstanceResources(self.message, namespace=self.vm_hash)
-            else:
+                if self.hypervisor == HypervisorType.firecracker:
+                    resources = AlephInstanceResources(self.message, namespace=self.vm_hash)
+                elif self.hypervisor == HypervisorType.qemu:
+                    resources = AlephQemuResources(self.message, namespace=self.vm_hash)
+
+            if not resources:
                 msg = "Unknown executable message type"
-                raise ValueError(msg)
+                raise ValueError(msg, repr(self.message))
             await resources.download_all()
             self.times.prepared_at = datetime.now(tz=timezone.utc)
             self.resources = resources
 
-    async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None) -> AlephFirecrackerExecutable:
+    async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None) -> AlephVmControllerInterface:
         if not self.resources:
             msg = "Execution resources must be configured first"
             raise ValueError(msg)
         self.times.starting_at = datetime.now(tz=timezone.utc)
 
-        vm: Union[AlephFirecrackerProgram, AlephFirecrackerInstance]
+        vm: AlephVmControllerInterface
         if self.is_program:
             assert isinstance(self.resources, AlephProgramResources)
             self.vm = vm = AlephFirecrackerProgram(
@@ -163,17 +176,32 @@ async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None)
                 hardware_resources=self.message.resources,
                 tap_interface=tap_interface,
             )
+        elif self.is_instance:
+            if self.hypervisor == HypervisorType.firecracker:
+                assert isinstance(self.resources, AlephInstanceResources)
+                self.vm = vm = AlephFirecrackerInstance(
+                    vm_id=vm_id,
+                    vm_hash=self.vm_hash,
+                    resources=self.resources,
+                    enable_networking=self.message.environment.internet,
+                    hardware_resources=self.message.resources,
+                    tap_interface=tap_interface,
+                )
+            elif self.hypervisor == HypervisorType.qemu:
+                assert isinstance(self.resources, AlephQemuResources)
+                self.vm = vm = AlephQemuInstance(
+                    vm_id=vm_id,
+                    vm_hash=self.vm_hash,
+                    resources=self.resources,
+                    enable_networking=self.message.environment.internet,
+                    hardware_resources=self.message.resources,
+                    tap_interface=tap_interface,
+                )
+            else:
+                raise Exception("Unknown VM")
         else:
-            assert self.is_instance
-            assert isinstance(self.resources, AlephInstanceResources)
-            self.vm = vm = AlephFirecrackerInstance(
-                vm_id=vm_id,
-                vm_hash=self.vm_hash,
-                resources=self.resources,
-                enable_networking=self.message.environment.internet,
-                hardware_resources=self.message.resources,
-                tap_interface=tap_interface,
-            )
+            raise Exception("Unknown VM")
+
         try:
             await vm.setup()
             await vm.start()
@@ -237,7 +265,7 @@ async def stop(self):
             self.cancel_expiration()
             self.cancel_update()
 
-            if isinstance(self.message, InstanceContent):
+            if self.vm.support_snapshot:
                 await self.snapshot_manager.stop_for(self.vm_hash)
             self.stop_event.set()
 
@@ -291,7 +319,7 @@ async def record_usage(self):
                     io_write_bytes=pid_info["process"]["io_counters"][3],
                     vcpus=self.vm.hardware_resources.vcpus,
                     memory=self.vm.hardware_resources.memory,
-                    network_tap=self.vm.tap_interface.device_name,
+                    network_tap=self.vm.tap_interface.device_name if self.vm.tap_interface else "",
                 )
             )
         else:
diff --git a/src/aleph/vm/orchestrator/README.md b/src/aleph/vm/orchestrator/README.md
index 9b51fe032..05a5c0dd6 100644
--- a/src/aleph/vm/orchestrator/README.md
+++ b/src/aleph/vm/orchestrator/README.md
@@ -87,7 +87,7 @@ is used to parse and validate Aleph messages.
 ```shell
 apt install -y --no-install-recommends --no-install-suggests python3-pip
 pip3 install pydantic[dotenv]
-pip3 install 'aleph-message==0.4.0'
+pip3 install 'aleph-message==0.4.1'
 ```
 
 ### 2.f. Create the jailer working directory:
diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index 4a19d934f..9dbfc93d2 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -253,6 +253,7 @@ async def run_instances(instances: list[ItemHash]) -> None:
 
     await asyncio.gather(*[start_instance(item_hash=instance_id) for instance_id in instances])
     await asyncio.Event().wait()  # wait forever
+    # TODO : should we really wait forever?
 
 
 @contextlib.contextmanager
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index d008dd0f4..9fc0f9bb6 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -62,8 +62,7 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
 
     if execution.vm is None:
         raise web.HTTPBadRequest(body=f"VM {vm_hash} is not running")
-
-    queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
+    queue = None
     try:
         ws = web.WebSocketResponse()
         await ws.prepare(request)
@@ -72,11 +71,7 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
             await authenticate_for_vm_or_403(execution, request, vm_hash, ws)
             await ws.send_json({"status": "connected"})
 
-            # Limit the number of queues per VM
-            if len(execution.vm.fvm.log_queues) > 20:
-                logger.warning("Too many log queues, dropping the oldest one")
-                execution.vm.fvm.log_queues.pop(0)
-            execution.vm.fvm.log_queues.append(queue)
+            queue = await execution.vm.get_log_queue()
 
             while True:
                 log_type, message = await queue.get()
@@ -86,9 +81,8 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
         finally:
             await ws.close()
     finally:
-        if queue in execution.vm.fvm.log_queues:
-            execution.vm.fvm.log_queues.remove(queue)
-        queue.empty()
+        if queue:
+            await execution.vm.unregister_queue(queue)
 
 
 async def authenticate_for_vm_or_403(execution, request, vm_hash, ws):
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 404d36bf2..997f85d34 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -95,8 +95,9 @@ async def create_a_vm(
 
             await execution.create(vm_id=vm_id, tap_interface=tap_interface)
 
+            assert execution.vm
             # Start VM snapshots automatically
-            if isinstance(message, InstanceContent):
+            if execution.vm.support_snapshot:
                 await self.snapshot_manager.start_for(vm=execution.vm)
         except Exception:
             # ensure the VM is removed from the pool on creation error

From a831236bc4c3e3f32b4353e58696f035c4d0a73e Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 8 Dec 2023 12:35:00 +0100
Subject: [PATCH 580/990] Problem: Log system won't work when we switch to
 systemd (#496)

* Qemu fix shutdown message
* Use journald for VM logs
* Make journald work with stream_logs endpoint

Co-authored-by: Hugo Herter 
---
 packaging/aleph-vm/DEBIAN/control             |   2 +-
 pyproject.toml                                |   3 +-
 .../vm/controllers/firecracker/executable.py  |   4 +-
 src/aleph/vm/controllers/qemu/instance.py     | 139 ++++++++++++------
 src/aleph/vm/orchestrator/views/operator.py   |  10 +-
 5 files changed, 102 insertions(+), 56 deletions(-)

diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index ab9e37366..82827420c 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils,python3-systemd
 Section: aleph-im
 Priority: Extra
diff --git a/pyproject.toml b/pyproject.toml
index 3886cc7fc..eca7b8e9b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,7 +42,8 @@ dependencies = [
   "msgpack~=1.0.7",
   "packaging~=23.2",
   "jsonschema==4.19.1",
-  "qmp==0.0.1"
+  "qmp==0.0.1",
+  "systemd-python"
 ]
 
 [project.urls]
diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index fcb8a6d96..4500afe36 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -283,7 +283,7 @@ async def teardown(self):
     async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
         raise NotImplementedError()
 
-    async def get_log_queue(self) -> asyncio.Queue:
+    def get_log_queue(self) -> asyncio.Queue:
         queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
         # Limit the number of queues per VM
 
@@ -293,7 +293,7 @@ async def get_log_queue(self) -> asyncio.Queue:
         self.fvm.log_queues.append(queue)
         return queue
 
-    async def unregister_queue(self, queue: asyncio.Queue):
+    def unregister_queue(self, queue: asyncio.Queue):
         if queue in self.fvm.log_queues:
             self.fvm.log_queues.remove(queue)
         queue.empty()
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index 592a01f44..b1d7af1eb 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -5,7 +5,7 @@
 import sys
 from asyncio import Task
 from asyncio.subprocess import Process
-from typing import Generic, Optional, TypeVar, Union
+from typing import Callable, Dict, Generic, Optional, Tuple, TypedDict, TypeVar, Union
 
 import psutil
 import qmp
@@ -13,6 +13,7 @@
 from aleph_message.models.execution.environment import MachineResources
 from aleph_message.models.execution.instance import RootfsVolume
 from aleph_message.models.execution.volume import PersistentVolume, VolumePersistence
+from systemd import journal
 
 from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.executable import (
@@ -78,6 +79,56 @@ async def make_writable_volume(self, parent_image_path, volume: Union[Persistent
 ConfigurationType = TypeVar("ConfigurationType")
 
 
+class EntryDict(TypedDict):
+    SYSLOG_IDENTIFIER: str
+    MESSAGE: str
+
+
+def make_logs_queue(stdout_identifier, stderr_identifier, skip_past=True) -> Tuple[asyncio.Queue, Callable[[], None]]:
+    """Create a queue which streams the logs for the process.
+
+    @param stdout_identifier: journald identifier for process stdout
+    @param stderr_identifier: journald identifier for process stderr
+    @param skip_past: Skip past history.
+    @return: queue and function to cancel the queue.
+
+    The consumer is required to call the queue cancel function when it's done consuming the queue.
+
+    Works by creating a journald reader, and using `add_reader` to call a callback when
+    data is available for reading.
+    In the callback we check the message type and fill the queue accordingly
+
+    For more information refer to the sd-journal(3) manpage
+    and systemd.journal module documentation.
+    """
+    r = journal.Reader()
+    r.add_match(SYSLOG_IDENTIFIER=stdout_identifier)
+    r.add_match(SYSLOG_IDENTIFIER=stderr_identifier)
+    queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
+
+    def _ready_for_read():
+        change_type = r.process()  # reset fd status
+        if change_type != journal.APPEND:
+            return
+        entry: EntryDict
+        for entry in r:
+            log_type = "stdout" if entry["SYSLOG_IDENTIFIER"] == stdout_identifier else "stderr"
+            msg = entry["MESSAGE"]
+            asyncio.create_task(queue.put((log_type, msg)))
+
+    if skip_past:
+        r.seek_tail()
+
+    loop = asyncio.get_event_loop()
+    loop.add_reader(r.fileno(), _ready_for_read)
+
+    def do_cancel():
+        loop.remove_reader(r.fileno())
+        r.close()
+
+    return queue, do_cancel
+
+
 class AlephQemuInstance(Generic[ConfigurationType], CloudInitMixin, AlephVmControllerInterface):
     vm_id: int
     vm_hash: ItemHash
@@ -91,6 +142,7 @@ class AlephQemuInstance(Generic[ConfigurationType], CloudInitMixin, AlephVmContr
     qemu_process: Optional[Process]
     support_snapshot = False
     qmp_socket_path = None
+    _queue_cancellers: Dict[asyncio.Queue, Callable] = {}
 
     def __repr__(self):
         return f""
@@ -150,6 +202,14 @@ def to_dict(self):
     async def setup(self):
         pass
 
+    @property
+    def _journal_stdout_name(self) -> str:
+        return f"vm-{self.vm_hash}-stdout"
+
+    @property
+    def _journal_stderr_name(self) -> str:
+        return f"vm-{self.vm_hash}-stderr"
+
     async def start(self):
         logger.debug(f"Starting Qemu: {self} ")
         # Based on the command
@@ -209,11 +269,14 @@ async def start(self):
 
         try:
             print(*args)
+
+            journal_stdout = journal.stream(self._journal_stdout_name)
+            journal_stderr = journal.stream(self._journal_stderr_name)
             self.qemu_process = proc = await asyncio.create_subprocess_exec(
                 *args,
                 stdin=asyncio.subprocess.DEVNULL,
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
+                stdout=journal_stdout,
+                stderr=journal_stderr,
             )
 
             logger.debug(f"setup done {self}, {proc}")
@@ -234,7 +297,7 @@ async def handle_termination(proc: Process):
             raise
 
         if self.enable_console:
-            self.process_logs()
+            self.print_logs()
 
         await self.wait_for_init()
         logger.debug(f"started qemu vm {self} on {self.get_ip()}")
@@ -273,16 +336,12 @@ async def start_guest_api(self):
     async def stop_guest_api(self):
         pass
 
-    stdout_task: Optional[Task] = None
-    stderr_task: Optional[Task] = None
+    print_task: Optional[Task] = None
     log_queues: list[asyncio.Queue] = []
 
     async def teardown(self):
-        if self.stdout_task:
-            self.stdout_task.cancel()
-        if self.stderr_task:
-            self.stderr_task.cancel()
-
+        if self.print_task:
+            self.print_task.cancel()
         self._shutdown()
 
         if self.enable_networking:
@@ -291,43 +350,24 @@ async def teardown(self):
                 await self.tap_interface.delete()
         await self.stop_guest_api()
 
-    async def _process_stdout(self):
-        while not self.qemu_process:
-            await asyncio.sleep(0.01)  # Todo: Use signal here
-        while True:
-            line = await self.qemu_process.stdout.readline()
-            if not line:  # FD is closed nothing more will come
-                print(self, "EOF")
-                return
-            for queue in self.log_queues:
-                await queue.put(("stdout", line))
-            print(self, line.decode().strip())
-
-    async def _process_stderr(self):
-        while not self.qemu_process:
-            await asyncio.sleep(0.01)  # Todo: Use signal here
-        while True:
-            line = await self.qemu_process.stderr.readline()
-            if not line:  # FD is closed nothing more will come
-                print(self, "EOF")
-                return
-            for queue in self.log_queues:
-                await queue.put(("stderr", line))
-            print(self, line.decode().strip(), file=sys.stderr)
+    def print_logs(self) -> None:
+        """Print logs to our output for debugging"""
+        queue = self.get_log_queue()
 
-    def process_logs(self) -> tuple[Task, Task]:
-        """Start two tasks to process the stdout and stderr
-
-        It will stream their content to queues registered on self.log_queues
-        It will also print them"""
+        async def print_logs():
+            try:
+                while True:
+                    log_type, message = await queue.get()
+                    fd = sys.stderr if log_type == "stderr" else sys.stdout
+                    print(self, message, file=fd)
+            finally:
+                self.unregister_queue(queue)
 
         loop = asyncio.get_running_loop()
-        self.stdout_task = loop.create_task(self._process_stdout())
-        self.stderr_task = loop.create_task(self._process_stderr())
-        return self.stdout_task, self.stderr_task
+        self.print_task = loop.create_task(print_logs(), name=f"{self}-print-logs")
 
     def _get_qmpclient(self) -> Optional[qmp.QEMUMonitorProtocol]:
-        if not self.qmp_socket_path:
+        if not (self.qmp_socket_path and self.qmp_socket_path.exists()):
             return None
         client = qmp.QEMUMonitorProtocol(str(self.qmp_socket_path))
         client.connect()
@@ -340,18 +380,21 @@ def _shutdown(self):
             if not resp == {}:
                 logger.warning("unexpected answer from VM", resp)
             client.close()
-            self.qmp_socket_path = None
+        self.qmp_socket_path = None
 
-    async def get_log_queue(self) -> asyncio.Queue:
-        queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
+    def get_log_queue(self) -> asyncio.Queue:
+        queue, canceller = make_logs_queue(self._journal_stdout_name, self._journal_stderr_name)
+        self._queue_cancellers[queue] = canceller
         # Limit the number of queues per VM
         if len(self.log_queues) > 20:
             logger.warning("Too many log queues, dropping the oldest one")
-            self.log_queues.pop(0)
+            self.unregister_queue(self.log_queues[1])
         self.log_queues.append(queue)
         return queue
 
-    async def unregister_queue(self, queue: asyncio.Queue):
+    def unregister_queue(self, queue: asyncio.Queue) -> None:
         if queue in self.log_queues:
+            self._queue_cancellers[queue]()
+            del self._queue_cancellers[queue]
             self.log_queues.remove(queue)
         queue.empty()
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 9fc0f9bb6..311a9b701 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -66,23 +66,25 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
     try:
         ws = web.WebSocketResponse()
         await ws.prepare(request)
-
         try:
             await authenticate_for_vm_or_403(execution, request, vm_hash, ws)
             await ws.send_json({"status": "connected"})
 
-            queue = await execution.vm.get_log_queue()
+            queue = execution.vm.get_log_queue()
 
             while True:
                 log_type, message = await queue.get()
                 assert log_type in ("stdout", "stderr")
 
-                await ws.send_json({"type": log_type, "message": message.decode()})
+                await ws.send_json({"type": log_type, "message": message})
+
         finally:
             await ws.close()
+            logger.info(f"connection  {ws} closed")
+
     finally:
         if queue:
-            await execution.vm.unregister_queue(queue)
+            execution.vm.unregister_queue(queue)
 
 
 async def authenticate_for_vm_or_403(execution, request, vm_hash, ws):

From 03186ab1d8d38807bdf33e5d7a98d7bb4f529d2d Mon Sep 17 00:00:00 2001
From: Reza Rahemtola 
Date: Mon, 11 Dec 2023 03:46:30 +0100
Subject: [PATCH 581/990] fix(docs): Link to Caddy doc and version bump

---
 doc/INSTALL-Debian-11.md    | 4 ++--
 doc/INSTALL-Debian-12.md    | 4 ++--
 doc/INSTALL-Ubuntu-22.04.md | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index a8bf92bfc..577fe4ac0 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.0/aleph-vm.debian-11.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.1/aleph-vm.debian-11.deb
 apt install /opt/aleph-vm.debian-11.deb
 ```
 
@@ -99,7 +99,7 @@ HTTPS/TLS certificates on time.
 
 First, create a domain name that points to the server on IPv4 (A) and IPv6 (AAAA).
 
-This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CONFIGURE_CADDY.md).
+This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](/CONFIGURE_CADDY.md).
 
 Again, run these commands as `root`:
 ```shell
diff --git a/doc/INSTALL-Debian-12.md b/doc/INSTALL-Debian-12.md
index 0740a1196..e3826b327 100644
--- a/doc/INSTALL-Debian-12.md
+++ b/doc/INSTALL-Debian-12.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.0/aleph-vm.debian-12.deb
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.1/aleph-vm.debian-12.deb
 apt install /opt/aleph-vm.debian-12.deb
 ```
 
@@ -100,7 +100,7 @@ HTTPS/TLS certificates on time.
 
 First, create a domain name that points to the server on IPv4 (A) and IPv6 (AAAA).
 
-This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CONFIGURE_CADDY.md).
+This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](/CONFIGURE_CADDY.md).
 
 Again, run these commands as `root`:
 ```shell
diff --git a/doc/INSTALL-Ubuntu-22.04.md b/doc/INSTALL-Ubuntu-22.04.md
index ead2b68be..b4fdfeb0d 100644
--- a/doc/INSTALL-Ubuntu-22.04.md
+++ b/doc/INSTALL-Ubuntu-22.04.md
@@ -37,7 +37,7 @@ docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector al
 Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
 The procedure is similar for updates.
 ```shell
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.0/aleph-vm.ubuntu-22.04.deb
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.1/aleph-vm.ubuntu-22.04.deb
 sudo apt install /opt/aleph-vm.ubuntu-22.04.deb
 ```
 
@@ -101,7 +101,7 @@ HTTPS/TLS certificates on time.
 
 First, create a domain name that points to the server on IPv4 (A) and IPv6 (AAAA).
 
-This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](CONFIGURE_CADDY.md).
+This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](/CONFIGURE_CADDY.md).
 
 Again, run these commands as `root`:
 ```shell

From 4313e89539c5040910334ebf2b4a532a57f2f760 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 11 Dec 2023 15:03:30 +0100
Subject: [PATCH 582/990] Fix: CI frequently failed due to new versions (#501)

* Fix: CI frequently failed due to new versions

Solution: Fix the version of all dependencies used in the project.

* fixup! Fix: CI frequently failed due to new versions
---
 pyproject.toml | 48 ++++++++++++++++++++++++------------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index eca7b8e9b..9c239312b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,23 +24,23 @@ classifiers = [
   "Topic :: System :: Distributed Computing",
 ]
 dependencies = [
-  "pydantic[dotenv]~=1.10.13",
-  "aiohttp~=3.8.6",
-  "aiodns~=3.1.0",
-  "alembic~=1.7.6",
-  "setproctitle~=1.3.3",
-  "pyyaml~=6.0.1",
-  "aleph-message~=0.4.1",
-  "jwskate~=0.8.0",
-  "eth-account~=0.9.0",
-  "sentry-sdk~=1.31.0",
-  "aioredis~=1.3.1",
-  "psutil~=5.9.5",
-  "py-cpuinfo~=9.0.0",
-  "schedule~=1.2.1",
+  "pydantic[dotenv]==1.10.13",
+  "aiohttp==3.8.6",
+  "aiodns==3.1.0",
+  "alembic==1.7.6",
+  "setproctitle==1.3.3",
+  "pyyaml==6.0.1",
+  "aleph-message==0.4.1",
+  "jwskate==0.8.0",
+  "eth-account==0.9.0",
+  "sentry-sdk==1.31.0",
+  "aioredis==1.3.1",
+  "psutil==5.9.5",
+  "py-cpuinfo==9.0.0",
+  "schedule==1.2.1",
   "nftables @ git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py",
-  "msgpack~=1.0.7",
-  "packaging~=23.2",
+  "msgpack==1.0.7",
+  "packaging==23.2",
   "jsonschema==4.19.1",
   "qmp==0.0.1",
   "systemd-python"
@@ -77,10 +77,10 @@ check = "aleph-vm controller run {args:--help}"
 
 [tool.hatch.envs.testing]
 dependencies = [
-  "coverage[toml]~=7.3.2",
-  "pytest~=7.4.2",
-  "pytest-mock~=3.11.1",
-  "pytest-asyncio~=0.21.1 ",
+  "coverage[toml]==7.3.2",
+  "pytest==7.4.2",
+  "pytest-mock==3.11.1",
+  "pytest-asyncio==0.21.1 ",
 ]
 [tool.hatch.envs.testing.scripts]
 test = "pytest {args:tests}"
@@ -100,10 +100,10 @@ python = ["3.9", "3.10", "3.11", "3.12"]
 [tool.hatch.envs.lint]
 detached = true
 dependencies = [
-  "black>=23.9.0",
-  "mypy>=1.6.0",
-  "ruff>=0.0.292",
-  "isort>=5.12.0",
+  "black==23.9.0",
+  "mypy==1.6.0",
+  "ruff==0.0.292",
+  "isort==5.12.0",
 ]
 [tool.hatch.envs.lint.scripts]
 typing = "mypy --install-types --non-interactive --ignore-missing-imports --explicit-package-bases {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}"

From 7fea7c6cb51da66ae42b697b5d3aa8d676504fd0 Mon Sep 17 00:00:00 2001
From: Bonjour Internet 
Date: Mon, 11 Dec 2023 14:58:37 +0100
Subject: [PATCH 583/990] fix: "typeError  (...) cannot read measured_at"

---
 src/aleph/vm/orchestrator/views/static/helpers.js | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/static/helpers.js b/src/aleph/vm/orchestrator/views/static/helpers.js
index aced57b66..8335e2faf 100644
--- a/src/aleph/vm/orchestrator/views/static/helpers.js
+++ b/src/aleph/vm/orchestrator/views/static/helpers.js
@@ -35,9 +35,12 @@ const isLatestRelease = async () => {
 
 const buildMetricViewset = (metricsMsg, hostname, metricsResult) => {
     const thisNode = metricsMsg.content.metrics.crn.find(node => node.url === hostname)
-    const factory = keyName => ({ time: thisNode.measured_at, value: thisNode[keyName] * 100 })
-    const keys = ['base_latency', 'base_latency_ipv4', 'diagnostic_vm_latency', 'full_check_latency']
-    keys.map(key => metricsResult[key].push(factory(key)))
+    // Fixes a bug if a node has no metrics for the given timeframe
+    if(thisNode){
+        const factory = keyName => ({ time: thisNode.measured_at, value: thisNode[keyName] * 100 })
+        const keys = ['base_latency', 'base_latency_ipv4', 'diagnostic_vm_latency', 'full_check_latency']
+        keys.map(key => metricsResult[key].push(factory(key)))
+    }
 }
 
 async function* fetchLatestMetrics (hostname, fromDate) {

From 11d1a4fd006a930d11eb2147792a22b507f478fb Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 11 Dec 2023 11:48:21 +0100
Subject: [PATCH 584/990] Fix: Kubo IPFS used default config

This disables features that are not expected on
aleph.im Compute Resource Nodes and limits the number of CIDs published on the P2P network.
---
 packaging/aleph-vm/etc/ipfs/KUBO.md              |  4 ++++
 packaging/aleph-vm/etc/ipfs/kubo.json            | 16 ++++++++++++++++
 .../aleph-vm/etc/systemd/system/ipfs.service     |  4 ++--
 3 files changed, 22 insertions(+), 2 deletions(-)
 create mode 100644 packaging/aleph-vm/etc/ipfs/KUBO.md
 create mode 100644 packaging/aleph-vm/etc/ipfs/kubo.json

diff --git a/packaging/aleph-vm/etc/ipfs/KUBO.md b/packaging/aleph-vm/etc/ipfs/KUBO.md
new file mode 100644
index 000000000..a21a9fa4f
--- /dev/null
+++ b/packaging/aleph-vm/etc/ipfs/KUBO.md
@@ -0,0 +1,4 @@
+The IP range `86.84.0.0/16` is managed by `KPN Internet`  is filtered out due to 
+an abuse letter sent to a node operator by Hetzner regarding "an attack" from the node.
+The content of this "attack" appears as legit IPFS traffic 
+(TCP packets from port 4001 to port 4001 and UDP packets from port 4001 to port 46024).
diff --git a/packaging/aleph-vm/etc/ipfs/kubo.json b/packaging/aleph-vm/etc/ipfs/kubo.json
new file mode 100644
index 000000000..56db34c10
--- /dev/null
+++ b/packaging/aleph-vm/etc/ipfs/kubo.json
@@ -0,0 +1,16 @@
+{
+  "AutoNAT": {
+    "ServiceMode": "disabled"
+  },
+  "AddrFilters": [
+  ],
+  "Reprovider": {
+    "Strategy": "roots"
+  },
+  "Swarm": {
+    "EnableHolePunching":false,
+    "RelayService": {
+      "Enabled": false
+    }
+  }
+}
diff --git a/packaging/aleph-vm/etc/systemd/system/ipfs.service b/packaging/aleph-vm/etc/systemd/system/ipfs.service
index 6f4a8c359..66a753036 100644
--- a/packaging/aleph-vm/etc/systemd/system/ipfs.service
+++ b/packaging/aleph-vm/etc/systemd/system/ipfs.service
@@ -25,7 +25,7 @@ After=network.target
 
 [Service]
 # hardening
-ReadOnlyPaths="/opt/kubo/"
+ReadOnlyPaths="/opt/kubo/" "/etc/ipfs"
 ReadWritePaths="/var/lib/ipfs/"
 NoNewPrivileges=true
 ProtectSystem=strict
@@ -72,7 +72,7 @@ Type=notify
 User=ipfs
 Group=ipfs
 Environment=IPFS_PATH="/var/lib/ipfs"
-ExecStart=/opt/kubo/ipfs daemon --init --migrate --init-profile=server
+ExecStart=/opt/kubo/ipfs daemon --init --migrate --init-profile=server --config-file /etc/ipfs/kubo.json
 Restart=on-failure
 KillSignal=SIGINT
 

From 2c1ed46927208ca79332f8609e5a2e62ad6274dc Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 12 Dec 2023 16:42:37 +0100
Subject: [PATCH 585/990] Fix: Package uninstall failed if this directory was
 absent

---
 packaging/aleph-vm/DEBIAN/postrm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/aleph-vm/DEBIAN/postrm b/packaging/aleph-vm/DEBIAN/postrm
index a2bdda35d..41f9b7ffe 100755
--- a/packaging/aleph-vm/DEBIAN/postrm
+++ b/packaging/aleph-vm/DEBIAN/postrm
@@ -3,7 +3,7 @@ set -euf -o pipefail
 
 rm -fr /srv/jailer  # Upgrade from < 0.1.11
 rm -fr /tmp/aleph/  # Upgrade from < 0.1.11
-rm -r /var/lib/aleph/vm/jailer
+rm -fr /var/lib/aleph/vm/jailer
 
 if [ "$1" = "purge" ]; then
     # Remove the directory when the package is purged

From f9261942f351c81fc8375136411fbd1af36fc60c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 12 Dec 2023 16:58:39 +0100
Subject: [PATCH 586/990] Fix: Operator could not configure Sentry sampling
 rate

---
 src/aleph/vm/conf.py               | 1 +
 src/aleph/vm/guest_api/__main__.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 581eb8f2e..9609de5ce 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -246,6 +246,7 @@ class Settings(BaseSettings):
     # Developer options
 
     SENTRY_DSN: Optional[str] = None
+    SENTRY_TRACES_SAMPLE_RATE: float = Field(ge=0, le=1.0, default=0.1)
     DEVELOPER_SSH_KEYS: Optional[list[str]] = []
     # Using an object here forces the value to come from Python code and not from an environment variable.
     USE_DEVELOPER_SSH_KEYS: Union[Literal[False], object] = False
diff --git a/src/aleph/vm/guest_api/__main__.py b/src/aleph/vm/guest_api/__main__.py
index a15ffafd7..72e2cd26e 100644
--- a/src/aleph/vm/guest_api/__main__.py
+++ b/src/aleph/vm/guest_api/__main__.py
@@ -8,6 +8,7 @@
 from aiohttp import web
 from setproctitle import setproctitle
 
+from aleph.vm.conf import settings
 from aleph.vm.version import get_version_from_apt, get_version_from_git
 
 try:
@@ -168,7 +169,7 @@ def run_guest_api(
             # Set traces_sample_rate to 1.0 to capture 100%
             # of transactions for performance monitoring.
             # We recommend adjusting this value in production.
-            traces_sample_rate=1.0,
+            traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE,
         )
         sentry_sdk.set_context(
             "version",

From a3ad7d10f067db6a0bf32e60ee475796a0482b35 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Thu, 14 Dec 2023 10:25:04 +0100
Subject: [PATCH 587/990] Fix: The instance VMs should be launched
 independently of the orchestrator.

Problem: The instance VMs should be launched independently of the orchestrator.

Solution: Launch VMs using systemd manager.

Co-authored-by: Andres D. Molins 
Co-authored-by: Olivier Le Thanh Duong 
Co-authored-by: Hugo Herter 
---
 packaging/aleph-vm/DEBIAN/control             |   2 +-
 .../system/aleph-vm-controller@.service       |  16 ++
 pyproject.toml                                |   3 +-
 src/aleph/vm/conf.py                          |   3 +
 src/aleph/vm/controllers/__main__.py          |  62 +++----
 src/aleph/vm/controllers/configuration.py     |  47 +++++
 .../vm/controllers/firecracker/executable.py  |  37 +++-
 .../vm/controllers/firecracker/instance.py    |  12 +-
 .../vm/controllers/firecracker/program.py     |   7 +-
 src/aleph/vm/controllers/interface.py         |   4 +
 src/aleph/vm/controllers/qemu/instance.py     | 164 ++++++------------
 src/aleph/vm/hypervisors/qemu/__init__.py     |   0
 src/aleph/vm/hypervisors/qemu/qemuvm.py       | 147 ++++++++++++++++
 src/aleph/vm/models.py                        |  17 +-
 src/aleph/vm/orchestrator/run.py              |   6 +-
 src/aleph/vm/orchestrator/views/__init__.py   |   3 +-
 src/aleph/vm/pool.py                          |  57 ++++--
 src/aleph/vm/systemd.py                       |  74 ++++++++
 18 files changed, 489 insertions(+), 172 deletions(-)
 create mode 100644 packaging/aleph-vm/etc/systemd/system/aleph-vm-controller@.service
 create mode 100644 src/aleph/vm/controllers/configuration.py
 create mode 100644 src/aleph/vm/hypervisors/qemu/__init__.py
 create mode 100644 src/aleph/vm/hypervisors/qemu/qemuvm.py
 create mode 100644 src/aleph/vm/systemd.py

diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index 82827420c..b91a89fa3 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils,python3-systemd
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils,python3-systemd,python3-dbus,btrfs-progs
 Section: aleph-im
 Priority: Extra
diff --git a/packaging/aleph-vm/etc/systemd/system/aleph-vm-controller@.service b/packaging/aleph-vm/etc/systemd/system/aleph-vm-controller@.service
new file mode 100644
index 000000000..c817aad1e
--- /dev/null
+++ b/packaging/aleph-vm/etc/systemd/system/aleph-vm-controller@.service
@@ -0,0 +1,16 @@
+[Unit]
+Description=Aleph VM %i Controller
+After=network.target
+
+[Service]
+Type=simple
+RestartSec=5s
+PrivateTmp=yes
+NoNewPrivileges=true
+WorkingDirectory=/opt/aleph-vm
+Environment=PYTHONPATH=/opt/aleph-vm/:$PYTHONPATH
+ExecStart=/usr/bin/python3 -m aleph.vm.controllers --config=/var/lib/aleph/vm/%i-controller.json
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
diff --git a/pyproject.toml b/pyproject.toml
index 9c239312b..ca9748828 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -43,7 +43,8 @@ dependencies = [
   "packaging==23.2",
   "jsonschema==4.19.1",
   "qmp==0.0.1",
-  "systemd-python"
+  "dbus-python==1.3.2",
+  "systemd-python==235",
 ]
 
 [project.urls]
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 9609de5ce..ec6a2c23d 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -184,6 +184,7 @@ class Settings(BaseSettings):
     DATA_CACHE: Path = Field(None, description="Default to CACHE_ROOT/data")
 
     EXECUTION_ROOT = Path("/var/lib/aleph/vm")
+    JAILER_BASE_DIRECTORY: Path = Field(None, description="Default to EXECUTION_ROOT/jailer")
     EXECUTION_DATABASE: Path = Field(
         None, description="Location of database file. Default to EXECUTION_ROOT/executions.sqlite3"
     )
@@ -359,6 +360,8 @@ def __init__(
             self.RUNTIME_CACHE = self.CACHE_ROOT / "runtime"
         if not self.DATA_CACHE:
             self.DATA_CACHE = self.CACHE_ROOT / "data"
+        if not self.JAILER_BASE_DIRECTORY:
+            self.JAILER_BASE_DIRECTORY = self.EXECUTION_ROOT / "jailer"
         if not self.PERSISTENT_VOLUMES_DIR:
             self.PERSISTENT_VOLUMES_DIR = self.EXECUTION_ROOT / "volumes" / "persistent"
         if not self.EXECUTION_LOG_DIRECTORY:
diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index d3071bcd3..8f2640012 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -2,11 +2,11 @@
 import asyncio
 import json
 import logging
+import signal
 import sys
 from pathlib import Path
 
-from pydantic import BaseModel
-
+from aleph.vm.hypervisors.qemu.qemuvm import QemuVM
 from aleph.vm.network.hostnetwork import Network, make_ipv6_allocator
 
 try:
@@ -14,24 +14,16 @@
 except ImportError:
     sentry_sdk = None
 
-from aleph.vm.conf import Settings, settings
 from aleph.vm.hypervisors.firecracker.microvm import MicroVM
 
-logger = logging.getLogger(__name__)
-
-
-class VMConfiguration(BaseModel):
-    use_jailer: bool
-    firecracker_bin_path: Path
-    jailer_bin_path: Path
-    config_file_path: Path
-    init_timeout: float
-
+from .configuration import (
+    Configuration,
+    HypervisorType,
+    QemuVMConfiguration,
+    VMConfiguration,
+)
 
-class Configuration(BaseModel):
-    vm_id: int
-    settings: Settings
-    vm_configuration: VMConfiguration
+logger = logging.getLogger(__name__)
 
 
 def configuration_from_file(path: Path):
@@ -68,22 +60,34 @@ def parse_args(args):
     return parser.parse_args(args)
 
 
-async def run_instance(config: Configuration):
-    execution = MicroVM(
-        vm_id=config.vm_id,
-        firecracker_bin_path=config.vm_configuration.firecracker_bin_path,
-        jailer_base_directory=config.settings.JAILER_BASE_DIR,
-        use_jailer=config.vm_configuration.use_jailer,
-        jailer_bin_path=config.vm_configuration.jailer_bin_path,
-        init_timeout=config.vm_configuration.init_timeout,
-    )
+async def run_persistent_vm(config: Configuration):
+    if config.hypervisor == HypervisorType.firecracker:
+        assert isinstance(config.vm_configuration, VMConfiguration)
+        execution = MicroVM(
+            vm_id=config.vm_id,
+            firecracker_bin_path=config.vm_configuration.firecracker_bin_path,
+            jailer_base_directory=config.settings.JAILER_BASE_DIR,
+            use_jailer=config.vm_configuration.use_jailer,
+            jailer_bin_path=config.vm_configuration.jailer_bin_path,
+            init_timeout=config.vm_configuration.init_timeout,
+        )
+
+        execution.prepare_start()
+        process = await execution.start(config.vm_configuration.config_file_path)
+    else:
+        assert isinstance(config.vm_configuration, QemuVMConfiguration)
+        execution = QemuVM(config.vm_configuration)
+        process = await execution.start()
+
+        # Catch the terminating signal and send a proper message to the vm to stop it so it close files properly
+        loop = asyncio.get_event_loop()
+        loop.add_signal_handler(signal.SIGTERM, execution.send_shutdown_message)
 
-    execution.prepare_start()
-    process = await execution.start(config.vm_configuration.config_file_path)
     if config.settings.PRINT_SYSTEM_LOGS:
         execution.start_printing_logs()
 
     await process.wait()
+    logger.info(f"Process terminated with {process.returncode}")
 
     return execution
 
@@ -125,7 +129,7 @@ def main():
 
         network.setup()
 
-    asyncio.run(run_instance(config))
+    asyncio.run(run_persistent_vm(config))
 
 
 if __name__ == "__main__":
diff --git a/src/aleph/vm/controllers/configuration.py b/src/aleph/vm/controllers/configuration.py
new file mode 100644
index 000000000..2255089bc
--- /dev/null
+++ b/src/aleph/vm/controllers/configuration.py
@@ -0,0 +1,47 @@
+from enum import Enum
+from pathlib import Path
+from typing import Optional, Union
+
+from pydantic import BaseModel
+
+from aleph.vm.conf import Settings, settings
+
+
+class VMConfiguration(BaseModel):
+    use_jailer: bool
+    firecracker_bin_path: Path
+    jailer_bin_path: Path
+    config_file_path: Path
+    init_timeout: float
+
+
+class QemuVMConfiguration(BaseModel):
+    qemu_bin_path: str
+    cloud_init_drive_path: Optional[str]
+    image_path: str
+    monitor_socket_path: Path
+    qmp_socket_path: Path
+    vcpu_count: int
+    mem_size_mb: int
+    interface_name: Optional[str]
+
+
+class HypervisorType(str, Enum):
+    qemu = "qemu"
+    firecracker = "firecracker"
+
+
+class Configuration(BaseModel):
+    vm_id: int
+    settings: Settings
+    vm_configuration: Union[QemuVMConfiguration, VMConfiguration]
+    hypervisor: HypervisorType = HypervisorType.firecracker
+
+
+def save_controller_configuration(vm_hash: str, configuration: Configuration) -> Path:
+    """Save VM configuration to be used by the controller service"""
+    config_file_path = Path(f"{settings.EXECUTION_ROOT}/{vm_hash}-controller.json")
+    with config_file_path.open("w") as controller_config_file:
+        controller_config_file.write(configuration.json(by_alias=True, exclude_none=True, indent=4))
+    config_file_path.chmod(0o644)
+    return config_file_path
diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index 4500afe36..a63db3970 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -15,6 +15,11 @@
 from aleph_message.models.execution.environment import MachineResources
 
 from aleph.vm.conf import settings
+from aleph.vm.controllers.configuration import (
+    Configuration,
+    VMConfiguration,
+    save_controller_configuration,
+)
 from aleph.vm.controllers.firecracker.snapshots import CompressedDiskVolumeSnapshot
 from aleph.vm.controllers.interface import AlephVmControllerInterface
 from aleph.vm.guest_api.__main__ import run_guest_api
@@ -149,7 +154,9 @@ class AlephFirecrackerExecutable(Generic[ConfigurationType], AlephVmControllerIn
     vm_configuration: Optional[ConfigurationType]
     guest_api_process: Optional[Process] = None
     is_instance: bool
+    persistent: bool
     _firecracker_config: Optional[FirecrackerConfig] = None
+    controller_configuration: Optional[Configuration] = None
     support_snapshot: bool
 
     def __init__(
@@ -161,6 +168,7 @@ def __init__(
         enable_console: Optional[bool] = None,
         hardware_resources: Optional[MachineResources] = None,
         tap_interface: Optional[TapInterface] = None,
+        persistent: bool = False,
     ):
         self.vm_id = vm_id
         self.vm_hash = vm_hash
@@ -171,6 +179,7 @@ def __init__(
         self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING
         self.hardware_resources = hardware_resources or MachineResources()
         self.tap_interface = tap_interface
+        self.persistent = persistent
 
         self.fvm = MicroVM(
             vm_id=self.vm_id,
@@ -227,6 +236,10 @@ async def start(self):
             msg = "No VM found. Call setup() before start()"
             raise ValueError(msg)
 
+        if self.is_instance or self.persistent:
+            msg = "VM should be started using SystemD Manager class"
+            raise ValueError(msg)
+
         try:
             firecracker_config_path = await self.fvm.save_configuration_file(self._firecracker_config)
             await self.fvm.start(firecracker_config_path)
@@ -245,6 +258,7 @@ async def start(self):
 
         await self.wait_for_init()
         logger.debug(f"started fvm {self.vm_id}")
+        await self.load_configuration()
 
     async def wait_for_init(self) -> None:
         """Wait for the init process of the virtual machine to be ready.
@@ -252,7 +266,28 @@ async def wait_for_init(self) -> None:
         return
 
     async def configure(self):
-        raise NotImplementedError()
+        """Configure the VM by saving controller service configuration"""
+        if self.persistent:
+            firecracker_config_path = await self.fvm.save_configuration_file(self._firecracker_config)
+            vm_configuration = VMConfiguration(
+                firecracker_bin_path=self.fvm.firecracker_bin_path,
+                use_jailer=self.fvm.use_jailer,
+                jailer_bin_path=self.fvm.jailer_bin_path,
+                init_timeout=self.fvm.init_timeout,
+                config_file_path=firecracker_config_path,
+            )
+
+            configuration = Configuration(
+                vm_id=self.vm_id,
+                settings=settings,
+                vm_configuration=vm_configuration,
+            )
+
+            save_controller_configuration(self.vm_hash, configuration)
+
+    async def load_configuration(self):
+        """Load configuration settings for programs."""
+        return
 
     async def start_guest_api(self):
         logger.debug(f"starting guest API for {self.vm_id}")
diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index fd44325f9..a72a82d9e 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -79,6 +79,7 @@ def __init__(
             enable_console,
             hardware_resources or MachineResources(),
             tap_interface,
+            persistent=True,
         )
 
     async def setup(self):
@@ -139,11 +140,6 @@ async def wait_for_init(self) -> None:
                 else:
                     raise
 
-    async def configure(self):
-        """Configure the VM by sending configuration info to it's init"""
-        # Configuration of instances is sent during `self.setup()` by passing it via a volume.
-        pass
-
     async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
         """Create a VM snapshot"""
         volume_path = await create_volume_file(self.resources.message_content.rootfs, self.resources.namespace)
@@ -168,7 +164,11 @@ def _get_hostname(self) -> str:
     def _encode_user_data(self) -> bytes:
         """Creates user data configuration file for cloud-init tool"""
 
-        ssh_authorized_keys = self.resources.message_content.authorized_keys or []
+        ssh_authorized_keys: list[str] | None
+        if settings.USE_DEVELOPER_SSH_KEYS:
+            ssh_authorized_keys = settings.DEVELOPER_SSH_KEYS or []
+        else:
+            ssh_authorized_keys = self.resources.message_content.authorized_keys or []
 
         config: dict[str, Union[str, bool, list[str]]] = {
             "hostname": self._get_hostname(),
diff --git a/src/aleph/vm/controllers/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
index 0cacf4123..4035666d6 100644
--- a/src/aleph/vm/controllers/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -269,6 +269,7 @@ def __init__(
         enable_console: bool | None = None,
         hardware_resources: MachineResources = MachineResources(),
         tap_interface: TapInterface | None = None,
+        persistent: bool = False,
     ):
         super().__init__(
             vm_id,
@@ -278,6 +279,7 @@ def __init__(
             enable_console,
             hardware_resources,
             tap_interface,
+            persistent,
         )
 
     async def setup(self):
@@ -320,9 +322,7 @@ async def wait_for_init(self) -> None:
         """Wait for the custom init inside the virtual machine to signal it is ready."""
         await self.fvm.wait_for_init()
 
-    async def configure(self) -> None:
-        """Configure the VM by sending configuration info to it's init"""
-
+    async def load_configuration(self):
         code: bytes | None
         volumes: list[Volume]
 
@@ -341,6 +341,7 @@ async def _setup_configuration(
     ):
         """Set up the VM configuration. The program mode uses a VSOCK connection to the custom init of the virtual
         machine to send this configuration. Other modes may use Cloud-init, ..."""
+        logger.debug("Sending configuration")
         reader, writer = await asyncio.open_unix_connection(path=self.fvm.vsock_path)
 
         ip = self.get_ip()
diff --git a/src/aleph/vm/controllers/interface.py b/src/aleph/vm/controllers/interface.py
index 32caf84a0..915fda2aa 100644
--- a/src/aleph/vm/controllers/interface.py
+++ b/src/aleph/vm/controllers/interface.py
@@ -71,6 +71,10 @@ async def configure(self) -> None:
         """Configuration done after the VM process is started"""
         raise NotImplementedError()
 
+    async def load_configuration(self) -> None:
+        """Load configuration just after the VM process is started"""
+        raise NotImplementedError()
+
     async def start_guest_api(self):
         raise NotImplementedError()
 
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index b1d7af1eb..bb72ea760 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -5,6 +5,7 @@
 import sys
 from asyncio import Task
 from asyncio.subprocess import Process
+from pathlib import Path
 from typing import Callable, Dict, Generic, Optional, Tuple, TypedDict, TypeVar, Union
 
 import psutil
@@ -16,6 +17,12 @@
 from systemd import journal
 
 from aleph.vm.conf import settings
+from aleph.vm.controllers.configuration import (
+    Configuration,
+    HypervisorType,
+    QemuVMConfiguration,
+    save_controller_configuration,
+)
 from aleph.vm.controllers.firecracker.executable import (
     AlephFirecrackerResources,
     VmSetupError,
@@ -142,7 +149,9 @@ class AlephQemuInstance(Generic[ConfigurationType], CloudInitMixin, AlephVmContr
     qemu_process: Optional[Process]
     support_snapshot = False
     qmp_socket_path = None
+    persistent = True
     _queue_cancellers: Dict[asyncio.Queue, Callable] = {}
+    controller_configuration: Configuration
 
     def __repr__(self):
         return f""
@@ -170,6 +179,7 @@ def __init__(
         self.hardware_resources = hardware_resources
         self.tap_interface = tap_interface
 
+    # TODO : wait for andress soltion for pid handling
     def to_dict(self):
         """Dict representation of the virtual machine. Used to record resource usage and for JSON serialization."""
         if self.qemu_process and psutil:
@@ -202,105 +212,59 @@ def to_dict(self):
     async def setup(self):
         pass
 
-    @property
-    def _journal_stdout_name(self) -> str:
-        return f"vm-{self.vm_hash}-stdout"
-
-    @property
-    def _journal_stderr_name(self) -> str:
-        return f"vm-{self.vm_hash}-stderr"
+    async def configure(self):
+        """Configure the VM by saving controller service configuration"""
 
-    async def start(self):
-        logger.debug(f"Starting Qemu: {self} ")
-        # Based on the command
-        #  qemu-system-x86_64 -enable-kvm -m 2048 -net nic,model=virtio
-        # -net tap,ifname=tap0,script=no,downscript=no -drive file=alpine.qcow2,media=disk,if=virtio -nographic
+        logger.debug(f"Making  Qemu configuration: {self} ")
+        monitor_socket_path = settings.EXECUTION_ROOT / (str(self.vm_id) + "-monitor.socket")
+        self.qmp_socket_path = qmp_socket_path = settings.EXECUTION_ROOT / (str(self.vm_id) + "-qmp.socket")
+        cloud_init_drive = await self._create_cloud_init_drive()
 
-        qemu_path = shutil.which("qemu-system-x86_64")
-        image_path = self.resources.rootfs_path
+        image_path = str(self.resources.rootfs_path)
         vcpu_count = self.hardware_resources.vcpus
         mem_size_mib = self.hardware_resources.memory
-        mem_size_mb = int(mem_size_mib / 1024 / 1024 * 1000 * 1000)
-        # hardware_resources.published ports -> not implemented at the moment
-        # hardware_resources.seconds -> only for microvm
+        mem_size_mb = str(int(mem_size_mib / 1024 / 1024 * 1000 * 1000))
 
-        monitor_socket_path = settings.EXECUTION_ROOT / (str(self.vm_id) + "-monitor.socket")
-        self.qmp_socket_path = qmp_socket_path = settings.EXECUTION_ROOT / (str(self.vm_id) + "-qmp.socket")
-
-        args = [
-            qemu_path,
-            "-enable-kvm",
-            "-nodefaults",
-            "-m",
-            str(mem_size_mb),
-            "-smp",
-            str(vcpu_count),
-            # Disable floppy
-            "-fda",
-            "",
-            # "-snapshot",  # Do not save anything to disk
-            "-drive",
-            f"file={image_path},media=disk,if=virtio",
-            # To debug you can pass gtk or curses instead
-            "-display",
-            "none",
-            "--no-reboot",  # Rebooting from inside the VM shuts down the machine
-            # Listen for commands on this socket
-            "-monitor",
-            f"unix:{monitor_socket_path},server,nowait",
-            # Listen for commands on this socket (QMP protocol in json). Supervisor use it to send shutdown or start
-            # command
-            "-qmp",
-            f"unix:{qmp_socket_path},server,nowait",
-            # Tell to put the output to std fd, so we can include them in the log
-            "-serial",
-            "stdio",
-            # Uncomment for debug
-            # "-serial", "telnet:localhost:4321,server,nowait",
-        ]
+        qemu_bin_path = shutil.which("qemu-system-x86_64")
+        interface_name = None
         if self.tap_interface:
             interface_name = self.tap_interface.device_name
-            # script=no, downscript=no tell qemu not to try to set up the network itself
-            args += ["-net", "nic,model=virtio", "-net", f"tap,ifname={interface_name},script=no,downscript=no"]
+        cloud_init_drive_path = str(cloud_init_drive.path_on_host) if cloud_init_drive else None
+        vm_configuration = QemuVMConfiguration(
+            qemu_bin_path=qemu_bin_path,
+            cloud_init_drive_path=cloud_init_drive_path,
+            image_path=image_path,
+            monitor_socket_path=monitor_socket_path,
+            qmp_socket_path=qmp_socket_path,
+            vcpu_count=vcpu_count,
+            mem_size_mb=mem_size_mb,
+            interface_name=interface_name,
+        )
 
-        cloud_init_drive = await self._create_cloud_init_drive()
-        if cloud_init_drive:
-            args += ["-cdrom", f"{cloud_init_drive.path_on_host}"]
-
-        try:
-            print(*args)
-
-            journal_stdout = journal.stream(self._journal_stdout_name)
-            journal_stderr = journal.stream(self._journal_stderr_name)
-            self.qemu_process = proc = await asyncio.create_subprocess_exec(
-                *args,
-                stdin=asyncio.subprocess.DEVNULL,
-                stdout=journal_stdout,
-                stderr=journal_stderr,
-            )
-
-            logger.debug(f"setup done {self}, {proc}")
-
-            async def handle_termination(proc: Process):
-                await proc.wait()
-                logger.info(f"{self} Process terminated with {proc.returncode} : {str(args)}")
-
-            loop = asyncio.get_running_loop()
-            loop.create_task(handle_termination(proc))
-        except Exception:
-            # Stop the VM and clear network interfaces in case any error prevented the start of the virtual machine.
-            logger.error("VM startup failed, cleaning up network")
-            if self.enable_networking:
-                teardown_nftables_for_vm(self.vm_id)
-            if self.tap_interface:
-                await self.tap_interface.delete()
-            raise
+        configuration = Configuration(
+            vm_id=self.vm_id, settings=settings, vm_configuration=vm_configuration, hypervisor=HypervisorType.qemu
+        )
 
-        if self.enable_console:
-            self.print_logs()
+        save_controller_configuration(self.vm_hash, configuration)
 
-        await self.wait_for_init()
-        logger.debug(f"started qemu vm {self} on {self.get_ip()}")
+    def save_controller_configuration(self):
+        """Save VM configuration to be used by the controller service"""
+        path = Path(f"{settings.EXECUTION_ROOT}/{self.vm_hash}-controller.json")
+        path.open("w").write(self.controller_configuration.json(by_alias=True, exclude_none=True, indent=4))
+        path.chmod(0o644)
+        return path
+
+    @property
+    def _journal_stdout_name(self) -> str:
+        return f"vm-{self.vm_hash}-stdout"
+
+    @property
+    def _journal_stderr_name(self) -> str:
+        return f"vm-{self.vm_hash}-stderr"
+
+    async def start(self):
+        # Start via systemd not here
+        raise NotImplementedError()
 
     async def wait_for_init(self) -> None:
         """Wait for the init process of the instance to be ready."""
@@ -326,10 +290,6 @@ async def wait_for_init(self) -> None:
                 else:
                     raise
 
-    async def configure(self):
-        """Nothing to configure, we do the configuration via cloud init"""
-        pass
-
     async def start_guest_api(self):
         pass
 
@@ -342,7 +302,6 @@ async def stop_guest_api(self):
     async def teardown(self):
         if self.print_task:
             self.print_task.cancel()
-        self._shutdown()
 
         if self.enable_networking:
             teardown_nftables_for_vm(self.vm_id)
@@ -366,26 +325,11 @@ async def print_logs():
         loop = asyncio.get_running_loop()
         self.print_task = loop.create_task(print_logs(), name=f"{self}-print-logs")
 
-    def _get_qmpclient(self) -> Optional[qmp.QEMUMonitorProtocol]:
-        if not (self.qmp_socket_path and self.qmp_socket_path.exists()):
-            return None
-        client = qmp.QEMUMonitorProtocol(str(self.qmp_socket_path))
-        client.connect()
-        return client
-
-    def _shutdown(self):
-        client = self._get_qmpclient()
-        if client:
-            resp = client.command("system_powerdown")
-            if not resp == {}:
-                logger.warning("unexpected answer from VM", resp)
-            client.close()
-        self.qmp_socket_path = None
-
     def get_log_queue(self) -> asyncio.Queue:
         queue, canceller = make_logs_queue(self._journal_stdout_name, self._journal_stderr_name)
         self._queue_cancellers[queue] = canceller
         # Limit the number of queues per VM
+        # TODO : fix
         if len(self.log_queues) > 20:
             logger.warning("Too many log queues, dropping the oldest one")
             self.unregister_queue(self.log_queues[1])
diff --git a/src/aleph/vm/hypervisors/qemu/__init__.py b/src/aleph/vm/hypervisors/qemu/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
new file mode 100644
index 000000000..8a77abb2d
--- /dev/null
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -0,0 +1,147 @@
+import asyncio
+import sys
+from asyncio import Task
+from asyncio.subprocess import Process
+from pathlib import Path
+from typing import Optional
+
+import qmp
+
+from aleph.vm.controllers.configuration import QemuVMConfiguration
+from aleph.vm.controllers.qemu.instance import logger
+
+
+class QemuVM(object):
+    qemu_bin_path: str
+    cloud_init_drive_path: Optional[str]
+    image_path: str
+    monitor_socket_path: Path
+    qmp_socket_path: Path
+    vcpu_count: int
+    mem_size_mb: int
+    interface_name: str
+    qemu_process = None
+
+    def __repr__(self):
+        return f""
+
+    def __init__(self, config: QemuVMConfiguration):
+        self.qemu_bin_path = config.qemu_bin_path
+        self.cloud_init_drive_path = config.cloud_init_drive_path
+        self.image_path = config.image_path
+        self.monitor_socket_path = config.monitor_socket_path
+        self.qmp_socket_path = config.qmp_socket_path
+        self.vcpu_count = config.vcpu_count
+        self.mem_size_mb = config.mem_size_mb
+        self.interface_name = config.interface_name
+
+    def prepare_start(self):
+        pass
+
+    async def start(
+        self,
+    ) -> Process:
+        # Based on the command
+        #  qemu-system-x86_64 -enable-kvm -m 2048 -net nic,model=virtio
+        # -net tap,ifname=tap0,script=no,downscript=no -drive file=alpine.qcow2,media=disk,if=virtio -nographic
+        # hardware_resources.published ports -> not implemented at the moment
+        # hardware_resources.seconds -> only for microvm
+        args = [
+            self.qemu_bin_path,
+            "-enable-kvm",
+            "-nodefaults",
+            "-m",
+            str(self.mem_size_mb),
+            "-smp",
+            str(self.vcpu_count),
+            "-drive",
+            f"file={self.image_path},media=disk,if=virtio",
+            # To debug you can pass gtk or curses instead
+            "-display",
+            "none",
+            "--no-reboot",  # Rebooting from inside the VM shuts down the machine
+            # Listen for commands on this socket
+            "-monitor",
+            f"unix:{self.monitor_socket_path},server,nowait",
+            # Listen for commands on this socket (QMP protocol in json). Supervisor use it to send shutdown or start
+            # command
+            "-qmp",
+            f"unix:{self.qmp_socket_path},server,nowait",
+            # Tell to put the output to std fd, so we can include them in the log
+            "-serial",
+            "stdio",
+            # Uncomment for debug
+            # "-serial", "telnet:localhost:4321,server,nowait",
+            # "-snapshot",  # Do not save anything to disk
+        ]
+        if self.interface_name:
+            # script=no, downscript=no tell qemu not to try to set up the network itself
+            args += ["-net", "nic,model=virtio", "-net", f"tap,ifname={self.interface_name},script=no,downscript=no"]
+
+        if self.cloud_init_drive_path:
+            args += ["-cdrom", f"{self.cloud_init_drive_path}"]
+        print(*args)
+        self.qemu_process = proc = await asyncio.create_subprocess_exec(
+            *args,
+            stdin=asyncio.subprocess.DEVNULL,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+
+        logger.debug(f"started qemu vm {self}, {proc}")
+        return proc
+
+    log_queues: list[asyncio.Queue] = []
+
+    # TODO : convert when merging with log fixing branch
+    async def _process_stderr(self):
+        while not self.qemu_process:
+            await asyncio.sleep(0.01)  # Todo: Use signal here
+        while True:
+            line = await self.qemu_process.stderr.readline()
+            if not line:  # FD is closed nothing more will come
+                print(self, "EOF")
+                return
+            for queue in self.log_queues:
+                await queue.put(("stderr", line))
+            print(self, line.decode().strip(), file=sys.stderr)
+
+    def start_printing_logs(self) -> tuple[Task, Task]:
+        """Start two tasks to process the stdout and stderr
+
+        It will stream their content to queues registered on self.log_queues
+        It will also print them"""
+
+        loop = asyncio.get_running_loop()
+        stdout_task = loop.create_task(self._process_stdout())
+        stderr_task = loop.create_task(self._process_stderr())
+        return stdout_task, stderr_task
+
+    async def _process_stdout(self):
+        while not self.qemu_process:
+            await asyncio.sleep(0.01)  # Todo: Use signal here
+        while True:
+            line = await self.qemu_process.stdout.readline()
+            if not line:  # FD is closed nothing more will come
+                print(self, "EOF")
+                return
+            for queue in self.log_queues:
+                await queue.put(("stdout", line))
+            print(self, line.decode().strip())
+
+    def _get_qmpclient(self) -> Optional[qmp.QEMUMonitorProtocol]:
+        if not (self.qmp_socket_path and self.qmp_socket_path.exists()):
+            return None
+        client = qmp.QEMUMonitorProtocol(str(self.qmp_socket_path))
+        client.connect()
+        return client
+
+    def send_shutdown_message(self):
+        print("sending shutdown message to vm")
+        client = self._get_qmpclient()
+        if client:
+            resp = client.command("system_powerdown")
+            if not resp == {}:
+                logger.warning("unexpected answer from VM", resp)
+            print("shutdown message sent")
+            client.close()
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index cb9f8b84d..bc3f5ef2b 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -106,12 +106,17 @@ def becomes_ready(self):
     def vm_id(self) -> Optional[int]:
         return self.vm.vm_id if self.vm else None
 
+    @property
+    def controller_service(self) -> str:
+        return f"aleph-vm-controller@{self.vm_hash}.service"
+
     def __init__(
         self,
         vm_hash: ItemHash,
         message: ExecutableContent,
         original: ExecutableContent,
         snapshot_manager: "SnapshotManager",
+        persistent: bool,
     ):
         self.uuid = uuid.uuid1()  # uuid1() includes the hardware address and timestamp
         self.vm_hash = vm_hash
@@ -121,10 +126,11 @@ def __init__(
         self.ready_event = asyncio.Event()
         self.concurrent_runs = 0
         self.runs_done_event = asyncio.Event()
-        self.stop_event = asyncio.Event()  #  triggered when the VM is stopped
+        self.stop_event = asyncio.Event()  # triggered when the VM is stopped
         self.preparation_pending_lock = asyncio.Lock()
         self.stop_pending_lock = asyncio.Lock()
         self.snapshot_manager = snapshot_manager
+        self.persistent = persistent
 
     def to_dict(self) -> dict:
         return {
@@ -175,6 +181,7 @@ async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None)
                 enable_networking=self.message.environment.internet,
                 hardware_resources=self.message.resources,
                 tap_interface=tap_interface,
+                persistent=self.persistent,
             )
         elif self.is_instance:
             if self.hypervisor == HypervisorType.firecracker:
@@ -204,7 +211,10 @@ async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None)
 
         try:
             await vm.setup()
-            await vm.start()
+            # Avoid VM start() method because it's only for ephemeral programs,
+            # for persistent and instances we will use SystemD manager
+            if not self.persistent:
+                await vm.start()
             await vm.configure()
             await vm.start_guest_api()
             self.times.started_at = datetime.now(tz=timezone.utc)
@@ -214,6 +224,9 @@ async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None)
             await vm.teardown()
             raise
 
+    async def wait_for_init(self):
+        await self.vm.wait_for_init()
+
     def stop_after_timeout(self, timeout: float = 5.0) -> Optional[Task]:
         if self.persistent:
             logger.debug("VM marked as long running. Ignoring timeout.")
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index 53cdd3157..b3c380634 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -50,7 +50,7 @@ async def build_event_scope(event) -> dict[str, Any]:
     }
 
 
-async def create_vm_execution(vm_hash: ItemHash, pool: VmPool) -> VmExecution:
+async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool = False) -> VmExecution:
     message, original_message = await load_updated_message(vm_hash)
     pool.message_cache[vm_hash] = message
 
@@ -61,6 +61,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool) -> VmExecution:
             vm_hash=vm_hash,
             message=message.content,
             original=original_message.content,
+            persistent=persistent,
         )
     except ResourceDownloadError as error:
         logger.exception(error)
@@ -263,13 +264,12 @@ async def start_persistent_vm(vm_hash: ItemHash, pubsub: Optional[PubSub], pool:
 
     if not execution:
         logger.info(f"Starting persistent virtual machine with id: {vm_hash}")
-        execution = await create_vm_execution(vm_hash=vm_hash, pool=pool)
+        execution = await create_vm_execution(vm_hash=vm_hash, pool=pool, persistent=True)
 
     await execution.becomes_ready()
 
     # If the VM was already running in lambda mode, it should not expire
     # as long as it is also scheduled as long-running
-    execution.persistent = True
     execution.cancel_expiration()
 
     if pubsub and settings.WATCH_FOR_UPDATES:
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 813efa202..a74784f3f 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -256,8 +256,7 @@ async def update_allocations(request: web.Request):
         if execution.vm_hash not in allocations and execution.is_running:
             vm_type = "instance" if execution.is_instance else "persistent program"
             logger.info("Stopping %s %s", vm_type, execution.vm_hash)
-            await execution.stop()
-            execution.persistent = False
+            await pool.stop_vm(execution.vm_hash)
 
     # Second start persistent VMs and instances sequentially to limit resource usage.
 
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 997f85d34..97181d736 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -9,6 +9,7 @@
 from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
 from aleph.vm.network.hostnetwork import Network, make_ipv6_allocator
+from aleph.vm.systemd import SystemDManager
 from aleph.vm.vm_type import VmType
 
 from .models import ExecutableContent, VmExecution
@@ -30,6 +31,7 @@ class VmPool:
     message_cache: dict[str, ExecutableMessage] = {}
     network: Optional[Network]
     snapshot_manager: SnapshotManager
+    systemd_manager: SystemDManager
 
     def __init__(self):
         self.counter = settings.START_ID_INDEX
@@ -51,6 +53,7 @@ def __init__(self):
             if settings.ALLOW_VM_NETWORKING
             else None
         )
+        self.systemd_manager = SystemDManager()
         self.snapshot_manager = SnapshotManager()
         logger.debug("Initializing SnapshotManager ...")
         self.snapshot_manager.run_snapshots()
@@ -66,7 +69,7 @@ def teardown(self) -> None:
             self.network.teardown()
 
     async def create_a_vm(
-        self, vm_hash: ItemHash, message: ExecutableContent, original: ExecutableContent
+        self, vm_hash: ItemHash, message: ExecutableContent, original: ExecutableContent, persistent: bool
     ) -> VmExecution:
         """Create a new Aleph Firecracker VM from an Aleph function message."""
 
@@ -80,6 +83,7 @@ async def create_a_vm(
                 message=message,
                 original=original,
                 snapshot_manager=self.snapshot_manager,
+                persistent=persistent,
             )
             self.executions[vm_hash] = execution
 
@@ -95,9 +99,14 @@ async def create_a_vm(
 
             await execution.create(vm_id=vm_id, tap_interface=tap_interface)
 
-            assert execution.vm
-            # Start VM snapshots automatically
-            if execution.vm.support_snapshot:
+            # Start VM and snapshots automatically
+            if execution.persistent:
+                self.systemd_manager.enable_and_start(execution.controller_service)
+                await execution.wait_for_init()
+                if execution.is_program and execution.vm:
+                    await execution.vm.load_configuration()
+
+            if execution.vm and execution.vm.support_snapshot:
                 await self.snapshot_manager.start_for(vm=execution.vm)
         except Exception:
             # ensure the VM is removed from the pool on creation error
@@ -152,11 +161,21 @@ async def stop_vm(self, vm_hash: ItemHash) -> Optional[VmExecution]:
         """Stop a VM."""
         execution = self.executions.get(vm_hash)
         if execution:
-            await execution.stop()
+            if execution.persistent:
+                await self.stop_persistent_execution(execution)
+            else:
+                await execution.stop()
             return execution
         else:
             return None
 
+    async def stop_persistent_execution(self, execution):
+        """Stop persistent VMs in the pool."""
+        assert execution.persistent, "Execution isn't persistent"
+        self.systemd_manager.stop_and_disable(execution.controller_service)
+        await execution.stop()
+        execution.persistent = False
+
     def forget_vm(self, vm_hash: ItemHash) -> None:
         """Remove a VM from the executions pool.
 
@@ -170,17 +189,27 @@ def forget_vm(self, vm_hash: ItemHash) -> None:
             pass
 
     async def stop(self):
-        """Stop all VMs in the pool."""
-
+        """Stop ephemeral VMs in the pool."""
         # Stop executions in parallel:
-        await asyncio.gather(*(execution.stop() for vm_hash, execution in self.executions.items()))
+        await asyncio.gather(*(execution.stop() for vm_hash, execution in self.get_ephemeral_executions()))
+
+    def get_ephemeral_executions(self) -> Iterable[VmExecution]:
+        return (
+            execution
+            for _vm_hash, execution in self.executions.items()
+            if execution.is_running and not execution.persistent
+        )
 
     def get_persistent_executions(self) -> Iterable[VmExecution]:
-        for _vm_hash, execution in self.executions.items():
-            if execution.persistent and execution.is_running:
-                yield execution
+        return (
+            execution
+            for _vm_hash, execution in self.executions.items()
+            if execution.is_running and execution.persistent
+        )
 
     def get_instance_executions(self) -> Iterable[VmExecution]:
-        for _vm_hash, execution in self.executions.items():
-            if execution.is_instance and execution.is_running:
-                yield execution
+        return (
+            execution
+            for _vm_hash, execution in self.executions.items()
+            if execution.is_running and execution.is_instance
+        )
diff --git a/src/aleph/vm/systemd.py b/src/aleph/vm/systemd.py
new file mode 100644
index 000000000..bbbaeed4c
--- /dev/null
+++ b/src/aleph/vm/systemd.py
@@ -0,0 +1,74 @@
+"""
+async SystemD Manager implementation.
+"""
+
+import logging
+import sys
+
+import dbus
+from dbus import DBusException, SystemBus
+from dbus.proxies import Interface
+
+logger = logging.getLogger(__name__)
+
+
+class SystemDManager:
+    """SystemD Manager class.
+
+    Used to manage the systemd services on the host on Linux.
+    """
+
+    bus: SystemBus
+    interface: Interface
+
+    def __init__(self):
+        self.bus = dbus.SystemBus()
+        systemd = self.bus.get_object("org.freedesktop.systemd1", "/org/freedesktop/systemd1")
+        self.interface = dbus.Interface(systemd, "org.freedesktop.systemd1.Manager")
+
+    def stop_and_disable(self, service: str) -> None:
+        if self.is_service_active(service):
+            self.stop(service)
+        if self.is_service_enabled(service):
+            self.disable(service)
+
+    def enable(self, service: str) -> None:
+        self.interface.EnableUnitFiles([service], False, True)
+        logger.debug(f"Enabled {service} service")
+
+    def start(self, service: str) -> None:
+        self.interface.StartUnit(service, "replace")
+        logger.debug(f"Started {service} service")
+
+    def stop(self, service: str) -> None:
+        self.interface.StopUnit(service, "replace")
+        logger.debug(f"Stopped {service} service")
+
+    def restart(self, service: str) -> None:
+        self.interface.RestartUnit(service, "replace")
+        logger.debug(f"Restarted {service} service")
+
+    def disable(self, service: str) -> None:
+        self.interface.DisableUnitFiles([service], False)
+        logger.debug(f"Disabled {service} service")
+
+    def is_service_enabled(self, service: str) -> bool:
+        try:
+            return self.interface.GetUnitFileState(service) == "enabled"
+        except DBusException as error:
+            logger.error(error)
+            return False
+
+    def is_service_active(self, service: str) -> bool:
+        try:
+            self.interface.GetUnit(service)
+            return True
+        except DBusException as error:
+            logger.error(error)
+            return False
+
+    def enable_and_start(self, service: str) -> None:
+        if not self.is_service_enabled(service):
+            self.enable(service)
+        if not self.is_service_active(service):
+            self.start(service)

From 3ab81e3141149462b29be7465f51ea2fb02afc11 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 14 Dec 2023 18:21:47 +0100
Subject: [PATCH 588/990] Problem: When the node operator restart the
 orchestrator with already running persistent VMs, at the start, it doesn't
 get already running persistent VMs.

Solution: Use the sqlite database to save all the executions and re-create it from the start.
---
 src/aleph/vm/conf.py                          |  2 +
 .../vm/controllers/firecracker/executable.py  |  4 +-
 .../vm/controllers/firecracker/instance.py    |  5 +-
 .../vm/controllers/firecracker/program.py     |  2 +
 src/aleph/vm/models.py                        | 63 +++++++++++++------
 src/aleph/vm/network/hostnetwork.py           |  9 ++-
 src/aleph/vm/orchestrator/metrics.py          | 41 ++++++++++--
 src/aleph/vm/orchestrator/supervisor.py       |  9 +--
 src/aleph/vm/orchestrator/views/operator.py   |  3 +-
 src/aleph/vm/pool.py                          | 54 +++++++++++++++-
 src/aleph/vm/systemd.py                       | 24 ++++---
 src/aleph/vm/utils.py                         | 13 +++-
 12 files changed, 184 insertions(+), 45 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index ec6a2c23d..640d765e3 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -364,6 +364,8 @@ def __init__(
             self.JAILER_BASE_DIRECTORY = self.EXECUTION_ROOT / "jailer"
         if not self.PERSISTENT_VOLUMES_DIR:
             self.PERSISTENT_VOLUMES_DIR = self.EXECUTION_ROOT / "volumes" / "persistent"
+        if not self.EXECUTION_DATABASE:
+            self.EXECUTION_DATABASE = self.EXECUTION_ROOT / "executions.sqlite3"
         if not self.EXECUTION_LOG_DIRECTORY:
             self.EXECUTION_LOG_DIRECTORY = self.EXECUTION_ROOT / "executions"
         if not self.JAILER_BASE_DIR:
diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index a63db3970..358524d36 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -169,6 +169,7 @@ def __init__(
         hardware_resources: Optional[MachineResources] = None,
         tap_interface: Optional[TapInterface] = None,
         persistent: bool = False,
+        prepare_jailer: bool = True,
     ):
         self.vm_id = vm_id
         self.vm_hash = vm_hash
@@ -189,7 +190,8 @@ def __init__(
             jailer_bin_path=settings.JAILER_PATH,
             init_timeout=settings.INIT_TIMEOUT,
         )
-        self.fvm.prepare_jailer()
+        if prepare_jailer:
+            self.fvm.prepare_jailer()
 
         # These properties are set later in the setup and configuration.
         self.vm_configuration = None
diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index a72a82d9e..427d73a65 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -69,8 +69,10 @@ def __init__(
         enable_console: Optional[bool] = None,
         hardware_resources: Optional[MachineResources] = None,
         tap_interface: Optional[TapInterface] = None,
+        prepare_jailer: bool = True,
     ):
         self.latest_snapshot = None
+        persistent = True
         super().__init__(
             vm_id,
             vm_hash,
@@ -79,7 +81,8 @@ def __init__(
             enable_console,
             hardware_resources or MachineResources(),
             tap_interface,
-            persistent=True,
+            persistent,
+            prepare_jailer,
         )
 
     async def setup(self):
diff --git a/src/aleph/vm/controllers/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
index 4035666d6..33ed763ca 100644
--- a/src/aleph/vm/controllers/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -270,6 +270,7 @@ def __init__(
         hardware_resources: MachineResources = MachineResources(),
         tap_interface: TapInterface | None = None,
         persistent: bool = False,
+        prepare_jailer: bool = True,
     ):
         super().__init__(
             vm_id,
@@ -280,6 +281,7 @@ def __init__(
             hardware_resources,
             tap_interface,
             persistent,
+            prepare_jailer,
         )
 
     async def setup(self):
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index bc3f5ef2b..91bf7df7e 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -10,7 +10,6 @@
     ExecutableContent,
     InstanceContent,
     ItemHash,
-    MessageType,
     ProgramContent,
 )
 from aleph_message.models.execution.environment import HypervisorType
@@ -28,6 +27,7 @@
 from aleph.vm.network.interfaces import TapInterface
 from aleph.vm.orchestrator.metrics import (
     ExecutionRecord,
+    delete_record,
     save_execution_data,
     save_record,
 )
@@ -37,6 +37,7 @@
 
 if TYPE_CHECKING:
     from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
+    from aleph.vm.systemd import SystemDManager
 
 logger = logging.getLogger(__name__)
 
@@ -83,7 +84,11 @@ class VmExecution:
 
     @property
     def is_running(self):
-        return self.times.starting_at and not self.times.stopping_at
+        return (
+            self.times.starting_at and not self.times.stopping_at
+            if not self.persistent
+            else self.systemd_manager.is_service_active(self.controller_service)
+        )
 
     @property
     def is_program(self):
@@ -116,6 +121,7 @@ def __init__(
         message: ExecutableContent,
         original: ExecutableContent,
         snapshot_manager: "SnapshotManager",
+        systemd_manager: "SystemDManager",
         persistent: bool,
     ):
         self.uuid = uuid.uuid1()  # uuid1() includes the hardware address and timestamp
@@ -130,6 +136,7 @@ def __init__(
         self.preparation_pending_lock = asyncio.Lock()
         self.stop_pending_lock = asyncio.Lock()
         self.snapshot_manager = snapshot_manager
+        self.systemd_manager = systemd_manager
         self.persistent = persistent
 
     def to_dict(self) -> dict:
@@ -141,7 +148,7 @@ def to_dict(self) -> dict:
     def to_json(self, indent: Optional[int] = None) -> str:
         return dumps_for_json(self.to_dict(), indent=indent)
 
-    async def prepare(self):
+    async def prepare(self, download: bool = True):
         """Download VM required files"""
         async with self.preparation_pending_lock:
             if self.resources:
@@ -161,15 +168,17 @@ async def prepare(self):
             if not resources:
                 msg = "Unknown executable message type"
                 raise ValueError(msg, repr(self.message))
-            await resources.download_all()
+            if download:
+                await resources.download_all()
             self.times.prepared_at = datetime.now(tz=timezone.utc)
             self.resources = resources
 
-    async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None) -> AlephVmControllerInterface:
+    def create(
+        self, vm_id: int, tap_interface: Optional[TapInterface] = None, prepare: bool = True
+    ) -> AlephVmControllerInterface:
         if not self.resources:
             msg = "Execution resources must be configured first"
             raise ValueError(msg)
-        self.times.starting_at = datetime.now(tz=timezone.utc)
 
         vm: AlephVmControllerInterface
         if self.is_program:
@@ -182,6 +191,7 @@ async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None)
                 hardware_resources=self.message.resources,
                 tap_interface=tap_interface,
                 persistent=self.persistent,
+                prepare_jailer=prepare,
             )
         elif self.is_instance:
             if self.hypervisor == HypervisorType.firecracker:
@@ -193,6 +203,7 @@ async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None)
                     enable_networking=self.message.environment.internet,
                     hardware_resources=self.message.resources,
                     tap_interface=tap_interface,
+                    prepare_jailer=prepare,
                 )
             elif self.hypervisor == HypervisorType.qemu:
                 assert isinstance(self.resources, AlephQemuResources)
@@ -209,19 +220,24 @@ async def create(self, vm_id: int, tap_interface: Optional[TapInterface] = None)
         else:
             raise Exception("Unknown VM")
 
+        return vm
+
+    async def start(self):
+        self.times.starting_at = datetime.now(tz=timezone.utc)
+
         try:
-            await vm.setup()
+            await self.vm.setup()
             # Avoid VM start() method because it's only for ephemeral programs,
             # for persistent and instances we will use SystemD manager
             if not self.persistent:
-                await vm.start()
-            await vm.configure()
-            await vm.start_guest_api()
+                await self.vm.start()
+            await self.vm.configure()
+            await self.vm.start_guest_api()
             self.times.started_at = datetime.now(tz=timezone.utc)
             self.ready_event.set()
-            return vm
+            await self.save()
         except Exception:
-            await vm.teardown()
+            await self.vm.teardown()
             raise
 
     async def wait_for_init(self):
@@ -310,16 +326,15 @@ async def all_runs_complete(self):
             logger.debug("Stop: waiting for runs to complete...")
             await self.runs_done_event.wait()
 
-    async def record_usage(self):
-        if settings.EXECUTION_LOG_ENABLED:
-            await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json())
+    async def save(self):
         pid_info = self.vm.to_dict()
         # Handle cases when the process cannot be accessed
-        if pid_info and pid_info.get("process"):
+        if not self.persistent and pid_info and pid_info.get("process"):
             await save_record(
                 ExecutionRecord(
                     uuid=str(self.uuid),
                     vm_hash=self.vm_hash,
+                    vm_id=self.vm_id,
                     time_defined=self.times.defined_at,
                     time_prepared=self.times.prepared_at,
                     time_started=self.times.started_at,
@@ -333,15 +348,18 @@ async def record_usage(self):
                     vcpus=self.vm.hardware_resources.vcpus,
                     memory=self.vm.hardware_resources.memory,
                     network_tap=self.vm.tap_interface.device_name if self.vm.tap_interface else "",
+                    message=self.message,
+                    original_message=self.original,
+                    persistent=self.persistent,
                 )
             )
         else:
-            # The process cannot be accessed. It has probably already exited
-            # and its metrics are not available anymore.
+            # The process cannot be accessed, or it's a persistent VM.
             await save_record(
                 ExecutionRecord(
                     uuid=str(self.uuid),
                     vm_hash=self.vm_hash,
+                    vm_id=self.vm_id,
                     time_defined=self.times.defined_at,
                     time_prepared=self.times.prepared_at,
                     time_started=self.times.started_at,
@@ -354,9 +372,17 @@ async def record_usage(self):
                     io_write_bytes=None,
                     vcpus=self.vm.hardware_resources.vcpus,
                     memory=self.vm.hardware_resources.memory,
+                    message=self.message.json(),
+                    original_message=self.original.json(),
+                    persistent=self.persistent,
                 )
             )
 
+    async def record_usage(self):
+        await delete_record(execution_uuid=str(self.uuid))
+        if settings.EXECUTION_LOG_ENABLED:
+            await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json())
+
     async def run_code(self, scope: Optional[dict] = None) -> bytes:
         if not self.vm:
             msg = "The VM has not been created yet"
@@ -365,6 +391,7 @@ async def run_code(self, scope: Optional[dict] = None) -> bytes:
         if not self.is_program:
             msg = "Code can ony be run on programs"
             raise ValueError(msg)
+
         assert isinstance(self.vm, AlephFirecrackerProgram)
 
         self.concurrent_runs += 1
diff --git a/src/aleph/vm/network/hostnetwork.py b/src/aleph/vm/network/hostnetwork.py
index 566a4b94c..f938f5e3c 100644
--- a/src/aleph/vm/network/hostnetwork.py
+++ b/src/aleph/vm/network/hostnetwork.py
@@ -203,8 +203,8 @@ def teardown(self) -> None:
         self.reset_ipv4_forwarding_state()
         self.reset_ipv6_forwarding_state()
 
-    async def create_tap(self, vm_id: int, vm_hash: ItemHash, vm_type: VmType) -> TapInterface:
-        """Create TAP interface to be used by VM"""
+    async def prepare_tap(self, vm_id: int, vm_hash: ItemHash, vm_type: VmType) -> TapInterface:
+        """Prepare TAP interface to be used by VM"""
         interface = TapInterface(
             f"vmtap{vm_id}",
             ip_network=self.get_network_for_tap(vm_id),
@@ -215,6 +215,9 @@ async def create_tap(self, vm_id: int, vm_hash: ItemHash, vm_type: VmType) -> Ta
             ),
             ndp_proxy=self.ndp_proxy,
         )
+        return interface
+
+    async def create_tap(self, vm_id: int, interface: TapInterface):
+        """Create TAP interface to be used by VM"""
         await interface.create()
         setup_nftables_for_vm(vm_id, interface)
-        return interface
diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index d195af215..3a695e9e0 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -4,7 +4,16 @@
 from typing import Any
 from uuid import UUID
 
-from sqlalchemy import Column, DateTime, Float, Integer, String, create_engine
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    Column,
+    DateTime,
+    Float,
+    Integer,
+    String,
+    create_engine,
+)
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import sessionmaker
 
@@ -13,7 +22,6 @@
 except ImportError:
     from sqlalchemy.ext.declarative import declarative_base
 
-
 from aleph.vm.conf import make_db_url, settings
 
 Session: sessionmaker
@@ -35,10 +43,11 @@ def create_tables(engine: Engine):
 
 
 class ExecutionRecord(Base):
-    __tablename__ = "records"
+    __tablename__ = "executions"
 
     uuid = Column(String, primary_key=True)
     vm_hash = Column(String, nullable=False)
+    vm_id = Column(Integer, nullable=True)
 
     time_defined = Column(DateTime, nullable=False)
     time_prepared = Column(DateTime)
@@ -57,8 +66,12 @@ class ExecutionRecord(Base):
     memory = Column(Integer, nullable=False)
     network_tap = Column(String, nullable=True)
 
+    message = Column(JSON, nullable=True)
+    original_message = Column(JSON, nullable=True)
+    persistent = Column(Boolean, nullable=True)
+
     def __repr__(self):
-        return f""
+        return f""
 
     def to_dict(self):
         return {c.name: getattr(self, c.name) for c in self.__table__.c}
@@ -81,6 +94,26 @@ async def save_record(record: ExecutionRecord):
         session.close()
 
 
+async def delete_record(execution_uuid: str):
+    """Delete the resource usage in database"""
+    session = Session()  # undefined name 'Session'
+    try:
+        session.query(ExecutionRecord).filter(ExecutionRecord.uuid == execution_uuid).delete()
+        session.commit()
+    finally:
+        session.close()
+
+
+async def delete_all_records():
+    """Delete all the resource usage in database"""
+    session = Session()  # undefined name 'Session'
+    try:
+        session.query(ExecutionRecord).delete()
+        session.commit()
+    finally:
+        session.close()
+
+
 async def get_execution_records() -> Iterable[ExecutionRecord]:
     """Get the execution records from the database."""
     session = Session()  # undefined name 'Session'
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index aa51cc577..ed94ff150 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -107,6 +107,10 @@ async def stop_all_vms(app: web.Application):
 def run():
     """Run the VM Supervisor."""
     settings.check()
+
+    engine = setup_engine()
+    create_tables(engine)
+
     pool = VmPool()
     pool.setup()
 
@@ -119,10 +123,7 @@ def run():
     app["secret_token"] = secret_token
     app["vm_pool"] = pool
 
-    print(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
-
-    engine = setup_engine()
-    create_tables(engine)
+    logger.debug(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
 
     try:
         if settings.WATCH_FOR_MESSAGES:
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 311a9b701..bc8153a60 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -171,6 +171,7 @@ async def operate_reboot(request: web.Request, authenticated_sender: str) -> web
     if execution.is_running:
         logger.info(f"Rebooting {execution.vm_hash}")
         await pool.stop_vm(vm_hash)
+        await pool.forget_vm(vm_hash)
 
         await create_vm_execution(vm_hash=vm_hash, pool=pool)
         return web.Response(status=200, body=f"Rebooted VM with ref {vm_hash}")
@@ -194,7 +195,7 @@ async def operate_erase(request: web.Request, authenticated_sender: str) -> web.
 
     # Stop the VM
     await execution.stop()
-    execution.persistent = False
+    await pool.forget_vm(execution.vm_hash)
 
     # Delete all data
     if execution.resources is not None:
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 97181d736..1f47b8c34 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -1,6 +1,8 @@
 import asyncio
+import json
 import logging
 from collections.abc import Iterable
+from datetime import datetime, timezone
 from typing import Optional
 
 from aleph_message.models import ExecutableMessage, ItemHash
@@ -9,7 +11,9 @@
 from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
 from aleph.vm.network.hostnetwork import Network, make_ipv6_allocator
+from aleph.vm.orchestrator.metrics import delete_all_records, get_execution_records
 from aleph.vm.systemd import SystemDManager
+from aleph.vm.utils import get_message_executable_content
 from aleph.vm.vm_type import VmType
 
 from .models import ExecutableContent, VmExecution
@@ -57,6 +61,9 @@ def __init__(self):
         self.snapshot_manager = SnapshotManager()
         logger.debug("Initializing SnapshotManager ...")
         self.snapshot_manager.run_snapshots()
+        logger.debug("Loading existing executions ...")
+        # asyncio.run(delete_all_records())
+        asyncio.run(self._load_persistent_executions())
 
     def setup(self) -> None:
         """Set up the VM pool and the network."""
@@ -83,6 +90,7 @@ async def create_a_vm(
                 message=message,
                 original=original,
                 snapshot_manager=self.snapshot_manager,
+                systemd_manager=self.systemd_manager,
                 persistent=persistent,
             )
             self.executions[vm_hash] = execution
@@ -93,11 +101,13 @@ async def create_a_vm(
 
             if self.network:
                 vm_type = VmType.from_message_content(message)
-                tap_interface = await self.network.create_tap(vm_id, vm_hash, vm_type)
+                tap_interface = await self.network.prepare_tap(vm_id, vm_hash, vm_type)
+                await self.network.create_tap(vm_id, tap_interface)
             else:
                 tap_interface = None
 
-            await execution.create(vm_id=vm_id, tap_interface=tap_interface)
+            execution.create(vm_id=vm_id, tap_interface=tap_interface)
+            await execution.start()
 
             # Start VM and snapshots automatically
             if execution.persistent:
@@ -188,6 +198,46 @@ def forget_vm(self, vm_hash: ItemHash) -> None:
         except KeyError:
             pass
 
+    async def _load_persistent_executions(self):
+        """Load persistent executions from the database."""
+        saved_executions = await get_execution_records()
+        for saved_execution in saved_executions:
+            # Prevent to load the same execution twice
+            if self.executions.get(saved_execution.vm_hash):
+                break
+
+            vm_id = saved_execution.vm_id
+            message_dict = json.loads(saved_execution.message)
+            original_dict = json.loads(saved_execution.original_message)
+            execution = VmExecution(
+                vm_hash=saved_execution.vm_hash,
+                message=get_message_executable_content(message_dict),
+                original=get_message_executable_content(message_dict),
+                snapshot_manager=self.snapshot_manager,
+                systemd_manager=self.systemd_manager,
+                persistent=saved_execution.persistent,
+            )
+            if execution.is_running:
+                # TODO: Improve the way that we re-create running execution
+                await execution.prepare(download=False)
+                if self.network:
+                    vm_type = VmType.from_message_content(execution.message)
+                    tap_interface = await self.network.prepare_tap(vm_id, execution.vm_hash, vm_type)
+                else:
+                    tap_interface = None
+
+                execution.create(vm_id=vm_id, tap_interface=tap_interface, prepare=False)
+                await execution.vm.start_guest_api()
+                execution.ready_event.set()
+                execution.times.started_at = datetime.now(tz=timezone.utc)
+
+                self.executions[execution.vm_hash] = execution
+            else:
+                execution.uuid = saved_execution.uuid
+                await execution.record_usage()
+
+        logger.debug(f"Loaded {len(self.executions)} executions")
+
     async def stop(self):
         """Stop ephemeral VMs in the pool."""
         # Stop executions in parallel:
diff --git a/src/aleph/vm/systemd.py b/src/aleph/vm/systemd.py
index bbbaeed4c..ac06d5c91 100644
--- a/src/aleph/vm/systemd.py
+++ b/src/aleph/vm/systemd.py
@@ -4,6 +4,7 @@
 
 import logging
 import sys
+from typing import Any
 
 import dbus
 from dbus import DBusException, SystemBus
@@ -19,12 +20,12 @@ class SystemDManager:
     """
 
     bus: SystemBus
-    interface: Interface
+    manager: Interface
 
     def __init__(self):
         self.bus = dbus.SystemBus()
         systemd = self.bus.get_object("org.freedesktop.systemd1", "/org/freedesktop/systemd1")
-        self.interface = dbus.Interface(systemd, "org.freedesktop.systemd1.Manager")
+        self.manager = dbus.Interface(systemd, "org.freedesktop.systemd1.Manager")
 
     def stop_and_disable(self, service: str) -> None:
         if self.is_service_active(service):
@@ -33,36 +34,39 @@ def stop_and_disable(self, service: str) -> None:
             self.disable(service)
 
     def enable(self, service: str) -> None:
-        self.interface.EnableUnitFiles([service], False, True)
+        self.manager.EnableUnitFiles([service], False, True)
         logger.debug(f"Enabled {service} service")
 
     def start(self, service: str) -> None:
-        self.interface.StartUnit(service, "replace")
+        self.manager.StartUnit(service, "replace")
         logger.debug(f"Started {service} service")
 
     def stop(self, service: str) -> None:
-        self.interface.StopUnit(service, "replace")
+        self.manager.StopUnit(service, "replace")
         logger.debug(f"Stopped {service} service")
 
     def restart(self, service: str) -> None:
-        self.interface.RestartUnit(service, "replace")
+        self.manager.RestartUnit(service, "replace")
         logger.debug(f"Restarted {service} service")
 
     def disable(self, service: str) -> None:
-        self.interface.DisableUnitFiles([service], False)
+        self.manager.DisableUnitFiles([service], False)
         logger.debug(f"Disabled {service} service")
 
     def is_service_enabled(self, service: str) -> bool:
         try:
-            return self.interface.GetUnitFileState(service) == "enabled"
+            return self.manager.GetUnitFileState(service) == "enabled"
         except DBusException as error:
             logger.error(error)
             return False
 
     def is_service_active(self, service: str) -> bool:
         try:
-            self.interface.GetUnit(service)
-            return True
+            systemd_service = self.bus.get_object("org.freedesktop.systemd1", object_path=self.manager.GetUnit(service))
+            unit = dbus.Interface(systemd_service, "org.freedesktop.systemd1.Unit")
+            unit_properties = dbus.Interface(unit, "org.freedesktop.DBus.Properties")
+            active_state = unit_properties.Get("org.freedesktop.systemd1.Unit", "ActiveState")
+            return active_state == "active"
         except DBusException as error:
             logger.error(error)
             return False
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index ee3deed6a..758deb169 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -9,14 +9,25 @@
 from dataclasses import asdict as dataclass_as_dict
 from dataclasses import is_dataclass
 from shutil import disk_usage
-from typing import Any, Optional
+from typing import Any, Dict, Optional
 
 import aiodns
 import msgpack
+from aleph_message.models import ExecutableContent, InstanceContent, ProgramContent
+from aleph_message.models.execution.base import MachineType
 
 logger = logging.getLogger(__name__)
 
 
+def get_message_executable_content(message_dict: Dict) -> ExecutableContent:
+    if message_dict["type"] == MachineType.vm_function:
+        return ProgramContent.parse_obj(message_dict)
+    elif message_dict["type"] == MachineType.vm_instance:
+        return InstanceContent.parse_obj(message_dict)
+    else:
+        raise ValueError(f"Unknown message type {message_dict['type']}")
+
+
 class MsgpackSerializable:
     def __post_init__(self, *args, **kwargs):
         if not is_dataclass(self):

From 7476c92c7a47e6f8b5a159115499a34ef459f976 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 14 Dec 2023 19:13:07 +0100
Subject: [PATCH 589/990] Fix: Solved Sentry issues for programs and related to
 update_allocations.

---
 src/aleph/vm/models.py                      | 4 ++--
 src/aleph/vm/orchestrator/views/__init__.py | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 91bf7df7e..c05f96d2a 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -348,8 +348,8 @@ async def save(self):
                     vcpus=self.vm.hardware_resources.vcpus,
                     memory=self.vm.hardware_resources.memory,
                     network_tap=self.vm.tap_interface.device_name if self.vm.tap_interface else "",
-                    message=self.message,
-                    original_message=self.original,
+                    message=self.message.json(),
+                    original_message=self.original.json(),
                     persistent=self.persistent,
                 )
             )
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index a74784f3f..106aa27e8 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -252,11 +252,12 @@ async def update_allocations(request: web.Request):
     # First free resources from persistent programs and instances that are not scheduled anymore.
     allocations = allocation.persistent_vms | allocation.instances
     # Make a copy since the pool is modified
-    for execution in list(pool.executions.values()):
+    for execution in list(pool.get_persistent_executions()):
         if execution.vm_hash not in allocations and execution.is_running:
             vm_type = "instance" if execution.is_instance else "persistent program"
             logger.info("Stopping %s %s", vm_type, execution.vm_hash)
             await pool.stop_vm(execution.vm_hash)
+            pool.forget_vm(execution.vm_hash)
 
     # Second start persistent VMs and instances sequentially to limit resource usage.
 

From 21bdbbba29ae0872c9df7dff74fc67447bda0d7c Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 14 Dec 2023 21:27:45 +0100
Subject: [PATCH 590/990] Fix: Solved Debian issue with event loop.

---
 src/aleph/vm/orchestrator/metrics.py | 10 ----------
 src/aleph/vm/pool.py                 |  7 ++++---
 2 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index 3a695e9e0..32b340830 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -104,16 +104,6 @@ async def delete_record(execution_uuid: str):
         session.close()
 
 
-async def delete_all_records():
-    """Delete all the resource usage in database"""
-    session = Session()  # undefined name 'Session'
-    try:
-        session.query(ExecutionRecord).delete()
-        session.commit()
-    finally:
-        session.close()
-
-
 async def get_execution_records() -> Iterable[ExecutionRecord]:
     """Get the execution records from the database."""
     session = Session()  # undefined name 'Session'
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 1f47b8c34..fb8a77a97 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -11,7 +11,7 @@
 from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
 from aleph.vm.network.hostnetwork import Network, make_ipv6_allocator
-from aleph.vm.orchestrator.metrics import delete_all_records, get_execution_records
+from aleph.vm.orchestrator.metrics import get_execution_records
 from aleph.vm.systemd import SystemDManager
 from aleph.vm.utils import get_message_executable_content
 from aleph.vm.vm_type import VmType
@@ -61,9 +61,10 @@ def __init__(self):
         self.snapshot_manager = SnapshotManager()
         logger.debug("Initializing SnapshotManager ...")
         self.snapshot_manager.run_snapshots()
+
         logger.debug("Loading existing executions ...")
-        # asyncio.run(delete_all_records())
-        asyncio.run(self._load_persistent_executions())
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(self._load_persistent_executions())
 
     def setup(self) -> None:
         """Set up the VM pool and the network."""

From f9b9a9bdfd813a83b9fb113d703c1a4e82e3cf1a Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Fri, 15 Dec 2023 17:59:05 +0100
Subject: [PATCH 591/990] Fix: Removed unneeded download parameter.

---
 src/aleph/vm/models.py | 5 ++---
 src/aleph/vm/pool.py   | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index c05f96d2a..0c45e5155 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -148,7 +148,7 @@ def to_dict(self) -> dict:
     def to_json(self, indent: Optional[int] = None) -> str:
         return dumps_for_json(self.to_dict(), indent=indent)
 
-    async def prepare(self, download: bool = True):
+    async def prepare(self):
         """Download VM required files"""
         async with self.preparation_pending_lock:
             if self.resources:
@@ -168,8 +168,7 @@ async def prepare(self, download: bool = True):
             if not resources:
                 msg = "Unknown executable message type"
                 raise ValueError(msg, repr(self.message))
-            if download:
-                await resources.download_all()
+            await resources.download_all()
             self.times.prepared_at = datetime.now(tz=timezone.utc)
             self.resources = resources
 
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index fb8a77a97..9b6badf65 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -220,7 +220,7 @@ async def _load_persistent_executions(self):
             )
             if execution.is_running:
                 # TODO: Improve the way that we re-create running execution
-                await execution.prepare(download=False)
+                await execution.prepare()
                 if self.network:
                     vm_type = VmType.from_message_content(execution.message)
                     tap_interface = await self.network.prepare_tap(vm_id, execution.vm_hash, vm_type)

From ddc1aada31668a9fd72020ccd2ef3e40e58221db Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 8 Dec 2023 12:35:48 +0100
Subject: [PATCH 592/990] Fix: Diagnostic could be improved

Solution:
- Add checks on the host connectivity, to help with nodes without IPv6 connectivity
- Add links to troubleshooting docs
- Move IPv6 Egress in details section, it was confusing to have it on the main page.
---
 src/aleph/vm/conf.py                          |   5 +
 src/aleph/vm/orchestrator/supervisor.py       |   2 +
 src/aleph/vm/orchestrator/views/__init__.py   |  27 +++
 .../vm/orchestrator/views/host_status.py      |  91 +++++++++
 .../vm/orchestrator/views/static/helpers.js   |  34 +++-
 .../vm/orchestrator/views/static/main.css     |  17 +-
 .../orchestrator/views/templates/index.html   | 172 +++++++++++++-----
 7 files changed, 298 insertions(+), 50 deletions(-)
 create mode 100644 src/aleph/vm/orchestrator/views/host_status.py

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 640d765e3..940f5b17c 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -127,6 +127,11 @@ class Settings(BaseSettings):
     WATCH_FOR_UPDATES = True
 
     API_SERVER = "https://official.aleph.cloud"
+    # Connect to the Quad9 VPN provider using their IPv4 and IPv6 addresses.
+    CONNECTIVITY_IPV4_URL = "https://9.9.9.9/"
+    CONNECTIVITY_IPV6_URL = "https://[2620:fe::fe]/"
+    CONNECTIVITY_DNS_HOSTNAME = "example.org"
+
     USE_JAILER = True
     # System logs make boot ~2x slower
     PRINT_SYSTEM_LOGS = False
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index ed94ff150..5297c9199 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -28,6 +28,7 @@
     run_code_from_hostname,
     run_code_from_path,
     status_check_fastapi,
+    status_check_host,
     status_check_version,
     status_public_config,
     update_allocations,
@@ -90,6 +91,7 @@ async def allow_cors_on_endpoint(request: web.Request):
             allow_cors_on_endpoint,
         ),
         web.get("/status/check/fastapi", status_check_fastapi),
+        web.get("/status/check/host", status_check_host),
         web.get("/status/check/version", status_check_version),
         web.get("/status/config", status_public_config),
         web.static("/static", Path(__file__).parent / "views/static"),
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 106aa27e8..11daae50f 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -26,6 +26,14 @@
 from aleph.vm.orchestrator.pubsub import PubSub
 from aleph.vm.orchestrator.resources import Allocation
 from aleph.vm.orchestrator.run import run_code_on_request, start_persistent_vm
+from aleph.vm.orchestrator.views.host_status import (
+    check_dns_ipv4,
+    check_dns_ipv6,
+    check_domain_resolution_ipv4,
+    check_domain_resolution_ipv6,
+    check_host_egress_ipv4,
+    check_host_egress_ipv6,
+)
 from aleph.vm.pool import VmPool
 from aleph.vm.utils import (
     HostNotFoundError,
@@ -169,6 +177,25 @@ async def status_check_fastapi(request: web.Request):
         return web.json_response(result, status=200 if all(result.values()) else 503)
 
 
+async def status_check_host(request: web.Request):
+    """Check that the platform is supported and configured correctly"""
+
+    result = {
+        "ipv4": {
+            "egress": await check_host_egress_ipv4(),
+            "dns": await check_dns_ipv4(),
+            "domain": await check_domain_resolution_ipv4(),
+        },
+        "ipv6": {
+            "egress": await check_host_egress_ipv6(),
+            "dns": await check_dns_ipv6(),
+            "domain": await check_domain_resolution_ipv6(),
+        },
+    }
+    result_status = 200 if all(result["ipv4"].values()) and all(result["ipv6"].values()) else 503
+    return web.json_response(result, status=result_status)
+
+
 async def status_check_version(request: web.Request):
     """Check if the software is running a version equal or newer than the given one"""
     reference_str: Optional[str] = request.query.get("reference")
diff --git a/src/aleph/vm/orchestrator/views/host_status.py b/src/aleph/vm/orchestrator/views/host_status.py
new file mode 100644
index 000000000..52b4899e1
--- /dev/null
+++ b/src/aleph/vm/orchestrator/views/host_status.py
@@ -0,0 +1,91 @@
+import socket
+from typing import Literal, Union
+
+import aiohttp
+
+from aleph.vm.conf import settings
+
+
+async def check_ip_connectivity(url: str, socket_family: socket.AddressFamily = socket.AF_UNSPEC) -> bool:
+    timeout = aiohttp.ClientTimeout(total=5)
+    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
+        async with session.get(url) as resp:
+            # We expect the Quad9 endpoints to return a 404 error, but other endpoints may return a 200
+            if resp.status not in (200, 404):
+                resp.raise_for_status()
+            return True
+
+
+async def check_host_egress_ipv4() -> bool:
+    """Check if the host has IPv4 connectivity."""
+    try:
+        return await check_ip_connectivity(settings.CONNECTIVITY_IPV4_URL)
+    except TimeoutError as exc:
+        print(f"IPv4 connectivity test failed: {exc}")
+        return False
+
+
+async def check_host_egress_ipv6() -> bool:
+    """Check if the host has IPv6 connectivity."""
+    try:
+        return await check_ip_connectivity(settings.CONNECTIVITY_IPV6_URL)
+    except TimeoutError as exc:
+        print(f"IPv6 connectivity test failed: {exc}")
+        return False
+
+
+async def resolve_dns(hostname: str) -> dict:
+    """Resolve a hostname to an IP address."""
+    info_inet, info_inet6 = socket.getaddrinfo(hostname, 80, proto=socket.IPPROTO_TCP)
+    ipv4 = info_inet[4][0]
+    ipv6 = info_inet6[4][0]
+    return {
+        "ipv4": ipv4,
+        "ipv6": ipv6,
+    }
+
+
+async def check_dns_ipv4() -> bool:
+    """Check if DNS resolution is working via IPv4."""
+    resolution = await resolve_dns(settings.CONNECTIVITY_DNS_HOSTNAME)
+    ipv4 = resolution["ipv4"]
+    return bool(ipv4)
+
+
+async def check_dns_ipv6() -> bool:
+    """Check if DNS resolution is working via IPv6."""
+    resolution = await resolve_dns(settings.CONNECTIVITY_DNS_HOSTNAME)
+    ipv6 = resolution["ipv6"]
+    return bool(ipv6)
+
+
+async def check_domain_resolution_ipv4() -> bool:
+    """Check if the host's hostname resolves to an IPv4 address."""
+    resolution = await resolve_dns(settings.DOMAIN_NAME)
+    ipv4 = resolution["ipv4"]
+    return bool(ipv4)
+
+
+async def check_domain_resolution_ipv6() -> bool:
+    """Check if the host's hostname resolves to an IPv6 address."""
+    resolution = await resolve_dns(settings.DOMAIN_NAME)
+    ipv6 = resolution["ipv6"]
+    return False
+
+
+async def check_domain_ipv4() -> bool:
+    """Check if the host's hostname is accessible via IPv4."""
+    try:
+        return await check_ip_connectivity(settings.DOMAIN_NAME, socket.AF_INET)
+    except TimeoutError as exc:
+        print(f"IPv4 connectivity test failed: {exc}")
+        return False
+
+
+async def check_domain_ipv6() -> bool:
+    """Check if the host's hostname is accessible via IPv6."""
+    try:
+        return await check_ip_connectivity(settings.DOMAIN_NAME, socket.AF_INET6)
+    except TimeoutError as exc:
+        print(f"IPv6 connectivity test failed: {exc}")
+        return False
diff --git a/src/aleph/vm/orchestrator/views/static/helpers.js b/src/aleph/vm/orchestrator/views/static/helpers.js
index 8335e2faf..46d12e4b6 100644
--- a/src/aleph/vm/orchestrator/views/static/helpers.js
+++ b/src/aleph/vm/orchestrator/views/static/helpers.js
@@ -1,4 +1,4 @@
-async function fetchApiStatus () {
+async function fetchFastapiCheckStatus () {
     const q = await fetch('/status/check/fastapi');
     let res = {
         status: q.status,
@@ -12,8 +12,10 @@ async function fetchApiStatus () {
             case 503:
                 res.status = " is not working properly ❌";
                 res.details = await q.json();
+                break;
             case 500:
                 res.status = " ❌ Failed";
+                break;
             default:
                 res.status = q.status;
         }
@@ -22,6 +24,36 @@ async function fetchApiStatus () {
     return res;
 }
 
+async function fetchHostCheckStatus () {
+    const q = await fetch('/status/check/host');
+    let res = {
+        status: q.status,
+        details: []
+    }
+    if(q.ok){
+        res.status = " is working properly ✅";
+    }
+    else {
+        switch(Number(q.status)){
+            case 503:
+                res.status = " is not working properly ❌";
+                res.details = await q.json();
+                break;
+            case 500:
+                res.status = " ❌ Failed";
+                break;
+            default:
+                res.status = q.status;
+        }
+    }
+
+    return res;
+}
+
+function objectToString (obj) {
+    return Object.entries(obj).reduce((acc, [k, v]) => acc + `
  • ${k}: ${v}
    • \n`, '');
+}
+
 const buildQueryParams = (params) => Object.entries(params).reduce((acc, [k, v]) => acc + `${k}=${v}&`, '?').slice(0, -1);
 
 const isLatestRelease = async () => {
diff --git a/src/aleph/vm/orchestrator/views/static/main.css b/src/aleph/vm/orchestrator/views/static/main.css
index 1c14ddd63..bf2cbbf85 100644
--- a/src/aleph/vm/orchestrator/views/static/main.css
+++ b/src/aleph/vm/orchestrator/views/static/main.css
@@ -1,11 +1,14 @@
 body {
     font-family: IBM Plex Regular, monospace;
+    white-space: normal;
     margin: auto;
+    max-width: 800px;
 }
 
 main {
     width: 90vw;
     margin: 2vh auto;
+    max-width: 800px;
 }
 
 progress {
@@ -36,29 +39,29 @@ progress {
 
 @keyframes move {
     0% {
-        height: 20px;
+        height: 10px;
     }
 
     50% {
-        height: 10px;
+        height: 5px;
     }
 
     100% {
-        height: 20px;
+        height: 10px;
     }
 }
 
 @keyframes move2 {
     0% {
-        height: 10px;
+        height: 5px;
     }
 
     50% {
-        height: 20px;
+        height: 10px;
     }
 
     100% {
-        height: 10px;
+        height: 5px;
     }
 }
 
@@ -97,4 +100,4 @@ progress {
 footer{
     font-size: 70%;
     opacity: .75;
-}
\ No newline at end of file
+}
diff --git a/src/aleph/vm/orchestrator/views/templates/index.html b/src/aleph/vm/orchestrator/views/templates/index.html
index 6e1cc5a0e..d7b449f21 100644
--- a/src/aleph/vm/orchestrator/views/templates/index.html
+++ b/src/aleph/vm/orchestrator/views/templates/index.html
@@ -16,10 +16,10 @@ 
    Aleph.im Compute Node
    
                 This is an Aleph.im compute resource node.
             
    
             
    
-                It executes user programs stored on the Aleph network in Virtual Machines.
+                It executes user programs stored on the aleph.im network in Virtual Machines.
             
    
             
    
-                See the repository for more info.
+                See the source code repository for more info.
             
    
 
         
    
@@ -44,34 +44,109 @@ 
    Multiaddr
    
         
    
         
    Diagnostic
    
 
-        
    
-            Virtualization 
-            
-                ... 
-                
-                    
-                    
-                    
+        
    
+            
    Virtualization
    
+            
    
+                Virtualization
+                
+                ...
+                    
+                        
+                        
+                        
+                    
                 
-            
-        
    
-
-        
    
-        
    
-            Diagnostics checks |
-            Open diagnostic VM
-        
    
-        
    
-            Egress IPv6
-            
-                is ...
-                
-                    
-                    
-                    
+            
    
+            
    
+                
      
+            
      
+            
+        
      
+
+        
      
+            
      Host connectivity
      
+            
      
+                Host
+                
+                ...
+                    
+                        
+                        
+                        
+                    
                 
-            
-        
      
+            
      
+            
      
+                
      IPv4
      
+                
        
+                
        IPv6
        
+                
          
+            
          
+            
+        
          
+
+        
          
+            ℹ️ More information
+
+            
          
+                
          Latest metrics
          
+                
          
+                    The aleph.im network measures the performance of all nodes in the network. New metrics are published
+                    every 10 minutes.
+                
          
+                
          
+                    🔍 Browse the metrics in the explorer
+                
          
+                
            
+            
            
+
+            
            VM Egress IPv6
            
+            
            
+                VM Egress IPv6 is a test to check if virtual machines are able to connect to the IPv6 internet.
+                Enabling VM IPv6 Egress requires a specific configuration that is not applied automatically. It is not yet
+                required to run virtual machines.
+            
            
+            
            
+                
            
+                    VM Egress IPv6
+                    
+                        is ...
+                        
+                            
+                            
+                            
+                        
+                    
+                
            
+            
            
+            
            APIs
            
+            
            
+                Host status check API: /status/check/host
+            
            
+            
            
+
+                Virtualization check API: /status/check/fastapi
+            
            
+            
            
+
+                VM Egress IPv6: 
            
+                /vm/$check_fastapi_vm_id/ip/6
+            
            
+        
            
+
         
            
 
         
            
@@ -80,7 +155,7 @@ 
            Version
            
                 Running version $version.
             
            
 
-            
            
+            
            
                 
                 
                 
            
 
             
            
                 
@@ -121,13 +196,26 @@ 
            Version
            
 
         (async () => {
             try {
-                const { status, details } = await fetchApiStatus();
-                document.getElementById('check').innerHTML = status;
-                const _checksDiv = document.getElementById("checks");
-                if(details.length > 0){
-                    const detailsDiv = document.createElement('div');
-                    detailsDiv.innerHTML = details;
-                    _checksDiv.appendChild(detailsDiv);
+                const { status, details } = await fetchFastapiCheckStatus();
+                document.getElementById('virtualization-check').innerHTML = status;
+                if(Object.keys(details).length > 0){
+                    const detailsDiv = document.querySelector("#virtualization-checks .details ul");
+                    detailsDiv.innerHTML = objectToString(details);
+                    document.querySelector("#virtualization-checks .help").style.display = "block";
+                }
+            } catch (err) {
+                console.error('Could not fetch api status', err);
+            }
+        })();
+
+        (async () => {
+            try {
+                const { status, details } = await fetchHostCheckStatus();
+                document.getElementById('host-check').innerHTML = status;
+                if(Object.keys(details).length > 0){
+                    document.querySelector("#host-checks .details ul.ipv4").innerHTML = objectToString(details["ipv4"]);
+                    document.querySelector("#host-checks .details ul.ipv6").innerHTML = objectToString(details["ipv6"]);
+                    document.querySelector("#host-checks .help").style.display = "block";
                 }
             } catch (err) {
                 console.error('Could not fetch api status', err);
@@ -238,21 +326,21 @@ 
            Version
            
         try{
             const response = await fetch('/vm/$check_fastapi_vm_id/ip/6');
             if (response.ok) {
-                document.getElementById("check_ipv6").innerHTML = "is working ✔️";
+                document.getElementById("ipv6-egress-check").innerHTML = "is working ✔️";
             }
             else if (response.status === 503) {
-                document.getElementById("check_ipv6").innerHTML = "fails to be tested ❌ ";
+                document.getElementById("ipv6-egress-check").innerHTML = "fails to be tested ❌ ";
             }
             else if (response.status === 500) {
-                document.getElementById("check_ipv6").innerHTML = "is not available ⛌";
+                document.getElementById("ipv6-egress-check").innerHTML = "is not yet available ⛌";
             }
             else {
-                document.getElementById("check_ipv6").innerText = response.status;
+                document.getElementById("ipv6-egress-check").innerText = response.status;
             }
         }
         catch(err){
             console.error(err);
-            document.getElementById("check_ipv6").innerHTML = "fails to be tested ❌ ";
+            document.getElementById("ipv6-egress-check").innerHTML = "fails to be tested ❌ ";
         }
     })();
 

From dd6a7caf10e6d6ef99e26a50bd4c08318241d81d Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Dec 2023 10:15:30 +0100
Subject: [PATCH 593/990] fixup! Fix: Diagnostic could be improved

---
 .../vm/orchestrator/views/host_status.py      | 68 +++++++++----------
 1 file changed, 31 insertions(+), 37 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/host_status.py b/src/aleph/vm/orchestrator/views/host_status.py
index 52b4899e1..67e6654a8 100644
--- a/src/aleph/vm/orchestrator/views/host_status.py
+++ b/src/aleph/vm/orchestrator/views/host_status.py
@@ -1,91 +1,85 @@
+import logging
 import socket
-from typing import Literal, Union
+from typing import Tuple, Callable
 
 import aiohttp
 
 from aleph.vm.conf import settings
 
 
+logger = logging.getLogger(__name__)
+
+
+def return_false_on_timeout(func):
+    async def wrapper(*args, **kwargs):
+        try:
+            return await func(*args, **kwargs)
+        except TimeoutError:
+            logger.warning(f"Timeout while checking {func.__name__}")
+            return False
+    return wrapper
+
+
 async def check_ip_connectivity(url: str, socket_family: socket.AddressFamily = socket.AF_UNSPEC) -> bool:
     timeout = aiohttp.ClientTimeout(total=5)
     async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
-        async with session.get(url) as resp:
+        async with session.get(url, socket_family=socket_family) as resp:
             # We expect the Quad9 endpoints to return a 404 error, but other endpoints may return a 200
             if resp.status not in (200, 404):
                 resp.raise_for_status()
             return True
 
 
+@return_false_on_timeout
 async def check_host_egress_ipv4() -> bool:
     """Check if the host has IPv4 connectivity."""
-    try:
-        return await check_ip_connectivity(settings.CONNECTIVITY_IPV4_URL)
-    except TimeoutError as exc:
-        print(f"IPv4 connectivity test failed: {exc}")
-        return False
+    return await check_ip_connectivity(settings.CONNECTIVITY_IPV4_URL)
 
 
+@return_false_on_timeout
 async def check_host_egress_ipv6() -> bool:
     """Check if the host has IPv6 connectivity."""
-    try:
-        return await check_ip_connectivity(settings.CONNECTIVITY_IPV6_URL)
-    except TimeoutError as exc:
-        print(f"IPv6 connectivity test failed: {exc}")
-        return False
+    return await check_ip_connectivity(settings.CONNECTIVITY_IPV6_URL)
 
 
-async def resolve_dns(hostname: str) -> dict:
-    """Resolve a hostname to an IP address."""
+async def resolve_dns(hostname: str) -> Tuple[str, str]:
     info_inet, info_inet6 = socket.getaddrinfo(hostname, 80, proto=socket.IPPROTO_TCP)
     ipv4 = info_inet[4][0]
     ipv6 = info_inet6[4][0]
-    return {
-        "ipv4": ipv4,
-        "ipv6": ipv6,
-    }
+    return ipv4, ipv6
 
 
 async def check_dns_ipv4() -> bool:
     """Check if DNS resolution is working via IPv4."""
-    resolution = await resolve_dns(settings.CONNECTIVITY_DNS_HOSTNAME)
-    ipv4 = resolution["ipv4"]
+    ipv4, _ = await resolve_dns(settings.CONNECTIVITY_DNS_HOSTNAME)
     return bool(ipv4)
 
 
 async def check_dns_ipv6() -> bool:
     """Check if DNS resolution is working via IPv6."""
-    resolution = await resolve_dns(settings.CONNECTIVITY_DNS_HOSTNAME)
-    ipv6 = resolution["ipv6"]
+    _, ipv6 = await resolve_dns(settings.CONNECTIVITY_DNS_HOSTNAME)
     return bool(ipv6)
 
 
 async def check_domain_resolution_ipv4() -> bool:
     """Check if the host's hostname resolves to an IPv4 address."""
-    resolution = await resolve_dns(settings.DOMAIN_NAME)
-    ipv4 = resolution["ipv4"]
+    ipv4, _ = await resolve_dns(settings.DOMAIN_NAME)
     return bool(ipv4)
 
 
 async def check_domain_resolution_ipv6() -> bool:
     """Check if the host's hostname resolves to an IPv6 address."""
-    resolution = await resolve_dns(settings.DOMAIN_NAME)
-    ipv6 = resolution["ipv6"]
-    return False
+    _, ipv6 = await resolve_dns(settings.DOMAIN_NAME)
+    return bool(ipv6)
 
 
+@return_false_on_timeout
 async def check_domain_ipv4() -> bool:
     """Check if the host's hostname is accessible via IPv4."""
-    try:
-        return await check_ip_connectivity(settings.DOMAIN_NAME, socket.AF_INET)
-    except TimeoutError as exc:
-        print(f"IPv4 connectivity test failed: {exc}")
-        return False
+    return await check_ip_connectivity(settings.DOMAIN_NAME, socket.AF_INET)
 
 
+@return_false_on_timeout
 async def check_domain_ipv6() -> bool:
     """Check if the host's hostname is accessible via IPv6."""
-    try:
-        return await check_ip_connectivity(settings.DOMAIN_NAME, socket.AF_INET6)
-    except TimeoutError as exc:
-        print(f"IPv6 connectivity test failed: {exc}")
-        return False
+    return await check_ip_connectivity(settings.DOMAIN_NAME, socket.AF_INET6)

From 33f95dd130607b38753b810457c49e2d1d750a57 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Dec 2023 10:20:23 +0100
Subject: [PATCH 594/990] fixup! Fix: Diagnostic could be improved

---
 src/aleph/vm/orchestrator/views/host_status.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/host_status.py b/src/aleph/vm/orchestrator/views/host_status.py
index 67e6654a8..7bea32604 100644
--- a/src/aleph/vm/orchestrator/views/host_status.py
+++ b/src/aleph/vm/orchestrator/views/host_status.py
@@ -1,22 +1,22 @@
 import logging
 import socket
-from typing import Tuple, Callable
+from typing import Any, Awaitable, Callable, Tuple
 
 import aiohttp
 
 from aleph.vm.conf import settings
 
-
 logger = logging.getLogger(__name__)
 
 
-def return_false_on_timeout(func):
-    async def wrapper(*args, **kwargs):
+def return_false_on_timeout(func: Callable[..., Awaitable[Any]]) -> Callable[..., Awaitable[bool]]:
+    async def wrapper(*args: Any, **kwargs: Any) -> bool:
         try:
             return await func(*args, **kwargs)
         except TimeoutError:
             logger.warning(f"Timeout while checking {func.__name__}")
             return False
+
     return wrapper
 
 

From 68e828c0824dfb820cce0382594c97f4463d115a Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 15 Dec 2023 15:55:53 +0100
Subject: [PATCH 595/990] WIP: Start the skeleton to monitor payments via
 balances and streams

---
 pyproject.toml                          |  2 +-
 src/aleph/vm/conf.py                    | 11 ++++++
 src/aleph/vm/orchestrator/payment.py    | 30 ++++++++++++++
 src/aleph/vm/orchestrator/supervisor.py |  9 ++++-
 src/aleph/vm/orchestrator/tasks.py      | 52 ++++++++++++++++++++++++-
 src/aleph/vm/pool.py                    | 18 ++++++++-
 6 files changed, 117 insertions(+), 5 deletions(-)
 create mode 100644 src/aleph/vm/orchestrator/payment.py

diff --git a/pyproject.toml b/pyproject.toml
index ca9748828..3eb13cae8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,7 +30,7 @@ dependencies = [
   "alembic==1.7.6",
   "setproctitle==1.3.3",
   "pyyaml==6.0.1",
-  "aleph-message==0.4.1",
+  "aleph-message @ git+https://github.com/aleph-im/aleph-message.git@mhh-pay-as-you-go-v1#egg=aleph-message",
   "jwskate==0.8.0",
   "eth-account==0.9.0",
   "sentry-sdk==1.31.0",
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 940f5b17c..131ca5e80 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -3,12 +3,14 @@
 import os
 import re
 from collections.abc import Iterable
+from decimal import Decimal
 from enum import Enum
 from os.path import abspath, exists, isdir, isfile, join
 from pathlib import Path
 from subprocess import CalledProcessError, check_output
 from typing import Any, Literal, NewType, Optional, Union
 
+from aleph_message.models import ItemHash
 from pydantic import BaseSettings, Field
 from pydantic.env_settings import DotenvType, env_file_sentinel
 from pydantic.typing import StrPath
@@ -206,6 +208,15 @@ class Settings(BaseSettings):
     MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000  # 10 MB
     MAX_DATA_ARCHIVE_SIZE = 10_000_000  # 10 MB
 
+    PAYMENT_MONITOR_INTERVAL: Decimal = Field(
+        default=60.0,
+        description="Interval in seconds between payment checks",
+    )
+    PAYMENT_RECEIVER_ADDRESS: str = Field(
+        description="Address of the account receiving payments",
+    )
+    PAYMENT_PRICING_AGGREGATE: ItemHash  # TODO: Missing
+
     SNAPSHOT_FREQUENCY: int = Field(
         default=60,
         description="Snapshot frequency interval in minutes. It will create a VM snapshot every X minutes.",
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
new file mode 100644
index 000000000..1b6f5b640
--- /dev/null
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -0,0 +1,30 @@
+from decimal import Decimal
+from typing import Iterable
+
+from aleph.vm.models import VmExecution
+
+
+async def get_balance(address: str) -> Decimal:
+    """Get the balance of the user from the PyAleph."""
+    # See https://github.com/aleph-im/pyaleph/blob/master/src/aleph/web/controllers/routes.py#L62
+    # "/api/v0/addresses/{address}/balance"
+    # TODO
+    raise NotImplementedError()
+
+
+def get_stream(sender, receiver, chain):
+    # See https://community.aleph.im/t/pay-as-you-go-using-superfluid/98/11
+    # TODO
+    raise NotImplementedError()
+
+
+async def get_required_balance(executions: Iterable[VmExecution]) -> Decimal:
+    """Get the balance required for the resources of the user from the messages and the pricing aggregate."""
+    # TODO
+    raise NotImplementedError()
+
+
+async def get_required_flow(executions: Iterable[VmExecution]) -> Decimal:
+    """Compute the flow required for the resources of the user from the messages and the pricing aggregate"""
+    # TODO
+    raise NotImplementedError()
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 5297c9199..d0a8be8e3 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -19,7 +19,12 @@
 
 from .metrics import create_tables, setup_engine
 from .resources import about_system_usage
-from .tasks import start_watch_for_messages_task, stop_watch_for_messages_task
+from .tasks import (
+    start_payment_monitoring_task,
+    start_watch_for_messages_task,
+    stop_balances_monitoring_task,
+    stop_watch_for_messages_task,
+)
 from .views import (
     about_config,
     about_execution_records,
@@ -130,7 +135,9 @@ def run():
     try:
         if settings.WATCH_FOR_MESSAGES:
             app.on_startup.append(start_watch_for_messages_task)
+            app.on_startup.append(start_payment_monitoring_task)
             app.on_cleanup.append(stop_watch_for_messages_task)
+            app.on_cleanup.append(stop_balances_monitoring_task)
             app.on_cleanup.append(stop_all_vms)
 
         web.run_app(app, host=settings.SUPERVISOR_HOST, port=settings.SUPERVISOR_PORT)
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 2cdeeb284..82fa3db5f 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -9,7 +9,13 @@
 import aiohttp
 import pydantic
 from aiohttp import web
-from aleph_message.models import AlephMessage, ItemHash, ProgramMessage, parse_message
+from aleph_message.models import (
+    AlephMessage,
+    ItemHash,
+    PaymentType,
+    ProgramMessage,
+    parse_message,
+)
 from yarl import URL
 
 from aleph.vm.conf import settings
@@ -17,6 +23,7 @@
 from aleph.vm.utils import create_task_log_exceptions
 
 from .messages import load_updated_message
+from .payment import get_balance, get_required_balance, get_required_flow, get_stream
 from .pubsub import PubSub
 from .reactor import Reactor
 
@@ -127,3 +134,46 @@ async def stop_watch_for_messages_task(app: web.Application):
         await app["messages_listener"]
     except asyncio.CancelledError:
         logger.debug("Task messages_listener is cancelled now")
+
+
+async def monitor_payments(app: web.Application):
+    logger.debug("Monitoring balances")
+    pool: VmPool = app["vm_pool"]
+    while True:
+        await asyncio.sleep(settings.PAYMENT_MONITOR_INTERVAL)
+
+        # Check if the balance held in the wallet is sufficient holder tier resources
+        for sender, executions in pool.get_executions_by_sender(payment=PaymentType.hold):
+            balance = await get_balance(sender)
+
+            # Stop executions until the required balance is reached
+            required_balance = await get_required_balance(executions)
+            while balance < required_balance:
+                last_execution = executions.pop(-1)
+                logger.debug(f"Stopping {last_execution} due to insufficient stream")
+                await last_execution.stop()
+                required_balance = await get_required_balance(executions)
+
+        # Check if the balance held in the wallet is sufficient stream tier resources
+        for sender, chain, executions in pool.get_executions_by_sender(payment=PaymentType.stream):
+            stream = await get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
+            required_stream = await get_required_flow(executions)
+
+            # Stop executions until the required stream is reached
+            while stream < required_stream:
+                last_execution = executions.pop(-1)
+                logger.debug(f"Stopping {last_execution} due to insufficient stream")
+                await last_execution.stop()
+                required_stream = await get_required_flow(executions)
+
+
+async def start_payment_monitoring_task(app: web.Application):
+    app["payments_monitor"] = create_task_log_exceptions(monitor_payments(app))
+
+
+async def stop_balances_monitoring_task(app: web.Application):
+    app["payments_monitor"].cancel()
+    try:
+        await app["payments_monitor"]
+    except asyncio.CancelledError:
+        logger.debug("Task payments_monitor is cancelled now")
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 9b6badf65..6f3341caf 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -3,10 +3,10 @@
 import logging
 from collections.abc import Iterable
 from datetime import datetime, timezone
-from typing import Optional
+from typing import Optional, Tuple
 
 from aleph_message.models import ExecutableMessage, ItemHash
-from aleph_message.models.execution.instance import InstanceContent
+from aleph_message.models.execution.base import PaymentType
 
 from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
@@ -264,3 +264,17 @@ def get_instance_executions(self) -> Iterable[VmExecution]:
             for _vm_hash, execution in self.executions.items()
             if execution.is_running and execution.is_instance
         )
+
+    def get_executions_by_sender(self, payment_type: PaymentType) -> Iterable[Tuple[str, list[VmExecution]]]:
+        """Return all executions of the given type, grouped by sender and by chain."""
+        executions_by_sender: dict[str, list[VmExecution]] = {}
+        for vm_hash, execution in self.executions.items():
+            if execution.is_stopping or execution.is_stopped:
+                # Ignore the execution that is stopping or not running anymore
+                continue
+            execution_payment_type = execution.message.payment.type if execution.message.payment else PaymentType.hold
+            if execution_payment_type == payment_type:
+                sender = execution.message.sender
+                chain = execution.message.chain
+                executions_by_sender.setdefault(sender, []).append(execution)
+        return executions_by_sender.items()

From f67812de3ae76a4b3749dc5b0f179d7f0923de6d Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 21 Dec 2023 21:28:10 +0100
Subject: [PATCH 596/990] Feature: Implemented skeleton methods using
 SuperFluid.

---
 pyproject.toml                       |   2 +
 src/aleph/vm/conf.py                 |   9 ++
 src/aleph/vm/constants.py            |   5 +
 src/aleph/vm/orchestrator/payment.py | 153 +++++++++++++++++++++++++--
 src/aleph/vm/utils.py                |  22 ++++
 5 files changed, 182 insertions(+), 9 deletions(-)
 create mode 100644 src/aleph/vm/constants.py

diff --git a/pyproject.toml b/pyproject.toml
index 3eb13cae8..52bc3a21b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,6 +45,8 @@ dependencies = [
   "qmp==0.0.1",
   "dbus-python==1.3.2",
   "systemd-python==235",
+  "systemd-python==235",
+  "superfluid @ git+https://github.com/Godspower-Eze/superfluid.py#egg=superfluid"
 ]
 
 [project.urls]
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 131ca5e80..37e2175ef 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -215,8 +215,17 @@ class Settings(BaseSettings):
     PAYMENT_RECEIVER_ADDRESS: str = Field(
         description="Address of the account receiving payments",
     )
+    PAYMENT_SUPER_TOKEN: str = Field(
+        default="0x1290248e01ed2f9f863a9752a8aad396ef3a1b00",
+        description="Address of the ALEPH SuperToken on SuperFluid",
+    )
     PAYMENT_PRICING_AGGREGATE: ItemHash  # TODO: Missing
 
+    PAYMENT_RPC_SERVER: Field(
+        default="https://api.avax-test.network/ext/bc/C/rpc",
+        description="Default to Avalanche Testnet RPC",
+    )
+
     SNAPSHOT_FREQUENCY: int = Field(
         default=60,
         description="Snapshot frequency interval in minutes. It will create a VM snapshot every X minutes.",
diff --git a/src/aleph/vm/constants.py b/src/aleph/vm/constants.py
new file mode 100644
index 000000000..519e7987e
--- /dev/null
+++ b/src/aleph/vm/constants.py
@@ -0,0 +1,5 @@
+KiB = 1024
+MiB = 1024 * 1024
+GiB = 1024 * 1024 * 1024
+Hour = 60 * 60
+Minute = 60
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 1b6f5b640..c9e10fad3 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -1,30 +1,165 @@
 from decimal import Decimal
 from typing import Iterable
 
+import aiohttp
+import math
+
+from aleph.vm.conf import settings
+from aleph.vm.constants import GiB, MiB, Hour
+from aleph.vm.controllers.firecracker.program import AlephProgramResources
 from aleph.vm.models import VmExecution
+from aleph.vm.utils import get_path_size
+
+from eth_typing import HexAddress, HexStr
+from eth_utils import to_hex, hexstr_if_str, is_address, from_wei
+from superfluid import CFA_V1, Web3FlowInfo
 
 
 async def get_balance(address: str) -> Decimal:
     """Get the balance of the user from the PyAleph."""
     # See https://github.com/aleph-im/pyaleph/blob/master/src/aleph/web/controllers/routes.py#L62
     # "/api/v0/addresses/{address}/balance"
-    # TODO
-    raise NotImplementedError()
+    async with aiohttp.ClientSession() as session:
+        url = f"{settings.API_SERVER}/api/v0/{address}/balance"
+        resp = await session.get(url)
+
+        if not resp.ok:
+            return Decimal(0)
 
+        resp_data = await resp.json()
+        return resp_data["balance"] if resp_data["balance"] else 0
 
-def get_stream(sender, receiver, chain):
+
+def get_stream(sender: str, receiver: str, chain) -> Decimal:
     # See https://community.aleph.im/t/pay-as-you-go-using-superfluid/98/11
-    # TODO
-    raise NotImplementedError()
+
+    # TODO: Convert chain str to ID
+    superfluid_instance = CFA_V1(settings.PAYMENT_RPC_SERVER, chain)
+
+    super_token: HexAddress = to_normalized_address(settings.PAYMENT_SUPER_TOKEN)
+    sender_address: HexAddress = to_normalized_address(sender)
+    receiver_address: HexAddress = to_normalized_address(receiver)
+
+    flow_data: Web3FlowInfo = superfluid_instance.get_flow(super_token, sender_address, receiver_address)
+    stream = from_wei(flow_data.flowRate, 'ether')
+    return Decimal(stream)
+
+
+def to_normalized_address(value: str) -> HexAddress:
+    """
+    Converts an address to its normalized hexadecimal representation.
+    """
+    try:
+        hex_address = hexstr_if_str(to_hex, value).lower()
+    except AttributeError:
+        raise TypeError(
+            "Value must be any string, instead got type {}".format(type(value))
+        )
+    if is_address(hex_address):
+        return HexAddress(HexStr(hex_address))
+    else:
+        raise ValueError(
+            "Unknown format {}, attempted to normalize to {}".format(value, hex_address)
+        )
 
 
 async def get_required_balance(executions: Iterable[VmExecution]) -> Decimal:
     """Get the balance required for the resources of the user from the messages and the pricing aggregate."""
-    # TODO
-    raise NotImplementedError()
+    balance = 0
+    for execution in executions:
+        balance += compute_execution_hold_cost(execution)
+
+    return Decimal(balance)
+
+
+def compute_execution_hold_cost(execution: VmExecution) -> Decimal:
+    compute_unit_cost = 200 if execution.persistent else 2000
+
+    compute_units_required = _get_nb_compute_units(execution)
+    compute_unit_multiplier = _get_compute_unit_multiplier(execution)
+
+    compute_unit_price = (
+            Decimal(compute_units_required) * compute_unit_multiplier * compute_unit_cost
+    )
+    price = compute_unit_price + _get_additional_storage_hold_price(execution)
+    return Decimal(price)
+
+
+def _get_additional_storage_hold_price(execution: VmExecution) -> Decimal:
+    nb_compute_units = execution.vm.hardware_resources.vcpus
+    free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
+
+    total_volume_size = _get_execution_storage_size(execution)
+    additional_storage = max(
+        total_volume_size - (free_storage_per_compute_unit * nb_compute_units), 0
+    )
+    price = Decimal(additional_storage) / 20 / MiB
+    return price
+
+
+def _get_nb_compute_units(execution: VmExecution) -> int:
+    cpu = execution.vm.hardware_resources.vcpus
+    memory = math.ceil(execution.vm.hardware_resources.memory / 2048)
+    nb_compute_units = cpu if cpu >= memory else memory
+    return nb_compute_units
+
+
+def _get_compute_unit_multiplier(execution: VmExecution) -> int:
+    compute_unit_multiplier = 1
+    if not execution.persistent and execution.message.environment.internet:
+        compute_unit_multiplier += 1
+    return compute_unit_multiplier
+
+
+def _get_execution_storage_size(execution: VmExecution) -> int:
+    size = 0
+
+    if execution.is_instance:
+        size += execution.message.rootfs.size_mib * MiB
+    elif execution.is_program:
+        if isinstance(execution.resources, AlephProgramResources):
+            size += get_path_size(execution.resources.code_path)
+            if execution.resources.data_path:
+                size += get_path_size(execution.resources.data_path)
+
+    for volume in execution.resources.volumes:
+        size += get_path_size(volume.path_on_host)
+
+    return size
 
 
 async def get_required_flow(executions: Iterable[VmExecution]) -> Decimal:
     """Compute the flow required for the resources of the user from the messages and the pricing aggregate"""
-    # TODO
-    raise NotImplementedError()
+    flow = 0
+    for execution in executions:
+        flow += compute_execution_flow_cost(execution)
+
+    return Decimal(flow)
+
+
+def compute_execution_flow_cost(execution: VmExecution) -> Decimal:
+    compute_unit_cost_hour = 0.011 if execution.persistent else 0.11  # TODO: Get from PAYG aggregate
+    compute_unit_cost_second = compute_unit_cost_hour * Hour
+
+    compute_units_required = _get_nb_compute_units(execution)
+    compute_unit_multiplier = _get_compute_unit_multiplier(execution)
+
+    compute_unit_price = (
+            Decimal(compute_units_required) * compute_unit_multiplier * compute_unit_cost_second
+    )
+    price = compute_unit_price + _get_additional_storage_flow_price(execution)
+    return Decimal(price)
+
+
+def _get_additional_storage_flow_price(execution: VmExecution) -> Decimal:
+    additional_storage_hour_price = 0.000000977  # TODO: Get from PAYG aggregate
+    additional_storage_second_price = Decimal(additional_storage_hour_price) / Hour
+    nb_compute_units = execution.vm.hardware_resources.vcpus
+    free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
+
+    total_volume_size = _get_execution_storage_size(execution)
+    additional_storage = max(
+        total_volume_size - (free_storage_per_compute_unit * nb_compute_units), 0
+    )
+    price = Decimal(additional_storage) / Decimal(additional_storage_second_price) / MiB
+    return price
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index 758deb169..7eb6fc7dd 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -8,6 +8,7 @@
 from collections.abc import Coroutine
 from dataclasses import asdict as dataclass_as_dict
 from dataclasses import is_dataclass
+from pathlib import Path
 from shutil import disk_usage
 from typing import Any, Dict, Optional
 
@@ -147,3 +148,24 @@ def check_disk_space(bytes_to_use: int) -> bool:
 
 class NotEnoughDiskSpaceError(OSError):
     pass
+
+
+def get_path_size(path: Path) -> int:
+    if path.is_dir():
+        return sum([f.stat().st_size for f in path.glob("**/*")])
+    elif path.is_block_device():
+        return get_block_device_size(str(path))
+    elif path.is_file():
+        return path.stat().st_size
+    else:
+        raise NotImplementedError
+
+
+def get_block_device_size(device: str) -> int:
+    output = subprocess.run(
+        ["lsblk", device, "--output", "SIZE", "--bytes", "--noheadings", "--nodeps"],
+        capture_output=True,
+        check=True,
+    )
+    size = int(output.stdout.decode())
+    return size

From 19922bf8fa6c2a6a1ab43bba98d4ca667cb82b5a Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 28 Dec 2023 20:04:54 +0100
Subject: [PATCH 597/990] Fix: Solved some issues, type checking and get stream
 flow config.

---
 src/aleph/vm/conf.py                      |  6 +--
 src/aleph/vm/controllers/configuration.py |  9 +++-
 src/aleph/vm/orchestrator/payment.py      | 53 ++++++++++-------------
 src/aleph/vm/orchestrator/tasks.py        | 45 ++++++++++---------
 src/aleph/vm/pool.py                      | 35 ++++++++++-----
 5 files changed, 84 insertions(+), 64 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 37e2175ef..d52c2bbee 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -208,7 +208,7 @@ class Settings(BaseSettings):
     MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000  # 10 MB
     MAX_DATA_ARCHIVE_SIZE = 10_000_000  # 10 MB
 
-    PAYMENT_MONITOR_INTERVAL: Decimal = Field(
+    PAYMENT_MONITOR_INTERVAL: float = Field(
         default=60.0,
         description="Interval in seconds between payment checks",
     )
@@ -219,9 +219,9 @@ class Settings(BaseSettings):
         default="0x1290248e01ed2f9f863a9752a8aad396ef3a1b00",
         description="Address of the ALEPH SuperToken on SuperFluid",
     )
-    PAYMENT_PRICING_AGGREGATE: ItemHash  # TODO: Missing
+    PAYMENT_PRICING_AGGREGATE: str = ""  # TODO: Missing
 
-    PAYMENT_RPC_SERVER: Field(
+    PAYMENT_RPC_SERVER: str = Field(
         default="https://api.avax-test.network/ext/bc/C/rpc",
         description="Default to Avalanche Testnet RPC",
     )
diff --git a/src/aleph/vm/controllers/configuration.py b/src/aleph/vm/controllers/configuration.py
index 2255089bc..be8d1986c 100644
--- a/src/aleph/vm/controllers/configuration.py
+++ b/src/aleph/vm/controllers/configuration.py
@@ -1,3 +1,4 @@
+import logging
 from enum import Enum
 from pathlib import Path
 from typing import Optional, Union
@@ -6,6 +7,8 @@
 
 from aleph.vm.conf import Settings, settings
 
+logger = logging.getLogger(__name__)
+
 
 class VMConfiguration(BaseModel):
     use_jailer: bool
@@ -42,6 +45,10 @@ def save_controller_configuration(vm_hash: str, configuration: Configuration) ->
     """Save VM configuration to be used by the controller service"""
     config_file_path = Path(f"{settings.EXECUTION_ROOT}/{vm_hash}-controller.json")
     with config_file_path.open("w") as controller_config_file:
-        controller_config_file.write(configuration.json(by_alias=True, exclude_none=True, indent=4))
+        controller_config_file.write(
+            configuration.json(
+                by_alias=True, exclude_none=True, indent=4, exclude={"settings": {"USE_DEVELOPER_SSH_KEYS"}}
+            )
+        )
     config_file_path.chmod(0o644)
     return config_file_path
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index c9e10fad3..8ad6d4a3c 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -1,18 +1,20 @@
+import logging
+import math
 from decimal import Decimal
 from typing import Iterable
 
 import aiohttp
-import math
+from eth_typing import HexAddress, HexStr
+from eth_utils import from_wei, hexstr_if_str, is_address, to_hex
+from superfluid import CFA_V1, Web3FlowInfo
 
 from aleph.vm.conf import settings
-from aleph.vm.constants import GiB, MiB, Hour
+from aleph.vm.constants import GiB, Hour, MiB
 from aleph.vm.controllers.firecracker.program import AlephProgramResources
 from aleph.vm.models import VmExecution
 from aleph.vm.utils import get_path_size
 
-from eth_typing import HexAddress, HexStr
-from eth_utils import to_hex, hexstr_if_str, is_address, from_wei
-from superfluid import CFA_V1, Web3FlowInfo
+logger = logging.getLogger(__name__)
 
 
 async def get_balance(address: str) -> Decimal:
@@ -34,14 +36,15 @@ def get_stream(sender: str, receiver: str, chain) -> Decimal:
     # See https://community.aleph.im/t/pay-as-you-go-using-superfluid/98/11
 
     # TODO: Convert chain str to ID
-    superfluid_instance = CFA_V1(settings.PAYMENT_RPC_SERVER, chain)
+    chain_id = 43113
+    superfluid_instance = CFA_V1(settings.PAYMENT_RPC_SERVER, chain_id)
 
     super_token: HexAddress = to_normalized_address(settings.PAYMENT_SUPER_TOKEN)
     sender_address: HexAddress = to_normalized_address(sender)
     receiver_address: HexAddress = to_normalized_address(receiver)
 
     flow_data: Web3FlowInfo = superfluid_instance.get_flow(super_token, sender_address, receiver_address)
-    stream = from_wei(flow_data.flowRate, 'ether')
+    stream = from_wei(flow_data["flowRate"], "ether")
     return Decimal(stream)
 
 
@@ -52,18 +55,14 @@ def to_normalized_address(value: str) -> HexAddress:
     try:
         hex_address = hexstr_if_str(to_hex, value).lower()
     except AttributeError:
-        raise TypeError(
-            "Value must be any string, instead got type {}".format(type(value))
-        )
+        raise TypeError("Value must be any string, instead got type {}".format(type(value)))
     if is_address(hex_address):
         return HexAddress(HexStr(hex_address))
     else:
-        raise ValueError(
-            "Unknown format {}, attempted to normalize to {}".format(value, hex_address)
-        )
+        raise ValueError("Unknown format {}, attempted to normalize to {}".format(value, hex_address))
 
 
-async def get_required_balance(executions: Iterable[VmExecution]) -> Decimal:
+def get_required_balance(executions: Iterable[VmExecution]) -> Decimal:
     """Get the balance required for the resources of the user from the messages and the pricing aggregate."""
     balance = 0
     for execution in executions:
@@ -78,9 +77,7 @@ def compute_execution_hold_cost(execution: VmExecution) -> Decimal:
     compute_units_required = _get_nb_compute_units(execution)
     compute_unit_multiplier = _get_compute_unit_multiplier(execution)
 
-    compute_unit_price = (
-            Decimal(compute_units_required) * compute_unit_multiplier * compute_unit_cost
-    )
+    compute_unit_price = Decimal(compute_units_required) * compute_unit_multiplier * compute_unit_cost
     price = compute_unit_price + _get_additional_storage_hold_price(execution)
     return Decimal(price)
 
@@ -90,9 +87,7 @@ def _get_additional_storage_hold_price(execution: VmExecution) -> Decimal:
     free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
 
     total_volume_size = _get_execution_storage_size(execution)
-    additional_storage = max(
-        total_volume_size - (free_storage_per_compute_unit * nb_compute_units), 0
-    )
+    additional_storage = max(total_volume_size - (free_storage_per_compute_unit * nb_compute_units), 0)
     price = Decimal(additional_storage) / 20 / MiB
     return price
 
@@ -128,7 +123,7 @@ def _get_execution_storage_size(execution: VmExecution) -> int:
     return size
 
 
-async def get_required_flow(executions: Iterable[VmExecution]) -> Decimal:
+def get_required_flow(executions: Iterable[VmExecution]) -> Decimal:
     """Compute the flow required for the resources of the user from the messages and the pricing aggregate"""
     flow = 0
     for execution in executions:
@@ -139,27 +134,27 @@ async def get_required_flow(executions: Iterable[VmExecution]) -> Decimal:
 
 def compute_execution_flow_cost(execution: VmExecution) -> Decimal:
     compute_unit_cost_hour = 0.011 if execution.persistent else 0.11  # TODO: Get from PAYG aggregate
-    compute_unit_cost_second = compute_unit_cost_hour * Hour
+    compute_unit_cost_second = compute_unit_cost_hour / Hour
 
     compute_units_required = _get_nb_compute_units(execution)
     compute_unit_multiplier = _get_compute_unit_multiplier(execution)
 
     compute_unit_price = (
-            Decimal(compute_units_required) * compute_unit_multiplier * compute_unit_cost_second
+        Decimal(compute_units_required) * Decimal(compute_unit_multiplier) * Decimal(compute_unit_cost_second)
     )
+
     price = compute_unit_price + _get_additional_storage_flow_price(execution)
+
     return Decimal(price)
 
 
 def _get_additional_storage_flow_price(execution: VmExecution) -> Decimal:
     additional_storage_hour_price = 0.000000977  # TODO: Get from PAYG aggregate
-    additional_storage_second_price = Decimal(additional_storage_hour_price) / Hour
+    additional_storage_second_price = additional_storage_hour_price / Hour
     nb_compute_units = execution.vm.hardware_resources.vcpus
     free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
 
     total_volume_size = _get_execution_storage_size(execution)
-    additional_storage = max(
-        total_volume_size - (free_storage_per_compute_unit * nb_compute_units), 0
-    )
-    price = Decimal(additional_storage) / Decimal(additional_storage_second_price) / MiB
-    return price
+    additional_storage = max(total_volume_size - (free_storage_per_compute_unit * nb_compute_units), 0)
+    price = additional_storage / additional_storage_second_price / MiB
+    return Decimal(price)
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 82fa3db5f..f151000cc 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -143,28 +143,33 @@ async def monitor_payments(app: web.Application):
         await asyncio.sleep(settings.PAYMENT_MONITOR_INTERVAL)
 
         # Check if the balance held in the wallet is sufficient holder tier resources
-        for sender, executions in pool.get_executions_by_sender(payment=PaymentType.hold):
-            balance = await get_balance(sender)
-
-            # Stop executions until the required balance is reached
-            required_balance = await get_required_balance(executions)
-            while balance < required_balance:
-                last_execution = executions.pop(-1)
-                logger.debug(f"Stopping {last_execution} due to insufficient stream")
-                await last_execution.stop()
-                required_balance = await get_required_balance(executions)
+        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.hold):
+            for chain, executions in chains.items():
+                balance = await get_balance(sender)
+
+                # Stop executions until the required balance is reached
+                required_balance = get_required_balance(executions)
+                while balance < required_balance:
+                    last_execution = executions.pop(-1)
+                    logger.debug(f"Stopping {last_execution} due to insufficient stream")
+                    await last_execution.stop()
+                    required_balance = get_required_balance(executions)
 
         # Check if the balance held in the wallet is sufficient stream tier resources
-        for sender, chain, executions in pool.get_executions_by_sender(payment=PaymentType.stream):
-            stream = await get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
-            required_stream = await get_required_flow(executions)
-
-            # Stop executions until the required stream is reached
-            while stream < required_stream:
-                last_execution = executions.pop(-1)
-                logger.debug(f"Stopping {last_execution} due to insufficient stream")
-                await last_execution.stop()
-                required_stream = await get_required_flow(executions)
+        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid):
+            for chain, executions in chains.items():
+                stream = get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
+                logger.debug(
+                    f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
+                )
+                required_stream = get_required_flow(executions)
+                logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
+                # Stop executions until the required stream is reached
+                while stream < required_stream:
+                    last_execution = executions.pop(-1)
+                    logger.debug(f"Stopping {last_execution} due to insufficient stream")
+                    await last_execution.stop()
+                    required_stream = get_required_flow(executions)
 
 
 async def start_payment_monitoring_task(app: web.Application):
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 6f3341caf..9cc468e4d 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -3,10 +3,15 @@
 import logging
 from collections.abc import Iterable
 from datetime import datetime, timezone
-from typing import Optional, Tuple
+from typing import Dict, Optional, Tuple
 
-from aleph_message.models import ExecutableMessage, ItemHash
-from aleph_message.models.execution.base import PaymentType
+from aleph_message.models import (
+    Chain,
+    ExecutableMessage,
+    ItemHash,
+    Payment,
+    PaymentType,
+)
 
 from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
@@ -265,16 +270,24 @@ def get_instance_executions(self) -> Iterable[VmExecution]:
             if execution.is_running and execution.is_instance
         )
 
-    def get_executions_by_sender(self, payment_type: PaymentType) -> Iterable[Tuple[str, list[VmExecution]]]:
+    def get_executions_by_sender(self, payment_type: PaymentType) -> Iterable[Tuple[str, Dict[str, list[VmExecution]]]]:
         """Return all executions of the given type, grouped by sender and by chain."""
-        executions_by_sender: dict[str, list[VmExecution]] = {}
+        executions_by_sender: Tuple[str, Dict[str, list[VmExecution]]] = {}
         for vm_hash, execution in self.executions.items():
-            if execution.is_stopping or execution.is_stopped:
+            if not execution.is_running:
                 # Ignore the execution that is stopping or not running anymore
                 continue
-            execution_payment_type = execution.message.payment.type if execution.message.payment else PaymentType.hold
-            if execution_payment_type == payment_type:
-                sender = execution.message.sender
-                chain = execution.message.chain
-                executions_by_sender.setdefault(sender, []).append(execution)
+            if execution.vm_hash == settings.CHECK_FASTAPI_VM_ID:
+                # Ignore Diagnostic VM execution
+                continue
+            execution_payment = (
+                execution.message.payment
+                if execution.message.payment
+                else Payment(chain=Chain.ETH, type=PaymentType.hold)
+            )
+            if execution_payment.type == payment_type:
+                sender = execution.message.address
+                chain = execution_payment.chain
+                executions_by_sender.setdefault(sender, {})
+                executions_by_sender[sender].setdefault(chain, []).append(execution)
         return executions_by_sender.items()

From fe59f3e889d5bbc09f5c22f1102ac7895d946310 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 28 Dec 2023 20:21:55 +0100
Subject: [PATCH 598/990] Fix: Solved MyPy issues.

---
 src/aleph/vm/orchestrator/payment.py | 4 ++--
 src/aleph/vm/orchestrator/tasks.py   | 4 ++--
 src/aleph/vm/pool.py                 | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 8ad6d4a3c..c3aa55789 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -64,7 +64,7 @@ def to_normalized_address(value: str) -> HexAddress:
 
 def get_required_balance(executions: Iterable[VmExecution]) -> Decimal:
     """Get the balance required for the resources of the user from the messages and the pricing aggregate."""
-    balance = 0
+    balance = Decimal(0)
     for execution in executions:
         balance += compute_execution_hold_cost(execution)
 
@@ -125,7 +125,7 @@ def _get_execution_storage_size(execution: VmExecution) -> int:
 
 def get_required_flow(executions: Iterable[VmExecution]) -> Decimal:
     """Compute the flow required for the resources of the user from the messages and the pricing aggregate"""
-    flow = 0
+    flow = Decimal(0)
     for execution in executions:
         flow += compute_execution_flow_cost(execution)
 
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index f151000cc..9618228a4 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -143,7 +143,7 @@ async def monitor_payments(app: web.Application):
         await asyncio.sleep(settings.PAYMENT_MONITOR_INTERVAL)
 
         # Check if the balance held in the wallet is sufficient holder tier resources
-        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.hold):
+        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.hold).items():
             for chain, executions in chains.items():
                 balance = await get_balance(sender)
 
@@ -156,7 +156,7 @@ async def monitor_payments(app: web.Application):
                     required_balance = get_required_balance(executions)
 
         # Check if the balance held in the wallet is sufficient stream tier resources
-        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid):
+        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():
             for chain, executions in chains.items():
                 stream = get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
                 logger.debug(
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 9cc468e4d..94a7b3227 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -270,9 +270,9 @@ def get_instance_executions(self) -> Iterable[VmExecution]:
             if execution.is_running and execution.is_instance
         )
 
-    def get_executions_by_sender(self, payment_type: PaymentType) -> Iterable[Tuple[str, Dict[str, list[VmExecution]]]]:
+    def get_executions_by_sender(self, payment_type: PaymentType) -> Dict[str, Dict[str, list[VmExecution]]]:
         """Return all executions of the given type, grouped by sender and by chain."""
-        executions_by_sender: Tuple[str, Dict[str, list[VmExecution]]] = {}
+        executions_by_sender: Dict[str, Dict[str, list[VmExecution]]] = {}
         for vm_hash, execution in self.executions.items():
             if not execution.is_running:
                 # Ignore the execution that is stopping or not running anymore
@@ -290,4 +290,4 @@ def get_executions_by_sender(self, payment_type: PaymentType) -> Iterable[Tuple[
                 chain = execution_payment.chain
                 executions_by_sender.setdefault(sender, {})
                 executions_by_sender[sender].setdefault(chain, []).append(execution)
-        return executions_by_sender.items()
+        return executions_by_sender

From 2067fb3f117f6e957347bfa446541e2cfdcde7b0 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Fri, 29 Dec 2023 12:16:00 +0100
Subject: [PATCH 599/990] Fix: Added missing default options and correct
 installation versions for packages.

---
 packaging/Makefile                             | 2 +-
 packaging/aleph-vm/etc/aleph-vm/supervisor.env | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index cc28a391c..c3b8f99c1 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.1' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message@git+https://github.com/aleph-im/aleph-message.git@mhh-pay-as-you-go-v1#egg=aleph-message' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo
diff --git a/packaging/aleph-vm/etc/aleph-vm/supervisor.env b/packaging/aleph-vm/etc/aleph-vm/supervisor.env
index 141fdcbbb..2e6b82e6e 100644
--- a/packaging/aleph-vm/etc/aleph-vm/supervisor.env
+++ b/packaging/aleph-vm/etc/aleph-vm/supervisor.env
@@ -1,2 +1,3 @@
 ALEPH_VM_PRINT_SYSTEM_LOGS=True
 ALEPH_VM_DOMAIN_NAME=vm.example.org
+ALEPH_VM_PAYMENT_RECEIVER_ADDRESS=0x0000000000000000000000000000000000000000

From 01cb8cb2fcf16ff63696e18515ab05960f46f0cb Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Fri, 29 Dec 2023 12:52:43 +0100
Subject: [PATCH 600/990] Fix: Added missing superfluid dependency on
 installation step.

---
 packaging/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index c3b8f99c1..64b758445 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message@git+https://github.com/aleph-im/aleph-message.git@mhh-pay-as-you-go-v1#egg=aleph-message' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message@git+https://github.com/aleph-im/aleph-message.git@mhh-pay-as-you-go-v1#egg=aleph-message' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid@git+https://github.com/Godspower-Eze/superfluid.py#egg=superfluid'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo

From 122a5ccdc683697ef3ff14c13bc5f7834098c9cc Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Fri, 29 Dec 2023 13:12:39 +0100
Subject: [PATCH 601/990] Fix: Solved price issue and set paymen_address param
 to empty by default.

---
 packaging/aleph-vm/etc/aleph-vm/supervisor.env | 2 +-
 src/aleph/vm/conf.py                           | 1 +
 src/aleph/vm/orchestrator/payment.py           | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/packaging/aleph-vm/etc/aleph-vm/supervisor.env b/packaging/aleph-vm/etc/aleph-vm/supervisor.env
index 2e6b82e6e..9793f2422 100644
--- a/packaging/aleph-vm/etc/aleph-vm/supervisor.env
+++ b/packaging/aleph-vm/etc/aleph-vm/supervisor.env
@@ -1,3 +1,3 @@
 ALEPH_VM_PRINT_SYSTEM_LOGS=True
 ALEPH_VM_DOMAIN_NAME=vm.example.org
-ALEPH_VM_PAYMENT_RECEIVER_ADDRESS=0x0000000000000000000000000000000000000000
+ALEPH_VM_PAYMENT_RECEIVER_ADDRESS=
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index d52c2bbee..9d61d28aa 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -213,6 +213,7 @@ class Settings(BaseSettings):
         description="Interval in seconds between payment checks",
     )
     PAYMENT_RECEIVER_ADDRESS: str = Field(
+        default="",
         description="Address of the account receiving payments",
     )
     PAYMENT_SUPER_TOKEN: str = Field(
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index c3aa55789..0f7984de9 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -133,7 +133,7 @@ def get_required_flow(executions: Iterable[VmExecution]) -> Decimal:
 
 
 def compute_execution_flow_cost(execution: VmExecution) -> Decimal:
-    compute_unit_cost_hour = 0.011 if execution.persistent else 0.11  # TODO: Get from PAYG aggregate
+    compute_unit_cost_hour = 0.11 if execution.persistent else 0.011  # TODO: Get from PAYG aggregate
     compute_unit_cost_second = compute_unit_cost_hour / Hour
 
     compute_units_required = _get_nb_compute_units(execution)

From d9f66c65297e7079d1210a66779345410712a2cd Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Fri, 29 Dec 2023 17:06:48 +0100
Subject: [PATCH 602/990] Feature: Added instance notification endpoint.

---
 src/aleph/vm/orchestrator/resources.py      |  4 ++
 src/aleph/vm/orchestrator/supervisor.py     |  2 +
 src/aleph/vm/orchestrator/views/__init__.py | 69 ++++++++++++++++++++-
 3 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 175b61bfd..ee9936280 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -124,3 +124,7 @@ class Allocation(BaseModel):
     instances: set[str] = Field(default_factory=set)
     on_demand_vms: Optional[set[str]] = None
     jobs: Optional[set[str]] = None
+
+
+class VMNotification(BaseModel):
+    instance: str = None
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index d0a8be8e3..988cce3aa 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -30,6 +30,7 @@
     about_execution_records,
     about_executions,
     about_login,
+    notify_allocation,
     run_code_from_hostname,
     run_code_from_path,
     status_check_fastapi,
@@ -86,6 +87,7 @@ async def allow_cors_on_endpoint(request: web.Request):
         web.get("/about/usage/system", about_system_usage),
         web.get("/about/config", about_config),
         web.post("/control/allocations", update_allocations),
+        web.post("/control/allocation", notify_allocation),
         web.get("/control/machine/{ref}/logs", stream_logs),
         web.post("/control/machine/{ref}/expire", operate_expire),
         web.post("/control/machine/{ref}/stop", operate_stop),
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 11daae50f..4d2adfb80 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -24,7 +24,7 @@
 from aleph.vm.orchestrator import status
 from aleph.vm.orchestrator.metrics import get_execution_records
 from aleph.vm.orchestrator.pubsub import PubSub
-from aleph.vm.orchestrator.resources import Allocation
+from aleph.vm.orchestrator.resources import Allocation, VMNotification
 from aleph.vm.orchestrator.run import run_code_on_request, start_persistent_vm
 from aleph.vm.orchestrator.views.host_status import (
     check_dns_ipv4,
@@ -279,8 +279,15 @@ async def update_allocations(request: web.Request):
     # First free resources from persistent programs and instances that are not scheduled anymore.
     allocations = allocation.persistent_vms | allocation.instances
     # Make a copy since the pool is modified
+
     for execution in list(pool.get_persistent_executions()):
-        if execution.vm_hash not in allocations and execution.is_running:
+        if (
+            execution.vm_hash not in allocations
+            and execution.is_running
+            and (
+                not execution.message.payment or (execution.message.payment and not execution.message.payment.is_stream)
+            )
+        ):
             vm_type = "instance" if execution.is_instance else "persistent program"
             logger.info("Stopping %s %s", vm_type, execution.vm_hash)
             await pool.stop_vm(execution.vm_hash)
@@ -346,3 +353,61 @@ async def update_allocations(request: web.Request):
         },
         status=status_code,
     )
+
+
+async def notify_allocation(request: web.Request):
+    if not authenticate_api_request(request):
+        return web.HTTPUnauthorized(text="Authentication token received is invalid")
+
+    try:
+        data = await request.json()
+        vm_notification = VMNotification.parse_obj(data)
+    except ValidationError as error:
+        return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code)
+
+    pubsub: PubSub = request.app["pubsub"]
+    pool: VmPool = request.app["vm_pool"]
+
+    # First free resources from persistent programs and instances that are not scheduled anymore.
+    instance = vm_notification.instance
+
+    # Exceptions that can be raised when starting a VM:
+    vm_creation_exceptions = (
+        UnknownHashError,
+        ResourceDownloadError,
+        FileTooLargeError,
+        VmSetupError,
+        MicroVMFailedInitError,
+        HostNotFoundError,
+    )
+
+    scheduling_errors: dict[ItemHash, Exception] = {}
+
+    instance_item_hash = ItemHash(instance)
+    try:
+        await start_persistent_vm(instance_item_hash, pubsub, pool)
+        successful = True
+    except vm_creation_exceptions as error:
+        logger.exception(error)
+        scheduling_errors[instance_item_hash] = error
+        successful = False
+
+    failing = set(scheduling_errors.keys())
+
+    status_code: int
+    if not failing:
+        status_code = 200  # OK
+    elif not successful:
+        status_code = 503  # Service Unavailable
+    else:
+        status_code = 207  # Multi-Status
+
+    return web.json_response(
+        data={
+            "success": not failing,
+            "successful": successful,
+            "failing": list(failing),
+            "errors": {vm_hash: repr(error) for vm_hash, error in scheduling_errors.items()},
+        },
+        status=status_code,
+    )

From c4c0bc9e7347d8217f0786808b9aa0ef34e99dbf Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Fri, 29 Dec 2023 17:10:39 +0100
Subject: [PATCH 603/990] Fix: Remove default value for notification payload

---
 src/aleph/vm/orchestrator/resources.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index ee9936280..6dfcfdba1 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -127,4 +127,4 @@ class Allocation(BaseModel):
 
 
 class VMNotification(BaseModel):
-    instance: str = None
+    instance: str

From 2021149a4236474028536ea90f0a6709d7d31162 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Fri, 29 Dec 2023 18:14:49 +0100
Subject: [PATCH 604/990] Fix: Solved some PR comments.

---
 src/aleph/vm/models.py                      |  4 +++
 src/aleph/vm/orchestrator/payment.py        | 39 +++++++++------------
 src/aleph/vm/orchestrator/views/__init__.py | 10 +-----
 src/aleph/vm/utils.py                       | 16 +++++++++
 4 files changed, 37 insertions(+), 32 deletions(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 0c45e5155..d7640f639 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -115,6 +115,10 @@ def vm_id(self) -> Optional[int]:
     def controller_service(self) -> str:
         return f"aleph-vm-controller@{self.vm_hash}.service"
 
+    @property
+    def is_payment_stream(self) -> bool:
+        return self.message.payment.is_stream if self.message.payment else False
+
     def __init__(
         self,
         vm_hash: ItemHash,
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 0f7984de9..2ded7e43d 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -4,15 +4,15 @@
 from typing import Iterable
 
 import aiohttp
-from eth_typing import HexAddress, HexStr
-from eth_utils import from_wei, hexstr_if_str, is_address, to_hex
+from eth_typing import HexAddress
+from eth_utils import from_wei
 from superfluid import CFA_V1, Web3FlowInfo
 
 from aleph.vm.conf import settings
 from aleph.vm.constants import GiB, Hour, MiB
 from aleph.vm.controllers.firecracker.program import AlephProgramResources
 from aleph.vm.models import VmExecution
-from aleph.vm.utils import get_path_size
+from aleph.vm.utils import get_path_size, to_normalized_address
 
 logger = logging.getLogger(__name__)
 
@@ -44,24 +44,11 @@ def get_stream(sender: str, receiver: str, chain) -> Decimal:
     receiver_address: HexAddress = to_normalized_address(receiver)
 
     flow_data: Web3FlowInfo = superfluid_instance.get_flow(super_token, sender_address, receiver_address)
+    # TODO:
     stream = from_wei(flow_data["flowRate"], "ether")
     return Decimal(stream)
 
 
-def to_normalized_address(value: str) -> HexAddress:
-    """
-    Converts an address to its normalized hexadecimal representation.
-    """
-    try:
-        hex_address = hexstr_if_str(to_hex, value).lower()
-    except AttributeError:
-        raise TypeError("Value must be any string, instead got type {}".format(type(value)))
-    if is_address(hex_address):
-        return HexAddress(HexStr(hex_address))
-    else:
-        raise ValueError("Unknown format {}, attempted to normalize to {}".format(value, hex_address))
-
-
 def get_required_balance(executions: Iterable[VmExecution]) -> Decimal:
     """Get the balance required for the resources of the user from the messages and the pricing aggregate."""
     balance = Decimal(0)
@@ -72,17 +59,19 @@ def get_required_balance(executions: Iterable[VmExecution]) -> Decimal:
 
 
 def compute_execution_hold_cost(execution: VmExecution) -> Decimal:
+    # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
     compute_unit_cost = 200 if execution.persistent else 2000
 
     compute_units_required = _get_nb_compute_units(execution)
     compute_unit_multiplier = _get_compute_unit_multiplier(execution)
 
-    compute_unit_price = Decimal(compute_units_required) * compute_unit_multiplier * compute_unit_cost
+    compute_unit_price = Decimal(compute_units_required) * Decimal(compute_unit_multiplier) * Decimal(compute_unit_cost)
     price = compute_unit_price + _get_additional_storage_hold_price(execution)
     return Decimal(price)
 
 
 def _get_additional_storage_hold_price(execution: VmExecution) -> Decimal:
+    # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
     nb_compute_units = execution.vm.hardware_resources.vcpus
     free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
 
@@ -133,7 +122,8 @@ def get_required_flow(executions: Iterable[VmExecution]) -> Decimal:
 
 
 def compute_execution_flow_cost(execution: VmExecution) -> Decimal:
-    compute_unit_cost_hour = 0.11 if execution.persistent else 0.011  # TODO: Get from PAYG aggregate
+    # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
+    compute_unit_cost_hour = 0.11 if execution.persistent else 0.011
     compute_unit_cost_second = compute_unit_cost_hour / Hour
 
     compute_units_required = _get_nb_compute_units(execution)
@@ -149,12 +139,15 @@ def compute_execution_flow_cost(execution: VmExecution) -> Decimal:
 
 
 def _get_additional_storage_flow_price(execution: VmExecution) -> Decimal:
-    additional_storage_hour_price = 0.000000977  # TODO: Get from PAYG aggregate
-    additional_storage_second_price = additional_storage_hour_price / Hour
+    # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
+    additional_storage_hour_price = 0.000000977
+    additional_storage_second_price = Decimal(additional_storage_hour_price) / Decimal(Hour)
     nb_compute_units = execution.vm.hardware_resources.vcpus
     free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
 
     total_volume_size = _get_execution_storage_size(execution)
-    additional_storage = max(total_volume_size - (free_storage_per_compute_unit * nb_compute_units), 0)
-    price = additional_storage / additional_storage_second_price / MiB
+    additional_storage = max(
+        Decimal(total_volume_size) - (Decimal(free_storage_per_compute_unit) * Decimal(nb_compute_units)), Decimal(0)
+    )
+    price = additional_storage / additional_storage_second_price / Decimal(MiB)
     return Decimal(price)
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 4d2adfb80..a94267c28 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -279,15 +279,8 @@ async def update_allocations(request: web.Request):
     # First free resources from persistent programs and instances that are not scheduled anymore.
     allocations = allocation.persistent_vms | allocation.instances
     # Make a copy since the pool is modified
-
     for execution in list(pool.get_persistent_executions()):
-        if (
-            execution.vm_hash not in allocations
-            and execution.is_running
-            and (
-                not execution.message.payment or (execution.message.payment and not execution.message.payment.is_stream)
-            )
-        ):
+        if execution.vm_hash not in allocations and execution.is_running and not execution.is_payment_stream:
             vm_type = "instance" if execution.is_instance else "persistent program"
             logger.info("Stopping %s %s", vm_type, execution.vm_hash)
             await pool.stop_vm(execution.vm_hash)
@@ -368,7 +361,6 @@ async def notify_allocation(request: web.Request):
     pubsub: PubSub = request.app["pubsub"]
     pool: VmPool = request.app["vm_pool"]
 
-    # First free resources from persistent programs and instances that are not scheduled anymore.
     instance = vm_notification.instance
 
     # Exceptions that can be raised when starting a VM:
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index 7eb6fc7dd..e59caf271 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -16,6 +16,8 @@
 import msgpack
 from aleph_message.models import ExecutableContent, InstanceContent, ProgramContent
 from aleph_message.models.execution.base import MachineType
+from eth_typing import HexAddress, HexStr
+from eth_utils import hexstr_if_str, is_address, to_hex
 
 logger = logging.getLogger(__name__)
 
@@ -169,3 +171,17 @@ def get_block_device_size(device: str) -> int:
     )
     size = int(output.stdout.decode())
     return size
+
+
+def to_normalized_address(value: str) -> HexAddress:
+    """
+    Converts an address to its normalized hexadecimal representation.
+    """
+    try:
+        hex_address = hexstr_if_str(to_hex, value).lower()
+    except AttributeError:
+        raise TypeError("Value must be any string, instead got type {}".format(type(value)))
+    if is_address(hex_address):
+        return HexAddress(HexStr(hex_address))
+    else:
+        raise ValueError("Unknown format {}, attempted to normalize to {}".format(value, hex_address))

From 0b23328be67e34652e25934c3683dbe01cfa4d65 Mon Sep 17 00:00:00 2001
From: mhh 
Date: Tue, 9 Jan 2024 17:21:36 +0100
Subject: [PATCH 605/990] Refactor metrics.py to use AsyncSession and await for
 committing and querying.

---
 src/aleph/vm/orchestrator/metrics.py | 35 +++++++++++++---------------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index 32b340830..0e0128b5b 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -4,6 +4,7 @@
 from typing import Any
 from uuid import UUID
 
+import sqlalchemy
 from sqlalchemy import (
     JSON,
     Boolean,
@@ -13,6 +14,7 @@
     Integer,
     String,
     create_engine,
+    select,
 )
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import sessionmaker
@@ -24,7 +26,7 @@
 
 from aleph.vm.conf import make_db_url, settings
 
-Session: sessionmaker
+AsyncSession: sessionmaker
 
 logger = logging.getLogger(__name__)
 
@@ -32,9 +34,9 @@
 
 
 def setup_engine():
-    global Session
+    global AsyncSession
     engine = create_engine(make_db_url(), echo=True)
-    Session = sessionmaker(bind=engine)
+    AsyncSession = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
     return engine
 
 
@@ -86,28 +88,23 @@ async def save_execution_data(execution_uuid: UUID, execution_data: str):
 
 async def save_record(record: ExecutionRecord):
     """Record the resource usage in database"""
-    session = Session()  # undefined name 'Session'
-    try:
+    async with AsyncSession() as session:  # Use AsyncSession in a context manager
         session.add(record)
-        session.commit()
-    finally:
-        session.close()
+        await session.commit()  # Use await for commit
 
 
 async def delete_record(execution_uuid: str):
     """Delete the resource usage in database"""
-    session = Session()  # undefined name 'Session'
-    try:
-        session.query(ExecutionRecord).filter(ExecutionRecord.uuid == execution_uuid).delete()
-        session.commit()
-    finally:
-        session.close()
+    async with AsyncSession() as session:
+        try:
+            await session.query(ExecutionRecord).filter(ExecutionRecord.uuid == execution_uuid).delete()
+            await session.commit()
+        finally:
+            await session.close()
 
 
 async def get_execution_records() -> Iterable[ExecutionRecord]:
     """Get the execution records from the database."""
-    session = Session()  # undefined name 'Session'
-    try:
-        return session.query(ExecutionRecord).all()
-    finally:
-        session.close()
+    async with AsyncSession() as session:  # Use AsyncSession in a context manager
+        result = await session.execute(select(ExecutionRecord))  # Use execute for querying
+        return result.scalars().all()

From 8811a6ecab680c580b05ff000551933a2cd28e9a Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Tue, 9 Jan 2024 17:58:55 +0100
Subject: [PATCH 606/990] Fix: Solved issue with compute units calculation.

---
 src/aleph/vm/orchestrator/payment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 2ded7e43d..3f0767695 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -72,7 +72,7 @@ def compute_execution_hold_cost(execution: VmExecution) -> Decimal:
 
 def _get_additional_storage_hold_price(execution: VmExecution) -> Decimal:
     # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
-    nb_compute_units = execution.vm.hardware_resources.vcpus
+    nb_compute_units = _get_nb_compute_units(execution)
     free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
 
     total_volume_size = _get_execution_storage_size(execution)

From 999f343d6f9675c799003301fde6d5b482dfa69c Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Wed, 10 Jan 2024 13:40:47 +0100
Subject: [PATCH 607/990] Fix: Solved to use the same method to get compute
 units.

---
 src/aleph/vm/orchestrator/payment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 3f0767695..37f3ff2f1 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -142,7 +142,7 @@ def _get_additional_storage_flow_price(execution: VmExecution) -> Decimal:
     # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
     additional_storage_hour_price = 0.000000977
     additional_storage_second_price = Decimal(additional_storage_hour_price) / Decimal(Hour)
-    nb_compute_units = execution.vm.hardware_resources.vcpus
+    nb_compute_units = _get_nb_compute_units(execution)
     free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
 
     total_volume_size = _get_execution_storage_size(execution)

From c67ea4eef72aeb415c15cfc991708af7d456eab0 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 11 Jan 2024 12:42:57 +0100
Subject: [PATCH 608/990] Fix: Solved some PR review comments.

---
 src/aleph/vm/conf.py                        |  5 ++-
 src/aleph/vm/constants.py                   |  4 +-
 src/aleph/vm/models.py                      |  2 +-
 src/aleph/vm/orchestrator/payment.py        | 45 +++++++++++----------
 src/aleph/vm/orchestrator/tasks.py          | 10 ++---
 src/aleph/vm/orchestrator/views/__init__.py |  3 +-
 src/aleph/vm/utils.py                       | 13 +++---
 7 files changed, 44 insertions(+), 38 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 9d61d28aa..f0233c3d2 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -11,7 +11,7 @@
 from typing import Any, Literal, NewType, Optional, Union
 
 from aleph_message.models import ItemHash
-from pydantic import BaseSettings, Field
+from pydantic import BaseSettings, Field, HttpUrl
 from pydantic.env_settings import DotenvType, env_file_sentinel
 from pydantic.typing import StrPath
 
@@ -216,13 +216,14 @@ class Settings(BaseSettings):
         default="",
         description="Address of the account receiving payments",
     )
+    # This address is the ALEPH SuperToken on SuperFluid Testnet
     PAYMENT_SUPER_TOKEN: str = Field(
         default="0x1290248e01ed2f9f863a9752a8aad396ef3a1b00",
         description="Address of the ALEPH SuperToken on SuperFluid",
     )
     PAYMENT_PRICING_AGGREGATE: str = ""  # TODO: Missing
 
-    PAYMENT_RPC_SERVER: str = Field(
+    PAYMENT_RPC_API: HttpUrl = Field(
         default="https://api.avax-test.network/ext/bc/C/rpc",
         description="Default to Avalanche Testnet RPC",
     )
diff --git a/src/aleph/vm/constants.py b/src/aleph/vm/constants.py
index 519e7987e..9701259e0 100644
--- a/src/aleph/vm/constants.py
+++ b/src/aleph/vm/constants.py
@@ -1,5 +1,5 @@
 KiB = 1024
 MiB = 1024 * 1024
 GiB = 1024 * 1024 * 1024
-Hour = 60 * 60
-Minute = 60
+HOUR = 60 * 60
+MINUTE = 60
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index d7640f639..e9a632d85 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -116,7 +116,7 @@ def controller_service(self) -> str:
         return f"aleph-vm-controller@{self.vm_hash}.service"
 
     @property
-    def is_payment_stream(self) -> bool:
+    def uses_payment_stream(self) -> bool:
         return self.message.payment.is_stream if self.message.payment else False
 
     def __init__(
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 37f3ff2f1..a913b0c6f 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -9,7 +9,7 @@
 from superfluid import CFA_V1, Web3FlowInfo
 
 from aleph.vm.conf import settings
-from aleph.vm.constants import GiB, Hour, MiB
+from aleph.vm.constants import GiB, HOUR, MiB
 from aleph.vm.controllers.firecracker.program import AlephProgramResources
 from aleph.vm.models import VmExecution
 from aleph.vm.utils import get_path_size, to_normalized_address
@@ -37,28 +37,28 @@ def get_stream(sender: str, receiver: str, chain) -> Decimal:
 
     # TODO: Convert chain str to ID
     chain_id = 43113
-    superfluid_instance = CFA_V1(settings.PAYMENT_RPC_SERVER, chain_id)
+    superfluid_instance = CFA_V1(settings.PAYMENT_RPC_API, chain_id)
 
     super_token: HexAddress = to_normalized_address(settings.PAYMENT_SUPER_TOKEN)
     sender_address: HexAddress = to_normalized_address(sender)
     receiver_address: HexAddress = to_normalized_address(receiver)
 
     flow_data: Web3FlowInfo = superfluid_instance.get_flow(super_token, sender_address, receiver_address)
-    # TODO:
+    # TODO: Implement and use the SDK to make the conversion
     stream = from_wei(flow_data["flowRate"], "ether")
     return Decimal(stream)
 
 
-def get_required_balance(executions: Iterable[VmExecution]) -> Decimal:
+async def compute_required_balance(executions: Iterable[VmExecution]) -> Decimal:
     """Get the balance required for the resources of the user from the messages and the pricing aggregate."""
     balance = Decimal(0)
     for execution in executions:
-        balance += compute_execution_hold_cost(execution)
+        balance += await compute_execution_hold_cost(execution)
 
     return Decimal(balance)
 
 
-def compute_execution_hold_cost(execution: VmExecution) -> Decimal:
+async def compute_execution_hold_cost(execution: VmExecution) -> Decimal:
     # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
     compute_unit_cost = 200 if execution.persistent else 2000
 
@@ -66,22 +66,24 @@ def compute_execution_hold_cost(execution: VmExecution) -> Decimal:
     compute_unit_multiplier = _get_compute_unit_multiplier(execution)
 
     compute_unit_price = Decimal(compute_units_required) * Decimal(compute_unit_multiplier) * Decimal(compute_unit_cost)
-    price = compute_unit_price + _get_additional_storage_hold_price(execution)
+    additional_storage_hold_price = await _get_additional_storage_hold_price(execution)
+    price = compute_unit_price + additional_storage_hold_price
     return Decimal(price)
 
 
-def _get_additional_storage_hold_price(execution: VmExecution) -> Decimal:
+async def _get_additional_storage_hold_price(execution: VmExecution) -> Decimal:
     # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
     nb_compute_units = _get_nb_compute_units(execution)
     free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
 
-    total_volume_size = _get_execution_storage_size(execution)
+    total_volume_size = await _get_execution_storage_size(execution)
     additional_storage = max(total_volume_size - (free_storage_per_compute_unit * nb_compute_units), 0)
     price = Decimal(additional_storage) / 20 / MiB
     return price
 
 
 def _get_nb_compute_units(execution: VmExecution) -> int:
+    """A compute unit is currently defined as: 1 vcpu, 2048 MB of memory."""
     cpu = execution.vm.hardware_resources.vcpus
     memory = math.ceil(execution.vm.hardware_resources.memory / 2048)
     nb_compute_units = cpu if cpu >= memory else memory
@@ -95,36 +97,36 @@ def _get_compute_unit_multiplier(execution: VmExecution) -> int:
     return compute_unit_multiplier
 
 
-def _get_execution_storage_size(execution: VmExecution) -> int:
+async def _get_execution_storage_size(execution: VmExecution) -> int:
     size = 0
 
     if execution.is_instance:
         size += execution.message.rootfs.size_mib * MiB
     elif execution.is_program:
         if isinstance(execution.resources, AlephProgramResources):
-            size += get_path_size(execution.resources.code_path)
+            size += await get_path_size(execution.resources.code_path)
             if execution.resources.data_path:
-                size += get_path_size(execution.resources.data_path)
+                size += await get_path_size(execution.resources.data_path)
 
     for volume in execution.resources.volumes:
-        size += get_path_size(volume.path_on_host)
+        size += await get_path_size(volume.path_on_host)
 
     return size
 
 
-def get_required_flow(executions: Iterable[VmExecution]) -> Decimal:
+async def get_required_flow(executions: Iterable[VmExecution]) -> Decimal:
     """Compute the flow required for the resources of the user from the messages and the pricing aggregate"""
     flow = Decimal(0)
     for execution in executions:
-        flow += compute_execution_flow_cost(execution)
+        flow += await compute_execution_flow_cost(execution)
 
     return Decimal(flow)
 
 
-def compute_execution_flow_cost(execution: VmExecution) -> Decimal:
+async def compute_execution_flow_cost(execution: VmExecution) -> Decimal:
     # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
     compute_unit_cost_hour = 0.11 if execution.persistent else 0.011
-    compute_unit_cost_second = compute_unit_cost_hour / Hour
+    compute_unit_cost_second = compute_unit_cost_hour / HOUR
 
     compute_units_required = _get_nb_compute_units(execution)
     compute_unit_multiplier = _get_compute_unit_multiplier(execution)
@@ -133,19 +135,20 @@ def compute_execution_flow_cost(execution: VmExecution) -> Decimal:
         Decimal(compute_units_required) * Decimal(compute_unit_multiplier) * Decimal(compute_unit_cost_second)
     )
 
-    price = compute_unit_price + _get_additional_storage_flow_price(execution)
+    additional_storage_flow_price = await _get_additional_storage_flow_price(execution)
+    price = compute_unit_price + additional_storage_flow_price
 
     return Decimal(price)
 
 
-def _get_additional_storage_flow_price(execution: VmExecution) -> Decimal:
+async def _get_additional_storage_flow_price(execution: VmExecution) -> Decimal:
     # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
     additional_storage_hour_price = 0.000000977
-    additional_storage_second_price = Decimal(additional_storage_hour_price) / Decimal(Hour)
+    additional_storage_second_price = Decimal(additional_storage_hour_price) / Decimal(HOUR)
     nb_compute_units = _get_nb_compute_units(execution)
     free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
 
-    total_volume_size = _get_execution_storage_size(execution)
+    total_volume_size = await _get_execution_storage_size(execution)
     additional_storage = max(
         Decimal(total_volume_size) - (Decimal(free_storage_per_compute_unit) * Decimal(nb_compute_units)), Decimal(0)
     )
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 9618228a4..3b87ae003 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -23,7 +23,7 @@
 from aleph.vm.utils import create_task_log_exceptions
 
 from .messages import load_updated_message
-from .payment import get_balance, get_required_balance, get_required_flow, get_stream
+from .payment import get_balance, compute_required_balance, get_required_flow, get_stream
 from .pubsub import PubSub
 from .reactor import Reactor
 
@@ -148,12 +148,12 @@ async def monitor_payments(app: web.Application):
                 balance = await get_balance(sender)
 
                 # Stop executions until the required balance is reached
-                required_balance = get_required_balance(executions)
+                required_balance = await compute_required_balance(executions)
                 while balance < required_balance:
                     last_execution = executions.pop(-1)
                     logger.debug(f"Stopping {last_execution} due to insufficient stream")
                     await last_execution.stop()
-                    required_balance = get_required_balance(executions)
+                    required_balance = await compute_required_balance(executions)
 
         # Check if the balance held in the wallet is sufficient stream tier resources
         for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():
@@ -162,14 +162,14 @@ async def monitor_payments(app: web.Application):
                 logger.debug(
                     f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
                 )
-                required_stream = get_required_flow(executions)
+                required_stream = await get_required_flow(executions)
                 logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
                 # Stop executions until the required stream is reached
                 while stream < required_stream:
                     last_execution = executions.pop(-1)
                     logger.debug(f"Stopping {last_execution} due to insufficient stream")
                     await last_execution.stop()
-                    required_stream = get_required_flow(executions)
+                    required_stream = await get_required_flow(executions)
 
 
 async def start_payment_monitoring_task(app: web.Application):
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index a94267c28..6bb178a44 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -280,7 +280,7 @@ async def update_allocations(request: web.Request):
     allocations = allocation.persistent_vms | allocation.instances
     # Make a copy since the pool is modified
     for execution in list(pool.get_persistent_executions()):
-        if execution.vm_hash not in allocations and execution.is_running and not execution.is_payment_stream:
+        if execution.vm_hash not in allocations and execution.is_running and not execution.uses_payment_stream:
             vm_type = "instance" if execution.is_instance else "persistent program"
             logger.info("Stopping %s %s", vm_type, execution.vm_hash)
             await pool.stop_vm(execution.vm_hash)
@@ -349,6 +349,7 @@ async def update_allocations(request: web.Request):
 
 
 async def notify_allocation(request: web.Request):
+    """Notify instance allocation, only used for Pay as you Go feature"""
     if not authenticate_api_request(request):
         return web.HTTPUnauthorized(text="Authentication token received is invalid")
 
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index e59caf271..da1b04c59 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -152,24 +152,25 @@ class NotEnoughDiskSpaceError(OSError):
     pass
 
 
-def get_path_size(path: Path) -> int:
+async def get_path_size(path: Path) -> int:
+    """Get the size in bytes of a given path."""
     if path.is_dir():
         return sum([f.stat().st_size for f in path.glob("**/*")])
     elif path.is_block_device():
-        return get_block_device_size(str(path))
+        return await get_block_device_size(str(path))
     elif path.is_file():
         return path.stat().st_size
     else:
         raise NotImplementedError
 
 
-def get_block_device_size(device: str) -> int:
-    output = subprocess.run(
+async def get_block_device_size(device: str) -> int:
+    """Get the size in bytes of a given device block."""
+    output = await run_in_subprocess(
         ["lsblk", device, "--output", "SIZE", "--bytes", "--noheadings", "--nodeps"],
-        capture_output=True,
         check=True,
     )
-    size = int(output.stdout.decode())
+    size = int(output.strip().decode())
     return size
 
 

From cda50734bc8b716ca375b6eaea6e8aa859792a46 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 11 Jan 2024 13:05:19 +0100
Subject: [PATCH 609/990] Fix: Solved AsyncSession issue.

---
 packaging/Makefile                      |  2 +-
 pyproject.toml                          |  5 +++-
 src/aleph/vm/conf.py                    |  2 +-
 src/aleph/vm/orchestrator/cli.py        | 17 ++++++++----
 src/aleph/vm/orchestrator/metrics.py    | 36 +++++++++++--------------
 src/aleph/vm/orchestrator/payment.py    |  2 +-
 src/aleph/vm/orchestrator/supervisor.py |  4 +++
 src/aleph/vm/orchestrator/tasks.py      |  7 ++++-
 8 files changed, 44 insertions(+), 31 deletions(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 64b758445..5e9b619e5 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message@git+https://github.com/aleph-im/aleph-message.git@mhh-pay-as-you-go-v1#egg=aleph-message' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid@git+https://github.com/Godspower-Eze/superfluid.py#egg=superfluid'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message@git+https://github.com/aleph-im/aleph-message.git@mhh-pay-as-you-go-v1#egg=aleph-message' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid@git+https://github.com/Godspower-Eze/superfluid.py#egg=superfluid' 'sqlalchemy[asyncio]' 'aiosqlite==0.19.0' 'alembic==1.13.1'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo
diff --git a/pyproject.toml b/pyproject.toml
index 52bc3a21b..349fdd1bb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,7 +46,10 @@ dependencies = [
   "dbus-python==1.3.2",
   "systemd-python==235",
   "systemd-python==235",
-  "superfluid @ git+https://github.com/Godspower-Eze/superfluid.py#egg=superfluid"
+  "superfluid @ git+https://github.com/Godspower-Eze/superfluid.py#egg=superfluid",
+  "sqlalchemy[asyncio]",
+  "aiosqlite==0.19.0",
+  "alembic==1.13.1"
 ]
 
 [project.urls]
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index f0233c3d2..18e7cfea5 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -405,7 +405,7 @@ class Config:
 
 
 def make_db_url():
-    return f"sqlite:///{settings.EXECUTION_DATABASE}"
+    return f"sqlite+aiosqlite:///{settings.EXECUTION_DATABASE}"
 
 
 # Settings singleton
diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index 9dbfc93d2..fdd20b3e5 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -10,6 +10,7 @@
 from typing import Callable
 
 from aiohttp.web import Request, Response
+from sqlalchemy.ext.asyncio import create_async_engine
 
 from aleph.vm.version import get_version_from_apt, get_version_from_git
 
@@ -161,7 +162,7 @@ async def benchmark(runs: int):
     with fake requests.
     """
     engine = metrics.setup_engine()
-    metrics.create_tables(engine)
+    await metrics.create_tables(engine)
 
     ref = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")
     settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM
@@ -266,16 +267,22 @@ def change_dir(directory: Path):
         os.chdir(current_directory)
 
 
-def run_db_migrations():
+def run_db_migrations(connection):
     project_dir = Path(__file__).parent
 
-    db_url = make_db_url()
     alembic_cfg = alembic.config.Config("alembic.ini")
     alembic_cfg.attributes["configure_logger"] = False
+    alembic_cfg.attributes["connection"] = connection
     logging.getLogger("alembic").setLevel(logging.CRITICAL)
 
     with change_dir(project_dir):
-        alembic.command.upgrade(alembic_cfg, "head", tag=db_url)
+        alembic.command.upgrade(alembic_cfg, "head")
+
+
+async def run_async_db_migrations():
+    async_engine = create_async_engine(make_db_url(), echo=True)
+    async with async_engine.begin() as conn:
+        await conn.run_sync(run_db_migrations)
 
 
 def main():
@@ -339,7 +346,7 @@ def main():
     settings.check()
 
     logger.debug("Initialising the DB...")
-    run_db_migrations()
+    asyncio.run(run_async_db_migrations())
     logger.debug("DB up to date.")
 
     if args.benchmark > 0:
diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index 0e0128b5b..ca89f0daa 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -4,19 +4,9 @@
 from typing import Any
 from uuid import UUID
 
-import sqlalchemy
-from sqlalchemy import (
-    JSON,
-    Boolean,
-    Column,
-    DateTime,
-    Float,
-    Integer,
-    String,
-    create_engine,
-    select,
-)
+from sqlalchemy import JSON, Boolean, Column, DateTime, Float, Integer, String, select
 from sqlalchemy.engine import Engine
+from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
 from sqlalchemy.orm import sessionmaker
 
 try:
@@ -26,7 +16,7 @@
 
 from aleph.vm.conf import make_db_url, settings
 
-AsyncSession: sessionmaker
+AsyncSessionMaker: sessionmaker
 
 logger = logging.getLogger(__name__)
 
@@ -34,14 +24,16 @@
 
 
 def setup_engine():
-    global AsyncSession
-    engine = create_engine(make_db_url(), echo=True)
-    AsyncSession = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
+    global AsyncSessionMaker
+    engine = create_async_engine(make_db_url(), echo=True)
+    AsyncSessionMaker = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
     return engine
 
 
-def create_tables(engine: Engine):
-    Base.metadata.create_all(engine)
+async def create_tables(engine: Engine):
+    async with engine.begin() as conn:
+        await conn.run_sync(Base.metadata.drop_all)
+        await conn.run_sync(Base.metadata.create_all)
 
 
 class ExecutionRecord(Base):
@@ -88,7 +80,7 @@ async def save_execution_data(execution_uuid: UUID, execution_data: str):
 
 async def save_record(record: ExecutionRecord):
     """Record the resource usage in database"""
-    async with AsyncSession() as session:  # Use AsyncSession in a context manager
+    async with AsyncSessionMaker() as session:  # Use AsyncSession in a context manager
         session.add(record)
         await session.commit()  # Use await for commit
 
@@ -105,6 +97,8 @@ async def delete_record(execution_uuid: str):
 
 async def get_execution_records() -> Iterable[ExecutionRecord]:
     """Get the execution records from the database."""
-    async with AsyncSession() as session:  # Use AsyncSession in a context manager
+    async with AsyncSessionMaker() as session:  # Use AsyncSession in a context manager
         result = await session.execute(select(ExecutionRecord))  # Use execute for querying
-        return result.scalars().all()
+        executions = result.scalars().all()
+        await session.commit()
+        return executions
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index a913b0c6f..87338e41b 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -9,7 +9,7 @@
 from superfluid import CFA_V1, Web3FlowInfo
 
 from aleph.vm.conf import settings
-from aleph.vm.constants import GiB, HOUR, MiB
+from aleph.vm.constants import HOUR, GiB, MiB
 from aleph.vm.controllers.firecracker.program import AlephProgramResources
 from aleph.vm.models import VmExecution
 from aleph.vm.utils import get_path_size, to_normalized_address
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 988cce3aa..904ea66c5 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -5,6 +5,7 @@
 At it's core, it is currently an asynchronous HTTP server using aiohttp, but this may
 evolve in the future.
 """
+import asyncio
 import logging
 from collections.abc import Awaitable
 from pathlib import Path
@@ -134,6 +135,9 @@ def run():
 
     logger.debug(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
 
+    engine = setup_engine()
+    asyncio.run(create_tables(engine))
+
     try:
         if settings.WATCH_FOR_MESSAGES:
             app.on_startup.append(start_watch_for_messages_task)
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 3b87ae003..a5ef2846e 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -23,7 +23,12 @@
 from aleph.vm.utils import create_task_log_exceptions
 
 from .messages import load_updated_message
-from .payment import get_balance, compute_required_balance, get_required_flow, get_stream
+from .payment import (
+    compute_required_balance,
+    get_balance,
+    get_required_flow,
+    get_stream,
+)
 from .pubsub import PubSub
 from .reactor import Reactor
 

From 022e8b5381c1e7623a91a85b2d2e104610f7d362 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 11 Jan 2024 16:24:21 +0100
Subject: [PATCH 610/990] Cleanup if else condition

---
 src/aleph/vm/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index e9a632d85..846b633ea 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -117,7 +117,7 @@ def controller_service(self) -> str:
 
     @property
     def uses_payment_stream(self) -> bool:
-        return self.message.payment.is_stream if self.message.payment else False
+        return self.message.payment and self.message.payment.is_stream
 
     def __init__(
         self,

From 09a931988c09465c604ebb75f4e48676dbe14282 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 12 Jan 2024 14:57:52 +0100
Subject: [PATCH 611/990] Fix: Conflict in pyproject.toml dependencies,
 outdated aleph-message

Two different versions of `alembic` were specified.

Solution: Update the requirement in all relevant places;

Co-authored-by: Mike Hukiewitz <70762838+MHHukiewitz@users.noreply.github.com>
---
 docker/vm_supervisor-dev.dockerfile    | 2 +-
 examples/volumes/Dockerfile            | 2 +-
 packaging/Makefile                     | 2 +-
 pyproject.toml                         | 5 ++---
 src/aleph/vm/orchestrator/README.md    | 2 +-
 src/aleph/vm/orchestrator/resources.py | 8 ++++++++
 6 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 402a1e014..07f0e6d50 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -19,7 +19,7 @@ RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://s3.amazonaws.com/spec.ccf
 RUN ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker
 RUN ln /opt/firecracker/release-*/jailer-v* /opt/firecracker/jailer
 
-RUN pip3 install typing-extensions 'aleph-message==0.4.1'
+RUN pip3 install typing-extensions 'aleph-message==0.4.2'
 
 RUN mkdir -p /var/lib/aleph/vm/jailer
 
diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile
index 96dcb6b73..4d9a54150 100644
--- a/examples/volumes/Dockerfile
+++ b/examples/volumes/Dockerfile
@@ -6,6 +6,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 RUN python3 -m venv /opt/venv
-RUN /opt/venv/bin/pip install 'aleph-message==0.4.1'
+RUN /opt/venv/bin/pip install 'aleph-message==0.4.2'
 
 CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs
diff --git a/packaging/Makefile b/packaging/Makefile
index 5e9b619e5..0226808c9 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message@git+https://github.com/aleph-im/aleph-message.git@mhh-pay-as-you-go-v1#egg=aleph-message' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid@git+https://github.com/Godspower-Eze/superfluid.py#egg=superfluid' 'sqlalchemy[asyncio]' 'aiosqlite==0.19.0' 'alembic==1.13.1'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.2' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]' 'aiosqlite==0.19.0' 'alembic==1.13.1'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo
diff --git a/pyproject.toml b/pyproject.toml
index 349fdd1bb..db7bd3efc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -27,10 +27,9 @@ dependencies = [
   "pydantic[dotenv]==1.10.13",
   "aiohttp==3.8.6",
   "aiodns==3.1.0",
-  "alembic==1.7.6",
   "setproctitle==1.3.3",
   "pyyaml==6.0.1",
-  "aleph-message @ git+https://github.com/aleph-im/aleph-message.git@mhh-pay-as-you-go-v1#egg=aleph-message",
+  "aleph-message==0.4.2",
   "jwskate==0.8.0",
   "eth-account==0.9.0",
   "sentry-sdk==1.31.0",
@@ -86,7 +85,7 @@ dependencies = [
   "coverage[toml]==7.3.2",
   "pytest==7.4.2",
   "pytest-mock==3.11.1",
-  "pytest-asyncio==0.21.1 ",
+  "pytest-asyncio==0.21.1",
 ]
 [tool.hatch.envs.testing.scripts]
 test = "pytest {args:tests}"
diff --git a/src/aleph/vm/orchestrator/README.md b/src/aleph/vm/orchestrator/README.md
index 05a5c0dd6..ba39e2cfb 100644
--- a/src/aleph/vm/orchestrator/README.md
+++ b/src/aleph/vm/orchestrator/README.md
@@ -87,7 +87,7 @@ is used to parse and validate Aleph messages.
 ```shell
 apt install -y --no-install-recommends --no-install-suggests python3-pip
 pip3 install pydantic[dotenv]
-pip3 install 'aleph-message==0.4.1'
+pip3 install 'aleph-message==0.4.2'
 ```
 
 ### 2.f. Create the jailer working directory:
diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 6dfcfdba1..fe26a7e69 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -120,6 +120,10 @@ async def about_system_usage(_: web.Request):
 
 
 class Allocation(BaseModel):
+    """An allocation is the set of resources that are currently allocated on this orchestrator.
+    It contains the item_hashes of all persistent VMs, instances, on-demand VMs and jobs.
+    """
+
     persistent_vms: set[str] = Field(default_factory=set)
     instances: set[str] = Field(default_factory=set)
     on_demand_vms: Optional[set[str]] = None
@@ -127,4 +131,8 @@ class Allocation(BaseModel):
 
 
 class VMNotification(BaseModel):
+    """A notification to the orchestrator that a VM has been created or destroyed.
+    This is typically sent by a user that just created a VM in order to quickly ensure the creation of the VM.
+    """
+
     instance: str

From 7e12f9e174f8a47022ad35674fb612a7a77e2fa6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 15 Jan 2024 12:06:15 +0100
Subject: [PATCH 612/990] Fix: Dependencies were downloaded from GitHub without
 version pinning

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index db7bd3efc..49fd5b81d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,7 +45,7 @@ dependencies = [
   "dbus-python==1.3.2",
   "systemd-python==235",
   "systemd-python==235",
-  "superfluid @ git+https://github.com/Godspower-Eze/superfluid.py#egg=superfluid",
+  "superfluid~=0.2.1",
   "sqlalchemy[asyncio]",
   "aiosqlite==0.19.0",
   "alembic==1.13.1"

From b44a89c09539497244e11e389eabc7abb5bc1608 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 15 Jan 2024 12:35:09 +0100
Subject: [PATCH 613/990] Doc: Improve docstrings

---
 src/aleph/vm/orchestrator/payment.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 87338e41b..61b24a5e5 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -18,9 +18,16 @@
 
 
 async def get_balance(address: str) -> Decimal:
-    """Get the balance of the user from the PyAleph."""
-    # See https://github.com/aleph-im/pyaleph/blob/master/src/aleph/web/controllers/routes.py#L62
-    # "/api/v0/addresses/{address}/balance"
+    """
+    Get the balance of the user from the PyAleph API.
+
+    API Endpoint:
+        GET /api/v0/addresses/{address}/balance
+
+    For more details, see the PyAleph API documentation:
+    https://github.com/aleph-im/pyaleph/blob/master/src/aleph/web/controllers/routes.py#L62
+    """
+
     async with aiohttp.ClientSession() as session:
         url = f"{settings.API_SERVER}/api/v0/{address}/balance"
         resp = await session.get(url)
@@ -33,8 +40,10 @@ async def get_balance(address: str) -> Decimal:
 
 
 def get_stream(sender: str, receiver: str, chain) -> Decimal:
-    # See https://community.aleph.im/t/pay-as-you-go-using-superfluid/98/11
-
+    """
+    Get the stream of the user from the Superfluid API.
+    See https://community.aleph.im/t/pay-as-you-go-using-superfluid/98/11
+    """
     # TODO: Convert chain str to ID
     chain_id = 43113
     superfluid_instance = CFA_V1(settings.PAYMENT_RPC_API, chain_id)

From e744ab18493388fb90347a242aafe17afba69336 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 15 Jan 2024 12:36:39 +0100
Subject: [PATCH 614/990] Fix: Failed requests could not be noticed

The function returned 0 when the request failed or the format of the response was incorrect.

Do not silently ignore errors and raise the appropriate error instead.
---
 src/aleph/vm/orchestrator/payment.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 61b24a5e5..a29b01e1a 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -32,11 +32,15 @@ async def get_balance(address: str) -> Decimal:
         url = f"{settings.API_SERVER}/api/v0/{address}/balance"
         resp = await session.get(url)
 
-        if not resp.ok:
+        # Consider the balance as null if the address is not found
+        if resp.status == 404:
             return Decimal(0)
 
+        # Raise an error if the request failed
+        resp.raise_for_status()
+
         resp_data = await resp.json()
-        return resp_data["balance"] if resp_data["balance"] else 0
+        return resp_data["balance"]
 
 
 def get_stream(sender: str, receiver: str, chain) -> Decimal:

From a050720914281e3f6058f3419ee3730d4e7a9bc1 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 15 Jan 2024 12:37:24 +0100
Subject: [PATCH 615/990] Fix: Requests were executed sequentially

Solution: Run the code in parallel using `asyncio.gather`
---
 src/aleph/vm/orchestrator/payment.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index a29b01e1a..46cdc77f5 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -1,3 +1,4 @@
+import asyncio
 import logging
 import math
 from decimal import Decimal
@@ -64,11 +65,8 @@ def get_stream(sender: str, receiver: str, chain) -> Decimal:
 
 async def compute_required_balance(executions: Iterable[VmExecution]) -> Decimal:
     """Get the balance required for the resources of the user from the messages and the pricing aggregate."""
-    balance = Decimal(0)
-    for execution in executions:
-        balance += await compute_execution_hold_cost(execution)
-
-    return Decimal(balance)
+    costs = await asyncio.gather(*(compute_execution_hold_cost(execution) for execution in executions))
+    return sum(costs, Decimal(0))
 
 
 async def compute_execution_hold_cost(execution: VmExecution) -> Decimal:

From 0a75211450b077b64482b3c404f355137a0d6687 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 15 Jan 2024 14:00:31 +0100
Subject: [PATCH 616/990] Doc: Add docstrings

---
 src/aleph/vm/orchestrator/resources.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index fe26a7e69..fe0eb9bd0 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -92,6 +92,7 @@ def get_machine_properties() -> MachineProperties:
 
 
 async def about_system_usage(_: web.Request):
+    """Public endpoint to expose information about the system usage."""
     period_start = datetime.now(timezone.utc).replace(second=0, microsecond=0)
 
     usage: MachineUsage = MachineUsage(

From 1cd0e5538763e39a5c748493fa027dfc0048b2c2 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 15 Jan 2024 14:01:09 +0100
Subject: [PATCH 617/990] Fix: Notifying of a PAYG allocation does not require
 authentication

---
 src/aleph/vm/orchestrator/views/__init__.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 6bb178a44..b4811fc6e 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -350,9 +350,6 @@ async def update_allocations(request: web.Request):
 
 async def notify_allocation(request: web.Request):
     """Notify instance allocation, only used for Pay as you Go feature"""
-    if not authenticate_api_request(request):
-        return web.HTTPUnauthorized(text="Authentication token received is invalid")
-
     try:
         data = await request.json()
         vm_notification = VMNotification.parse_obj(data)

From 1f1425d0c85e0841dee0f9235a2672c4a11a79f2 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 15 Jan 2024 14:34:25 +0100
Subject: [PATCH 618/990] Fix: Main event loop is blocked by network call

The function `superfluid_instance.get_flow` runs a network call in a blocking manner.

Solution: Run the network request in a background thread and wait for it to complete.
---
 src/aleph/vm/orchestrator/payment.py | 8 ++++++--
 src/aleph/vm/orchestrator/tasks.py   | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 46cdc77f5..f212f0adc 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -44,7 +44,7 @@ async def get_balance(address: str) -> Decimal:
         return resp_data["balance"]
 
 
-def get_stream(sender: str, receiver: str, chain) -> Decimal:
+async def get_stream(sender: str, receiver: str, chain) -> Decimal:
     """
     Get the stream of the user from the Superfluid API.
     See https://community.aleph.im/t/pay-as-you-go-using-superfluid/98/11
@@ -57,7 +57,11 @@ def get_stream(sender: str, receiver: str, chain) -> Decimal:
     sender_address: HexAddress = to_normalized_address(sender)
     receiver_address: HexAddress = to_normalized_address(receiver)
 
-    flow_data: Web3FlowInfo = superfluid_instance.get_flow(super_token, sender_address, receiver_address)
+    # Run the network request in a background thread and wait for it to complete.
+    loop = asyncio.get_event_loop()
+    flow_data: Web3FlowInfo = await loop.run_in_executor(
+        None, superfluid_instance.get_flow, super_token, sender_address, receiver_address
+    )
     # TODO: Implement and use the SDK to make the conversion
     stream = from_wei(flow_data["flowRate"], "ether")
     return Decimal(stream)
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index a5ef2846e..17479a835 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -163,7 +163,7 @@ async def monitor_payments(app: web.Application):
         # Check if the balance held in the wallet is sufficient stream tier resources
         for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():
             for chain, executions in chains.items():
-                stream = get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
+                stream = await get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
                 logger.debug(
                     f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
                 )

From 7221273b83e4179f9d6ebcbdb211ea6e038a6a60 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 16 Jan 2024 14:39:21 +0100
Subject: [PATCH 619/990] Fix: Issues with asyncio and sqlite tables creation

---
 src/aleph/vm/orchestrator/supervisor.py | 5 +----
 src/aleph/vm/pool.py                    | 3 +--
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 904ea66c5..8e4ef8f81 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -119,7 +119,7 @@ def run():
     settings.check()
 
     engine = setup_engine()
-    create_tables(engine)
+    asyncio.run(create_tables(engine))
 
     pool = VmPool()
     pool.setup()
@@ -135,9 +135,6 @@ def run():
 
     logger.debug(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
 
-    engine = setup_engine()
-    asyncio.run(create_tables(engine))
-
     try:
         if settings.WATCH_FOR_MESSAGES:
             app.on_startup.append(start_watch_for_messages_task)
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 94a7b3227..78518a341 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -68,8 +68,7 @@ def __init__(self):
         self.snapshot_manager.run_snapshots()
 
         logger.debug("Loading existing executions ...")
-        loop = asyncio.get_event_loop()
-        loop.run_until_complete(self._load_persistent_executions())
+        asyncio.run(self._load_persistent_executions())
 
     def setup(self) -> None:
         """Set up the VM pool and the network."""

From b5b26889b8bb4e8573586a31dc8f5164f62d4041 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 16 Jan 2024 15:02:07 +0100
Subject: [PATCH 620/990] Fix: Use ValueError instead of NotImplemented Error

---
 src/aleph/vm/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index da1b04c59..f190dceea 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -161,7 +161,7 @@ async def get_path_size(path: Path) -> int:
     elif path.is_file():
         return path.stat().st_size
     else:
-        raise NotImplementedError
+        raise ValueError(f"Unknown path type for {path}")
 
 
 async def get_block_device_size(device: str) -> int:

From 3ab82471d36403c58b4840cbba1c9771f92af83c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 16 Jan 2024 15:03:05 +0100
Subject: [PATCH 621/990] Fix: ItemHash was specified as type `str`

---
 src/aleph/vm/orchestrator/resources.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index fe0eb9bd0..6589c37ee 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -6,6 +6,7 @@
 import cpuinfo
 import psutil
 from aiohttp import web
+from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import CpuProperties
 from pydantic import BaseModel, Field
 
@@ -136,4 +137,4 @@ class VMNotification(BaseModel):
     This is typically sent by a user that just created a VM in order to quickly ensure the creation of the VM.
     """
 
-    instance: str
+    instance: ItemHash

From d6dffaae62b1ca3c2760d64087603fd8694d52a8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 22 Jan 2024 19:10:14 +0100
Subject: [PATCH 622/990] Fix: Wrong way to specify socket family in aiohttp,
 crashed

---
 src/aleph/vm/orchestrator/views/host_status.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/host_status.py b/src/aleph/vm/orchestrator/views/host_status.py
index 7bea32604..6ef41c0ee 100644
--- a/src/aleph/vm/orchestrator/views/host_status.py
+++ b/src/aleph/vm/orchestrator/views/host_status.py
@@ -22,8 +22,8 @@ async def wrapper(*args: Any, **kwargs: Any) -> bool:
 
 async def check_ip_connectivity(url: str, socket_family: socket.AddressFamily = socket.AF_UNSPEC) -> bool:
     timeout = aiohttp.ClientTimeout(total=5)
-    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
-        async with session.get(url, socket_family=socket_family) as resp:
+    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(family=socket_family), timeout=timeout) as session:
+        async with session.get(url) as resp:
             # We expect the Quad9 endpoints to return a 404 error, but other endpoints may return a 200
             if resp.status not in (200, 404):
                 resp.raise_for_status()

From c49da8818bd246a1bb214c45d5be9f1bde702214 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 22 Jan 2024 19:48:35 +0100
Subject: [PATCH 623/990] Fix: API changes in SqlAlchemy caused a crash

In new versions of SqlAlchemy:
AttributeError: 'AsyncSession' object has no attribute 'query'
---
 src/aleph/vm/orchestrator/metrics.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index ca89f0daa..b009732dc 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -4,7 +4,7 @@
 from typing import Any
 from uuid import UUID
 
-from sqlalchemy import JSON, Boolean, Column, DateTime, Float, Integer, String, select
+from sqlalchemy import JSON, Boolean, Column, DateTime, Float, Integer, String, select, delete
 from sqlalchemy.engine import Engine
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
 from sqlalchemy.orm import sessionmaker
@@ -89,7 +89,8 @@ async def delete_record(execution_uuid: str):
     """Delete the resource usage in database"""
     async with AsyncSession() as session:
         try:
-            await session.query(ExecutionRecord).filter(ExecutionRecord.uuid == execution_uuid).delete()
+            statement = delete(ExecutionRecord).where(ExecutionRecord.uuid == execution_uuid)
+            await session.execute(statement)
             await session.commit()
         finally:
             await session.close()

From 80922a13654da2b90cd5e8471b526840975026db Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 10:53:08 +0100
Subject: [PATCH 624/990] Fix: Invalid addresses were difficult to debug

There was little information about what address caused an issue and why.

Solution: Parse each address separately and raise a new InvalidAddressError if an address could not be parsed.
---
 src/aleph/vm/orchestrator/payment.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index f212f0adc..021f63ac6 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -44,6 +44,11 @@ async def get_balance(address: str) -> Decimal:
         return resp_data["balance"]
 
 
+class InvalidAddressError(ValueError):
+    """The blockchain address could not be parsed."""
+    pass
+
+
 async def get_stream(sender: str, receiver: str, chain) -> Decimal:
     """
     Get the stream of the user from the Superfluid API.
@@ -53,9 +58,20 @@ async def get_stream(sender: str, receiver: str, chain) -> Decimal:
     chain_id = 43113
     superfluid_instance = CFA_V1(settings.PAYMENT_RPC_API, chain_id)
 
-    super_token: HexAddress = to_normalized_address(settings.PAYMENT_SUPER_TOKEN)
-    sender_address: HexAddress = to_normalized_address(sender)
-    receiver_address: HexAddress = to_normalized_address(receiver)
+    try:
+        super_token: HexAddress = to_normalized_address(settings.PAYMENT_SUPER_TOKEN)
+    except ValueError as error:
+        raise InvalidAddressError(f"Invalid token address '{settings.PAYMENT_SUPER_TOKEN}' - {error.args}") from error
+
+    try:
+        sender_address: HexAddress = to_normalized_address(sender)
+    except ValueError as error:
+        raise InvalidAddressError(f"Invalid sender address '{sender}' - {error.args}") from error
+
+    try:
+        receiver_address: HexAddress = to_normalized_address(receiver)
+    except ValueError as error:
+        raise InvalidAddressError(f"Invalid receiver address '{receiver}' - {error.args}") from error
 
     # Run the network request in a background thread and wait for it to complete.
     loop = asyncio.get_event_loop()

From a4b3b05b6a531c37c04be2750e667f832f16dca7 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 10:59:33 +0100
Subject: [PATCH 625/990] Fix: Balance and flow checks did not use the new API

Solution: Rely on the new API `/api/v0/price/{item_hash}` for all price calculations instead of computing it locally.
---
 src/aleph/vm/orchestrator/payment.py | 89 ++++++++++++++--------------
 src/aleph/vm/orchestrator/tasks.py   | 10 ++--
 2 files changed, 51 insertions(+), 48 deletions(-)

diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 021f63ac6..7801a4434 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -2,9 +2,10 @@
 import logging
 import math
 from decimal import Decimal
-from typing import Iterable
+from typing import Iterable, Optional
 
 import aiohttp
+from aleph_message.models import ItemHash, PaymentType
 from eth_typing import HexAddress
 from eth_utils import from_wei
 from superfluid import CFA_V1, Web3FlowInfo
@@ -18,7 +19,7 @@
 logger = logging.getLogger(__name__)
 
 
-async def get_balance(address: str) -> Decimal:
+async def fetch_balance_of_address(address: str) -> Decimal:
     """
     Get the balance of the user from the PyAleph API.
 
@@ -44,6 +45,44 @@ async def get_balance(address: str) -> Decimal:
         return resp_data["balance"]
 
 
+async def fetch_execution_flow_price(item_hash: ItemHash) -> Decimal:
+    """Fetch the flow price of an execution from the reference API server."""
+    async with aiohttp.ClientSession() as session:
+        url = f"{settings.API_SERVER}/api/v0/price/{item_hash}"
+        resp = await session.get(url)
+        # Raise an error if the request failed
+        resp.raise_for_status()
+
+        resp_data = await resp.json()
+        required_flow: float = resp_data["required_tokens"]
+        payment_type: Optional[str] = resp_data["payment_type"]
+
+        if payment_type is None:
+            raise ValueError("Payment type must be specified in the message")
+        elif payment_type != PaymentType.superfluid:
+            raise ValueError(f"Payment type {payment_type} is not supported")
+
+        return Decimal(required_flow)
+
+
+async def fetch_execution_hold_price(item_hash: ItemHash) -> Decimal:
+    """Fetch the hold price of an execution from the reference API server."""
+    async with aiohttp.ClientSession() as session:
+        url = f"{settings.API_SERVER}/api/v0/price/{item_hash}"
+        resp = await session.get(url)
+        # Raise an error if the request failed
+        resp.raise_for_status()
+
+        resp_data = await resp.json()
+        required_hold: float = resp_data["required_tokens"]
+        payment_type: Optional[str] = resp_data["payment_type"]
+
+        if payment_type not in (None, PaymentType.hold):
+            raise ValueError(f"Payment type {payment_type} is not supported")
+
+        return Decimal(required_hold)
+
+
 class InvalidAddressError(ValueError):
     """The blockchain address could not be parsed."""
     pass
@@ -85,7 +124,7 @@ async def get_stream(sender: str, receiver: str, chain) -> Decimal:
 
 async def compute_required_balance(executions: Iterable[VmExecution]) -> Decimal:
     """Get the balance required for the resources of the user from the messages and the pricing aggregate."""
-    costs = await asyncio.gather(*(compute_execution_hold_cost(execution) for execution in executions))
+    costs = await asyncio.gather(*(fetch_execution_hold_price(execution.vm_hash) for execution in executions))
     return sum(costs, Decimal(0))
 
 
@@ -145,43 +184,7 @@ async def _get_execution_storage_size(execution: VmExecution) -> int:
     return size
 
 
-async def get_required_flow(executions: Iterable[VmExecution]) -> Decimal:
-    """Compute the flow required for the resources of the user from the messages and the pricing aggregate"""
-    flow = Decimal(0)
-    for execution in executions:
-        flow += await compute_execution_flow_cost(execution)
-
-    return Decimal(flow)
-
-
-async def compute_execution_flow_cost(execution: VmExecution) -> Decimal:
-    # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
-    compute_unit_cost_hour = 0.11 if execution.persistent else 0.011
-    compute_unit_cost_second = compute_unit_cost_hour / HOUR
-
-    compute_units_required = _get_nb_compute_units(execution)
-    compute_unit_multiplier = _get_compute_unit_multiplier(execution)
-
-    compute_unit_price = (
-        Decimal(compute_units_required) * Decimal(compute_unit_multiplier) * Decimal(compute_unit_cost_second)
-    )
-
-    additional_storage_flow_price = await _get_additional_storage_flow_price(execution)
-    price = compute_unit_price + additional_storage_flow_price
-
-    return Decimal(price)
-
-
-async def _get_additional_storage_flow_price(execution: VmExecution) -> Decimal:
-    # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
-    additional_storage_hour_price = 0.000000977
-    additional_storage_second_price = Decimal(additional_storage_hour_price) / Decimal(HOUR)
-    nb_compute_units = _get_nb_compute_units(execution)
-    free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
-
-    total_volume_size = await _get_execution_storage_size(execution)
-    additional_storage = max(
-        Decimal(total_volume_size) - (Decimal(free_storage_per_compute_unit) * Decimal(nb_compute_units)), Decimal(0)
-    )
-    price = additional_storage / additional_storage_second_price / Decimal(MiB)
-    return Decimal(price)
+async def compute_total_flow(executions: Iterable[VmExecution]) -> Decimal:
+    """Compute the flow required for a collection of executions, typically all executions from a specific address"""
+    flows = await asyncio.gather(*(compute_execution_hold_cost(execution) for execution in executions))
+    return sum(flows, Decimal(0))
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 17479a835..135ea39fb 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -25,8 +25,8 @@
 from .messages import load_updated_message
 from .payment import (
     compute_required_balance,
-    get_balance,
-    get_required_flow,
+    compute_total_flow,
+    fetch_balance_of_address,
     get_stream,
 )
 from .pubsub import PubSub
@@ -150,7 +150,7 @@ async def monitor_payments(app: web.Application):
         # Check if the balance held in the wallet is sufficient holder tier resources
         for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.hold).items():
             for chain, executions in chains.items():
-                balance = await get_balance(sender)
+                balance = await fetch_balance_of_address(sender)
 
                 # Stop executions until the required balance is reached
                 required_balance = await compute_required_balance(executions)
@@ -167,14 +167,14 @@ async def monitor_payments(app: web.Application):
                 logger.debug(
                     f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
                 )
-                required_stream = await get_required_flow(executions)
+                required_stream = await compute_total_flow(executions)
                 logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
                 # Stop executions until the required stream is reached
                 while stream < required_stream:
                     last_execution = executions.pop(-1)
                     logger.debug(f"Stopping {last_execution} due to insufficient stream")
                     await last_execution.stop()
-                    required_stream = await get_required_flow(executions)
+                    required_stream = await compute_total_flow(executions)
 
 
 async def start_payment_monitoring_task(app: web.Application):

From 6093110ed1c0e638dd6eff17a735780320ec3df0 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 11:01:48 +0100
Subject: [PATCH 626/990] Fix: An instance could be created without token or
 flow

Solution: Add payment check in the `notify_allocation` function, raising proper HTTP errors when the action is invalid.
---
 src/aleph/vm/orchestrator/views/__init__.py | 55 ++++++++++++++++++---
 1 file changed, 48 insertions(+), 7 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index b4811fc6e..1c8e89926 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -1,7 +1,9 @@
 import binascii
 import logging
 from collections.abc import Awaitable
+from decimal import Decimal
 from hashlib import sha256
+from json import JSONDecodeError
 from pathlib import Path
 from string import Template
 from typing import Optional
@@ -11,7 +13,7 @@
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPNotFound
 from aleph_message.exceptions import UnknownHashError
-from aleph_message.models import ItemHash
+from aleph_message.models import ItemHash, MessageType
 from pydantic import ValidationError
 
 from aleph.vm.conf import settings
@@ -22,7 +24,13 @@
 from aleph.vm.controllers.firecracker.program import FileTooLargeError
 from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInitError
 from aleph.vm.orchestrator import status
+from aleph.vm.orchestrator.messages import try_get_message
 from aleph.vm.orchestrator.metrics import get_execution_records
+from aleph.vm.orchestrator.payment import (
+    InvalidAddressError,
+    fetch_execution_flow_price,
+    get_stream,
+)
 from aleph.vm.orchestrator.pubsub import PubSub
 from aleph.vm.orchestrator.resources import Allocation, VMNotification
 from aleph.vm.orchestrator.run import run_code_on_request, start_persistent_vm
@@ -353,13 +361,48 @@ async def notify_allocation(request: web.Request):
     try:
         data = await request.json()
         vm_notification = VMNotification.parse_obj(data)
+    except JSONDecodeError as error:
+        raise web.HTTPBadRequest(reason="Body is not valid JSON") from error
     except ValidationError as error:
-        return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code)
+        raise web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code) from error
 
     pubsub: PubSub = request.app["pubsub"]
     pool: VmPool = request.app["vm_pool"]
 
-    instance = vm_notification.instance
+    item_hash: ItemHash = vm_notification.instance
+    message = await try_get_message(item_hash)
+    if message.type != MessageType.instance:
+        raise web.HTTPBadRequest(reason="Message is not an instance")
+
+    if not message.content.payment:
+        raise web.HTTPBadRequest(reason="Message does not have payment information")
+
+    if message.content.payment.receiver != settings.PAYMENT_RECEIVER_ADDRESS:
+        raise web.HTTPBadRequest(reason="Message is not for this instance")
+
+    # Check that there is a payment stream for this instance
+    try:
+        active_flow: Decimal = await get_stream(
+            sender=message.sender, receiver=message.content.payment.receiver, chain=message.content.payment.chain
+        )
+    except InvalidAddressError as error:
+        logger.warning(f"Invalid address {error}", exc_info=True)
+        raise web.HTTPBadRequest(reason=f"Invalid address {error}") from error
+
+    if not active_flow:
+        raise web.HTTPPaymentRequired(reason="Empty payment stream for this instance")
+
+    required_flow: Decimal = await fetch_execution_flow_price(item_hash)
+
+    if active_flow < required_flow:
+        active_flow_per_month = active_flow * 60 * 60 * 24 * (Decimal("30.41666666666923904761904784"))
+        required_flow_per_month = required_flow * 60 * 60 * 24 * Decimal("30.41666666666923904761904784")
+        raise web.HTTPPaymentRequired(
+            reason="Insufficient payment stream",
+            text="Insufficient payment stream for this instance\n\n"
+            f"Required: {required_flow_per_month} / month (flow = {required_flow})\n"
+            f"Present: {active_flow_per_month} / month (flow = {active_flow})",
+        )
 
     # Exceptions that can be raised when starting a VM:
     vm_creation_exceptions = (
@@ -372,14 +415,12 @@ async def notify_allocation(request: web.Request):
     )
 
     scheduling_errors: dict[ItemHash, Exception] = {}
-
-    instance_item_hash = ItemHash(instance)
     try:
-        await start_persistent_vm(instance_item_hash, pubsub, pool)
+        await start_persistent_vm(item_hash, pubsub, pool)
         successful = True
     except vm_creation_exceptions as error:
         logger.exception(error)
-        scheduling_errors[instance_item_hash] = error
+        scheduling_errors[item_hash] = error
         successful = False
 
     failing = set(scheduling_errors.keys())

From e5585d6ee3f4cb26a48acde5ef31424699ad329e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 11:04:32 +0100
Subject: [PATCH 627/990] Fix: Accessing an unknown URL raised a weird HTTP
 Error

Solution: Raise a standard 404 within paths `/about/`, `/control/` and `/status/`.
---
 src/aleph/vm/orchestrator/supervisor.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 8e4ef8f81..067d61e2d 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -102,6 +102,10 @@ async def allow_cors_on_endpoint(request: web.Request):
         web.get("/status/check/host", status_check_host),
         web.get("/status/check/version", status_check_version),
         web.get("/status/config", status_public_config),
+        # Raise an HTTP Error 404 if attempting to access an unknown URL within these paths.
+        web.get("/about/{suffix:.*}", lambda _: web.HTTPNotFound()),
+        web.get("/control/{suffix:.*}", lambda _: web.HTTPNotFound()),
+        web.get("/status/{suffix:.*}", lambda _: web.HTTPNotFound()),
         web.static("/static", Path(__file__).parent / "views/static"),
         web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
         web.route("*", "/{suffix:.*}", run_code_from_hostname),

From b1e61f541679e9376e9fc0e63147ef54c7380a5c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 11:05:06 +0100
Subject: [PATCH 628/990] Cleanup: Black formatting.

---
 src/aleph/vm/orchestrator/metrics.py | 12 +++++++++++-
 src/aleph/vm/orchestrator/payment.py |  1 +
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index b009732dc..56b239d36 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -4,7 +4,17 @@
 from typing import Any
 from uuid import UUID
 
-from sqlalchemy import JSON, Boolean, Column, DateTime, Float, Integer, String, select, delete
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    Column,
+    DateTime,
+    Float,
+    Integer,
+    String,
+    delete,
+    select,
+)
 from sqlalchemy.engine import Engine
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
 from sqlalchemy.orm import sessionmaker
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 7801a4434..ebc7e3df0 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -85,6 +85,7 @@ async def fetch_execution_hold_price(item_hash: ItemHash) -> Decimal:
 
 class InvalidAddressError(ValueError):
     """The blockchain address could not be parsed."""
+
     pass
 
 

From 695f22a407b59aa8fb6a6ae1530a6c00f3fa6cab Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 11 Dec 2023 15:43:26 +0100
Subject: [PATCH 629/990] Fix: ISP KPN Internet (kpn.com) reports IPFS as DDoS

See https://github.com/ipfs/kubo/issues/10250

Node operators receive abuse letters from KPN.

Solution: Blacklist the IPv4 range of KPN Internet
---
 packaging/aleph-vm/etc/ipfs/kubo.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packaging/aleph-vm/etc/ipfs/kubo.json b/packaging/aleph-vm/etc/ipfs/kubo.json
index 56db34c10..9957b142e 100644
--- a/packaging/aleph-vm/etc/ipfs/kubo.json
+++ b/packaging/aleph-vm/etc/ipfs/kubo.json
@@ -3,6 +3,7 @@
     "ServiceMode": "disabled"
   },
   "AddrFilters": [
+    "/ip4/86.84.0.0/ipcidr/16"
   ],
   "Reprovider": {
     "Strategy": "roots"

From 01b89e76a2780889aaa24264534b713b3939909c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 12:10:12 +0100
Subject: [PATCH 630/990] Fix: Trailing `/` in API_SERVER setting caused
 invalid path

Solution: Strip the trailing `/` during settings setup
---
 src/aleph/vm/conf.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 18e7cfea5..e77aeed82 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -346,6 +346,8 @@ def setup(self):
         os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True)
         os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True)
 
+        self.API_SERVER = self.API_SERVER.rstrip("/")
+
         if not self.NETWORK_INTERFACE:
             self.NETWORK_INTERFACE = get_default_interface()
 

From 0820617b6d54d1a69e6a893c4e1b3a9b2b58cebe Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 13:54:33 +0100
Subject: [PATCH 631/990] Fix: Wrong session object crashed SQL query

---
 src/aleph/vm/orchestrator/metrics.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index 56b239d36..0578845a8 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -97,7 +97,7 @@ async def save_record(record: ExecutionRecord):
 
 async def delete_record(execution_uuid: str):
     """Delete the resource usage in database"""
-    async with AsyncSession() as session:
+    async with AsyncSessionMaker() as session:
         try:
             statement = delete(ExecutionRecord).where(ExecutionRecord.uuid == execution_uuid)
             await session.execute(statement)

From 4c05ab8db9fd83eb162916a4fd5f8663910530f9 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 18:10:14 +0100
Subject: [PATCH 632/990] Fix: Executions were not stopped correctly

---
 src/aleph/vm/orchestrator/tasks.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 135ea39fb..6f0f83091 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -154,10 +154,12 @@ async def monitor_payments(app: web.Application):
 
                 # Stop executions until the required balance is reached
                 required_balance = await compute_required_balance(executions)
+                logger.debug(f"Required balance for Sender {sender} executions: {required_balance}")
+                # Stop executions until the required balance is reached
                 while balance < required_balance:
                     last_execution = executions.pop(-1)
                     logger.debug(f"Stopping {last_execution} due to insufficient stream")
-                    await last_execution.stop()
+                    await pool.stop_vm(last_execution.vm_hash)
                     required_balance = await compute_required_balance(executions)
 
         # Check if the balance held in the wallet is sufficient stream tier resources
@@ -173,7 +175,7 @@ async def monitor_payments(app: web.Application):
                 while stream < required_stream:
                     last_execution = executions.pop(-1)
                     logger.debug(f"Stopping {last_execution} due to insufficient stream")
-                    await last_execution.stop()
+                    await pool.stop_vm(last_execution.vm_hash)
                     required_stream = await compute_total_flow(executions)
 
 

From 0f02656803c531f3cc04a5626bf7c2ed91d1f880 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 25 Jan 2024 16:28:16 +0100
Subject: [PATCH 633/990] Fix: Mixed up balance with flows

Also refactor compute_total_flow -> compute_required_flow
and remove deprecated code.

Co-authored-by: Hugo Herter 
---
 src/aleph/vm/orchestrator/payment.py | 65 ++--------------------------
 src/aleph/vm/orchestrator/tasks.py   |  6 +--
 2 files changed, 6 insertions(+), 65 deletions(-)

diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index ebc7e3df0..3b27c4412 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -1,6 +1,5 @@
 import asyncio
 import logging
-import math
 from decimal import Decimal
 from typing import Iterable, Optional
 
@@ -11,10 +10,8 @@
 from superfluid import CFA_V1, Web3FlowInfo
 
 from aleph.vm.conf import settings
-from aleph.vm.constants import HOUR, GiB, MiB
-from aleph.vm.controllers.firecracker.program import AlephProgramResources
 from aleph.vm.models import VmExecution
-from aleph.vm.utils import get_path_size, to_normalized_address
+from aleph.vm.utils import to_normalized_address
 
 logger = logging.getLogger(__name__)
 
@@ -129,63 +126,7 @@ async def compute_required_balance(executions: Iterable[VmExecution]) -> Decimal
     return sum(costs, Decimal(0))
 
 
-async def compute_execution_hold_cost(execution: VmExecution) -> Decimal:
-    # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
-    compute_unit_cost = 200 if execution.persistent else 2000
-
-    compute_units_required = _get_nb_compute_units(execution)
-    compute_unit_multiplier = _get_compute_unit_multiplier(execution)
-
-    compute_unit_price = Decimal(compute_units_required) * Decimal(compute_unit_multiplier) * Decimal(compute_unit_cost)
-    additional_storage_hold_price = await _get_additional_storage_hold_price(execution)
-    price = compute_unit_price + additional_storage_hold_price
-    return Decimal(price)
-
-
-async def _get_additional_storage_hold_price(execution: VmExecution) -> Decimal:
-    # TODO: Use PAYMENT_PRICING_AGGREGATE when possible
-    nb_compute_units = _get_nb_compute_units(execution)
-    free_storage_per_compute_unit = 2 * GiB if not execution.persistent else 20 * GiB
-
-    total_volume_size = await _get_execution_storage_size(execution)
-    additional_storage = max(total_volume_size - (free_storage_per_compute_unit * nb_compute_units), 0)
-    price = Decimal(additional_storage) / 20 / MiB
-    return price
-
-
-def _get_nb_compute_units(execution: VmExecution) -> int:
-    """A compute unit is currently defined as: 1 vcpu, 2048 MB of memory."""
-    cpu = execution.vm.hardware_resources.vcpus
-    memory = math.ceil(execution.vm.hardware_resources.memory / 2048)
-    nb_compute_units = cpu if cpu >= memory else memory
-    return nb_compute_units
-
-
-def _get_compute_unit_multiplier(execution: VmExecution) -> int:
-    compute_unit_multiplier = 1
-    if not execution.persistent and execution.message.environment.internet:
-        compute_unit_multiplier += 1
-    return compute_unit_multiplier
-
-
-async def _get_execution_storage_size(execution: VmExecution) -> int:
-    size = 0
-
-    if execution.is_instance:
-        size += execution.message.rootfs.size_mib * MiB
-    elif execution.is_program:
-        if isinstance(execution.resources, AlephProgramResources):
-            size += await get_path_size(execution.resources.code_path)
-            if execution.resources.data_path:
-                size += await get_path_size(execution.resources.data_path)
-
-    for volume in execution.resources.volumes:
-        size += await get_path_size(volume.path_on_host)
-
-    return size
-
-
-async def compute_total_flow(executions: Iterable[VmExecution]) -> Decimal:
+async def compute_required_flow(executions: Iterable[VmExecution]) -> Decimal:
     """Compute the flow required for a collection of executions, typically all executions from a specific address"""
-    flows = await asyncio.gather(*(compute_execution_hold_cost(execution) for execution in executions))
+    flows = await asyncio.gather(*(fetch_execution_flow_price(execution.vm_hash) for execution in executions))
     return sum(flows, Decimal(0))
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 6f0f83091..5b10c430f 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -25,7 +25,7 @@
 from .messages import load_updated_message
 from .payment import (
     compute_required_balance,
-    compute_total_flow,
+    compute_required_flow,
     fetch_balance_of_address,
     get_stream,
 )
@@ -169,14 +169,14 @@ async def monitor_payments(app: web.Application):
                 logger.debug(
                     f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
                 )
-                required_stream = await compute_total_flow(executions)
+                required_stream = await compute_required_flow(executions)
                 logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
                 # Stop executions until the required stream is reached
                 while stream < required_stream:
                     last_execution = executions.pop(-1)
                     logger.debug(f"Stopping {last_execution} due to insufficient stream")
                     await pool.stop_vm(last_execution.vm_hash)
-                    required_stream = await compute_total_flow(executions)
+                    required_stream = await compute_required_flow(executions)
 
 
 async def start_payment_monitoring_task(app: web.Application):

From 583081214a4200df3fe3350fdf1c58321ce6a9bd Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 21:19:26 +0100
Subject: [PATCH 634/990] Fix: TapInterface was not serialized on APIs

Instead, something like ``` appeared as a string.
---
 src/aleph/vm/network/interfaces.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index 109a1d8b9..41674c7f8 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -44,6 +44,13 @@ def guest_ipv6(self) -> IPv6Interface:
     def host_ipv6(self) -> IPv6Interface:
         return IPv6Interface(f"{self.ipv6_network[0]}/{self.ipv6_network.prefixlen}")
 
+    def to_dict(self):
+        return {
+            "device": self.device_name,
+            "ipv4": str(self.ip_network),
+            "ipv6": str(self.ipv6_network),
+        }
+
     async def create(self):
         logger.debug("Create network interface")
 

From e52f883faf2352533337a8a8df75ffe6e00f97f3 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 21:20:42 +0100
Subject: [PATCH 635/990] Fix: Browser could not access executions list

Solution: Add a dedicated endpoint, that also exposes the IPv6 of the VMs.
---
 src/aleph/vm/orchestrator/supervisor.py     |  4 +++-
 src/aleph/vm/orchestrator/views/__init__.py | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 067d61e2d..75c5f34f9 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -31,6 +31,7 @@
     about_execution_records,
     about_executions,
     about_login,
+    list_executions,
     notify_allocation,
     run_code_from_hostname,
     run_code_from_path,
@@ -83,7 +84,8 @@ async def allow_cors_on_endpoint(request: web.Request):
 app.add_routes(
     [
         web.get("/about/login", about_login),
-        web.get("/about/executions", about_executions),
+        web.get("/about/executions/list", list_executions),
+        web.get("/about/executions/details", about_executions),
         web.get("/about/executions/records", about_execution_records),
         web.get("/about/usage/system", about_system_usage),
         web.get("/about/config", about_config),
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 1c8e89926..3189acbb8 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -129,6 +129,23 @@ async def about_executions(request: web.Request) -> web.Response:
     )
 
 
+async def list_executions(request: web.Request) -> web.Response:
+    pool: VmPool = request.app["vm_pool"]
+    return web.json_response(
+        {
+            item_hash: {
+                "networking": {
+                    "ipv4": execution.vm.tap_interface.ip_network,
+                    "ipv6": execution.vm.tap_interface.ipv6_network,
+                },
+            }
+            for item_hash, execution in pool.executions.items()
+            if execution.is_running
+        },
+        dumps=dumps_for_json,
+    )
+
+
 async def about_config(request: web.Request) -> web.Response:
     authenticate_request(request)
     return web.json_response(

From 377b317cedc20c96ccfbeaaa5c5e288a066d9fb3 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 21:21:11 +0100
Subject: [PATCH 636/990] Fix: Browsers could not call APIs due to CORS

---
 src/aleph/vm/orchestrator/supervisor.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 75c5f34f9..9b906d87c 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -104,6 +104,13 @@ async def allow_cors_on_endpoint(request: web.Request):
         web.get("/status/check/host", status_check_host),
         web.get("/status/check/version", status_check_version),
         web.get("/status/config", status_public_config),
+        # Allow CORS on endpoints expected to be called from a web browser
+        web.options("/about/executions/list", allow_cors_on_endpoint),
+        web.options("/about/usage/system", allow_cors_on_endpoint),
+        web.options(
+            "/control/machine/{ref}/{view:.*}",
+            allow_cors_on_endpoint,
+        ),
         # Raise an HTTP Error 404 if attempting to access an unknown URL within these paths.
         web.get("/about/{suffix:.*}", lambda _: web.HTTPNotFound()),
         web.get("/control/{suffix:.*}", lambda _: web.HTTPNotFound()),

From 5d3704ece64fd145a6be1cda7ba326eeb05ae024 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 21:21:29 +0100
Subject: [PATCH 637/990] Cleanup: Add comments about HTTP routes

---
 src/aleph/vm/orchestrator/supervisor.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 9b906d87c..ab2763ed6 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -83,12 +83,14 @@ async def allow_cors_on_endpoint(request: web.Request):
 
 app.add_routes(
     [
+        # /about APIs return information about the VM Orchestrator
         web.get("/about/login", about_login),
         web.get("/about/executions/list", list_executions),
         web.get("/about/executions/details", about_executions),
         web.get("/about/executions/records", about_execution_records),
         web.get("/about/usage/system", about_system_usage),
         web.get("/about/config", about_config),
+        # /control APIs are used to control the VMs and access their logs
         web.post("/control/allocations", update_allocations),
         web.post("/control/allocation", notify_allocation),
         web.get("/control/machine/{ref}/logs", stream_logs),
@@ -96,10 +98,7 @@ async def allow_cors_on_endpoint(request: web.Request):
         web.post("/control/machine/{ref}/stop", operate_stop),
         web.post("/control/machine/{ref}/erase", operate_erase),
         web.post("/control/machine/{ref}/reboot", operate_reboot),
-        web.options(
-            "/control/machine/{ref}/{view:.*}",
-            allow_cors_on_endpoint,
-        ),
+        # /status APIs are used to check that the VM Orchestrator is running properly
         web.get("/status/check/fastapi", status_check_fastapi),
         web.get("/status/check/host", status_check_host),
         web.get("/status/check/version", status_check_version),
@@ -115,7 +114,9 @@ async def allow_cors_on_endpoint(request: web.Request):
         web.get("/about/{suffix:.*}", lambda _: web.HTTPNotFound()),
         web.get("/control/{suffix:.*}", lambda _: web.HTTPNotFound()),
         web.get("/status/{suffix:.*}", lambda _: web.HTTPNotFound()),
+        # /static is used to serve static files
         web.static("/static", Path(__file__).parent / "views/static"),
+        # /vm is used to launch VMs on-demand
         web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
         web.route("*", "/{suffix:.*}", run_code_from_hostname),
     ]

From d9f6118eb3b8697a37b1a835360aa4e25b431f1b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 21:42:51 +0100
Subject: [PATCH 638/990] Fix: Floating point comparisons led to issues

When values are very close and expected to be similar, like `0.999998 != 1.000001`
---
 src/aleph/vm/conf.py               | 4 ++++
 src/aleph/vm/orchestrator/tasks.py | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index e77aeed82..f3885f80e 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -227,6 +227,10 @@ class Settings(BaseSettings):
         default="https://api.avax-test.network/ext/bc/C/rpc",
         description="Default to Avalanche Testnet RPC",
     )
+    PAYMENT_BUFFER: Decimal = Field(
+        default=Decimal("0.0000000001"),
+        description="Buffer to add to the required payment to prevent floating point errors",
+    )
 
     SNAPSHOT_FREQUENCY: int = Field(
         default=60,
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 5b10c430f..3e98cab12 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -156,7 +156,7 @@ async def monitor_payments(app: web.Application):
                 required_balance = await compute_required_balance(executions)
                 logger.debug(f"Required balance for Sender {sender} executions: {required_balance}")
                 # Stop executions until the required balance is reached
-                while balance < required_balance:
+                while balance < (required_balance + settings.PAYMENT_BUFFER):
                     last_execution = executions.pop(-1)
                     logger.debug(f"Stopping {last_execution} due to insufficient stream")
                     await pool.stop_vm(last_execution.vm_hash)
@@ -172,7 +172,7 @@ async def monitor_payments(app: web.Application):
                 required_stream = await compute_required_flow(executions)
                 logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
                 # Stop executions until the required stream is reached
-                while stream < required_stream:
+                while stream < (required_stream + settings.PAYMENT_BUFFER):
                     last_execution = executions.pop(-1)
                     logger.debug(f"Stopping {last_execution} due to insufficient stream")
                     await pool.stop_vm(last_execution.vm_hash)

From 9a26d20e7af91e064b5b13627fd8c85d1572f97d Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 22:49:20 +0100
Subject: [PATCH 639/990] Fix: Testnet was used instead of Mainnet

---
 src/aleph/vm/conf.py                 | 12 ++++++++++--
 src/aleph/vm/orchestrator/payment.py |  3 +--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index f3885f80e..c0016d743 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -218,15 +218,23 @@ class Settings(BaseSettings):
     )
     # This address is the ALEPH SuperToken on SuperFluid Testnet
     PAYMENT_SUPER_TOKEN: str = Field(
-        default="0x1290248e01ed2f9f863a9752a8aad396ef3a1b00",
+        default="0xc0Fbc4967259786C743361a5885ef49380473dCF",  # Mainnet
+        # default="0x1290248e01ed2f9f863a9752a8aad396ef3a1b00",  # Testnet
         description="Address of the ALEPH SuperToken on SuperFluid",
     )
     PAYMENT_PRICING_AGGREGATE: str = ""  # TODO: Missing
 
     PAYMENT_RPC_API: HttpUrl = Field(
-        default="https://api.avax-test.network/ext/bc/C/rpc",
+        default="https://api.avax.network/ext/bc/C/rpc",
+        # default="https://api.avax-test.network/ext/bc/C/rpc",
         description="Default to Avalanche Testnet RPC",
     )
+    PAYMENT_CHAIN_ID: int = Field(
+        default=43114,  # Avalanche Mainnet
+        # default=43113,  # Avalanche Fuji Testnet
+        description="Avalanche chain ID",
+    )
+
     PAYMENT_BUFFER: Decimal = Field(
         default=Decimal("0.0000000001"),
         description="Buffer to add to the required payment to prevent floating point errors",
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 3b27c4412..a34d382bf 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -91,8 +91,7 @@ async def get_stream(sender: str, receiver: str, chain) -> Decimal:
     Get the stream of the user from the Superfluid API.
     See https://community.aleph.im/t/pay-as-you-go-using-superfluid/98/11
     """
-    # TODO: Convert chain str to ID
-    chain_id = 43113
+    chain_id = settings.PAYMENT_CHAIN_ID
     superfluid_instance = CFA_V1(settings.PAYMENT_RPC_API, chain_id)
 
     try:

From d59375c7be47a915c476d4e816232f213a124e4c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 25 Jan 2024 23:30:30 +0100
Subject: [PATCH 640/990] Fix: URL to notify of new allocation was confusing

---
 src/aleph/vm/orchestrator/supervisor.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index ab2763ed6..815db59e6 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -92,7 +92,7 @@ async def allow_cors_on_endpoint(request: web.Request):
         web.get("/about/config", about_config),
         # /control APIs are used to control the VMs and access their logs
         web.post("/control/allocations", update_allocations),
-        web.post("/control/allocation", notify_allocation),
+        web.post("/control/allocation/notify", notify_allocation),
         web.get("/control/machine/{ref}/logs", stream_logs),
         web.post("/control/machine/{ref}/expire", operate_expire),
         web.post("/control/machine/{ref}/stop", operate_stop),
@@ -106,6 +106,7 @@ async def allow_cors_on_endpoint(request: web.Request):
         # Allow CORS on endpoints expected to be called from a web browser
         web.options("/about/executions/list", allow_cors_on_endpoint),
         web.options("/about/usage/system", allow_cors_on_endpoint),
+        web.options("/control/allocation/notify", allow_cors_on_endpoint),
         web.options(
             "/control/machine/{ref}/{view:.*}",
             allow_cors_on_endpoint,

From 262ccba6043303e009f2e5104423e5e9ba2aee78 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 26 Jan 2024 00:55:48 +0100
Subject: [PATCH 641/990] Fix: There was no single API to check all IPv6
 connectivity

---
 src/aleph/vm/orchestrator/supervisor.py     |  3 +++
 src/aleph/vm/orchestrator/views/__init__.py | 13 +++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 815db59e6..d8611670b 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -37,6 +37,7 @@
     run_code_from_path,
     status_check_fastapi,
     status_check_host,
+    status_check_ipv6,
     status_check_version,
     status_public_config,
     update_allocations,
@@ -102,6 +103,7 @@ async def allow_cors_on_endpoint(request: web.Request):
         web.get("/status/check/fastapi", status_check_fastapi),
         web.get("/status/check/host", status_check_host),
         web.get("/status/check/version", status_check_version),
+        web.get("/status/check/ipv6", status_check_ipv6),
         web.get("/status/config", status_public_config),
         # Allow CORS on endpoints expected to be called from a web browser
         web.options("/about/executions/list", allow_cors_on_endpoint),
@@ -111,6 +113,7 @@ async def allow_cors_on_endpoint(request: web.Request):
             "/control/machine/{ref}/{view:.*}",
             allow_cors_on_endpoint,
         ),
+        web.options("/status/check/ipv6", allow_cors_on_endpoint),
         # Raise an HTTP Error 404 if attempting to access an unknown URL within these paths.
         web.get("/about/{suffix:.*}", lambda _: web.HTTPNotFound()),
         web.get("/control/{suffix:.*}", lambda _: web.HTTPNotFound()),
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 3189acbb8..ddeb4e7de 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -221,6 +221,19 @@ async def status_check_host(request: web.Request):
     return web.json_response(result, status=result_status)
 
 
+async def status_check_ipv6(request: web.Request):
+    """Check that the platform has IPv6 egress connectivity"""
+    timeout = aiohttp.ClientTimeout(total=2)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        try:
+            vm_ipv6 = await status.check_ipv6(session)
+        except aiohttp.ClientTimeout:
+            vm_ipv6 = False
+
+    result = {"host": await check_host_egress_ipv6(), "vm": vm_ipv6}
+    return web.json_response(result, headers={"Access-Control-Allow-Origin:": "*"})
+
+
 async def status_check_version(request: web.Request):
     """Check if the software is running a version equal or newer than the given one"""
     reference_str: Optional[str] = request.query.get("reference")

From 608f144081e9a456afc8bbed225d6017fcf318de Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 26 Jan 2024 01:27:43 +0100
Subject: [PATCH 642/990] Fix: Diagnostic VMs were stopped due to low balance

---
 src/aleph/vm/conf.py | 1 +
 src/aleph/vm/pool.py | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index c0016d743..5fcbcbdd0 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -282,6 +282,7 @@ class Settings(BaseSettings):
     FAKE_INSTANCE_MESSAGE = Path(abspath(join(__file__, "../../examples/instance_message_from_aleph.json")))
 
     CHECK_FASTAPI_VM_ID = "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af"
+    LEGACY_CHECK_FASTAPI_VM_ID = "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
 
     # Developer options
 
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 78518a341..f61b4dbde 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -273,6 +273,10 @@ def get_executions_by_sender(self, payment_type: PaymentType) -> Dict[str, Dict[
         """Return all executions of the given type, grouped by sender and by chain."""
         executions_by_sender: Dict[str, Dict[str, list[VmExecution]]] = {}
         for vm_hash, execution in self.executions.items():
+            if execution.vm_hash in (settings.CHECK_FASTAPI_VM_ID, settings.LEGACY_CHECK_FASTAPI_VM_ID):
+                # Ignore Diagnostic VM execution
+                continue
+
             if not execution.is_running:
                 # Ignore the execution that is stopping or not running anymore
                 continue

From 445a678c4e9f96863fa848137e77e0fbf0862a3e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 26 Jan 2024 01:44:58 +0100
Subject: [PATCH 643/990] Fix: Public payment settings were not exposed

---
 src/aleph/vm/orchestrator/views/__init__.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index ddeb4e7de..bc4f52b6a 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -285,6 +285,11 @@ async def status_public_config(request: web.Request):
                 "DEBUG_ASYNCIO": settings.DEBUG_ASYNCIO,
                 "EXECUTION_LOG_ENABLED": settings.EXECUTION_LOG_ENABLED,
             },
+            "payment": {
+                "PAYMENT_RECEIVER_ADDRESS": settings.PAYMENT_RECEIVER_ADDRESS,
+                "PAYMENT_SUPER_TOKEN": settings.PAYMENT_SUPER_TOKEN,
+                "PAYMENT_CHAIN_ID": settings.PAYMENT_CHAIN_ID,
+            },
         },
         dumps=dumps_for_json,
     )

From 3f97bf6223128561e67f63056948f2fba259d900 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 26 Jan 2024 01:58:22 +0100
Subject: [PATCH 644/990] Fix: CORS did not work

---
 src/aleph/vm/orchestrator/resources.py      |  4 +---
 src/aleph/vm/orchestrator/views/__init__.py | 21 +++++++++++++--------
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 6589c37ee..5be767dac 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -116,9 +116,7 @@ async def about_system_usage(_: web.Request):
         ),
         properties=get_machine_properties(),
     )
-    return web.json_response(
-        text=usage.json(exclude_none=True),
-    )
+    return web.json_response(text=usage.json(exclude_none=True), headers={"Access-Control-Allow-Origin:": "*"})
 
 
 class Allocation(BaseModel):
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index bc4f52b6a..466f08c60 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -143,6 +143,7 @@ async def list_executions(request: web.Request) -> web.Response:
             if execution.is_running
         },
         dumps=dumps_for_json,
+        headers={"Access-Control-Allow-Origin": "*"},
     )
 
 
@@ -156,10 +157,7 @@ async def about_config(request: web.Request) -> web.Response:
 
 async def about_execution_records(_: web.Request):
     records = await get_execution_records()
-    return web.json_response(
-        records,
-        dumps=dumps_for_json,
-    )
+    return web.json_response(records, dumps=dumps_for_json, headers={"Access-Control-Allow-Origin": "*"})
 
 
 async def index(request: web.Request):
@@ -199,7 +197,9 @@ async def status_check_fastapi(request: web.Request):
                 # "ipv6": await status.check_ipv6(session),
             }
 
-        return web.json_response(result, status=200 if all(result.values()) else 503)
+        return web.json_response(
+            result, status=200 if all(result.values()) else 503, headers={"Access-Control-Allow-Origin": "*"}
+        )
 
 
 async def status_check_host(request: web.Request):
@@ -218,7 +218,7 @@ async def status_check_host(request: web.Request):
         },
     }
     result_status = 200 if all(result["ipv4"].values()) and all(result["ipv6"].values()) else 503
-    return web.json_response(result, status=result_status)
+    return web.json_response(result, status=result_status, headers={"Access-Control-Allow-Origin": "*"})
 
 
 async def status_check_ipv6(request: web.Request):
@@ -231,7 +231,7 @@ async def status_check_ipv6(request: web.Request):
             vm_ipv6 = False
 
     result = {"host": await check_host_egress_ipv6(), "vm": vm_ipv6}
-    return web.json_response(result, headers={"Access-Control-Allow-Origin:": "*"})
+    return web.json_response(result, headers={"Access-Control-Allow-Origin": "*"})
 
 
 async def status_check_version(request: web.Request):
@@ -250,7 +250,11 @@ async def status_check_version(request: web.Request):
         raise web.HTTPServiceUnavailable(text=error.args[0]) from error
 
     if current >= reference:
-        return web.Response(status=200, text=f"Up-to-date: version {current} >= {reference}")
+        return web.Response(
+            status=200,
+            text=f"Up-to-date: version {current} >= {reference}",
+            headers={"Access-Control-Allow-Origin": "*"},
+        )
     else:
         return web.HTTPForbidden(text=f"Outdated: version {current} < {reference}")
 
@@ -292,6 +296,7 @@ async def status_public_config(request: web.Request):
             },
         },
         dumps=dumps_for_json,
+        headers={"Access-Control-Allow-Origin": "*"},
     )
 
 

From c17188051df7df62862e952c02a0de8954bca0e0 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Fri, 26 Jan 2024 08:57:53 +0100
Subject: [PATCH 645/990] Fix: CORS issue on notify endpoint

Co-authored-by: Andres D. Molins 
---
 src/aleph/vm/orchestrator/views/__init__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 466f08c60..6dadb3ff6 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -404,7 +404,9 @@ async def notify_allocation(request: web.Request):
     except JSONDecodeError as error:
         raise web.HTTPBadRequest(reason="Body is not valid JSON") from error
     except ValidationError as error:
-        raise web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code) from error
+        raise web.json_response(
+            data=error.json(), status=web.HTTPBadRequest.status_code, headers={"Access-Control-Allow-Origin": "*"}
+        ) from error
 
     pubsub: PubSub = request.app["pubsub"]
     pool: VmPool = request.app["vm_pool"]
@@ -481,4 +483,5 @@ async def notify_allocation(request: web.Request):
             "errors": {vm_hash: repr(error) for vm_hash, error in scheduling_errors.items()},
         },
         status=status_code,
+        headers={"Access-Control-Allow-Origin": "*"},
     )

From 0988671f4291f32eb5c8fb96922f320c93c938e1 Mon Sep 17 00:00:00 2001
From: Gustavo Delfino 
Date: Sun, 28 Jan 2024 13:56:58 -0500
Subject: [PATCH 646/990] added missing Debian 12 install doc link

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index f57d3c2d9..9b3733300 100644
--- a/README.md
+++ b/README.md
@@ -18,6 +18,7 @@ Writing programs in Python using ASGI compatible frameworks (
 Install Aleph-VM to run an Aleph.im Compute Resource Node easily from official pre-built packages.
 
 - [On Debian 11](./doc/INSTALL-Debian-11.md)
+- [On Debian 12](./doc/INSTALL-Debian-12.md)
 - [On Ubuntu 22.04](./doc/INSTALL-Ubuntu-22.04.md)
 
 ## 2. Install Aleph-VM from source

From 8da7c586fe31d9c0007068e6ba8f2f67c0270f3e Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Tue, 30 Jan 2024 21:23:43 +0100
Subject: [PATCH 647/990] Fix: We are not managing well the holding tier on
 instances, so will be better to disable that control.

---
 src/aleph/vm/orchestrator/tasks.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 3e98cab12..1dcab2345 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -147,20 +147,20 @@ async def monitor_payments(app: web.Application):
     while True:
         await asyncio.sleep(settings.PAYMENT_MONITOR_INTERVAL)
 
-        # Check if the balance held in the wallet is sufficient holder tier resources
-        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.hold).items():
-            for chain, executions in chains.items():
-                balance = await fetch_balance_of_address(sender)
-
-                # Stop executions until the required balance is reached
-                required_balance = await compute_required_balance(executions)
-                logger.debug(f"Required balance for Sender {sender} executions: {required_balance}")
-                # Stop executions until the required balance is reached
-                while balance < (required_balance + settings.PAYMENT_BUFFER):
-                    last_execution = executions.pop(-1)
-                    logger.debug(f"Stopping {last_execution} due to insufficient stream")
-                    await pool.stop_vm(last_execution.vm_hash)
-                    required_balance = await compute_required_balance(executions)
+        # Check if the balance held in the wallet is sufficient holder tier resources (Not do it yet)
+        # for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.hold).items():
+        #     for chain, executions in chains.items():
+        #         balance = await fetch_balance_of_address(sender)
+        #
+        #         # Stop executions until the required balance is reached
+        #         required_balance = await compute_required_balance(executions)
+        #         logger.debug(f"Required balance for Sender {sender} executions: {required_balance}")
+        #         # Stop executions until the required balance is reached
+        #         while balance < (required_balance + settings.PAYMENT_BUFFER):
+        #             last_execution = executions.pop(-1)
+        #             logger.debug(f"Stopping {last_execution} due to insufficient balance")
+        #             await pool.stop_vm(last_execution.vm_hash)
+        #             required_balance = await compute_required_balance(executions)
 
         # Check if the balance held in the wallet is sufficient stream tier resources
         for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():

From 94f6dc0698fcf2364b25395b8fbc490b60e673df Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 1 Feb 2024 12:08:20 +0100
Subject: [PATCH 648/990] Fix: Syntax `|` is not compatible with Python 3.9

---
 src/aleph/vm/controllers/firecracker/instance.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index 427d73a65..266401994 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -167,7 +167,7 @@ def _get_hostname(self) -> str:
     def _encode_user_data(self) -> bytes:
         """Creates user data configuration file for cloud-init tool"""
 
-        ssh_authorized_keys: list[str] | None
+        ssh_authorized_keys: Optional[list[str]]
         if settings.USE_DEVELOPER_SSH_KEYS:
             ssh_authorized_keys = settings.DEVELOPER_SSH_KEYS or []
         else:

From 7f17d6433c2557c701d56f0be4cd176a73168f4c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 1 Feb 2024 12:11:19 +0100
Subject: [PATCH 649/990] Cleanup: Update `black ruff isort` and apply new
 rules

---
 examples/example_django/example_django/settings.py    |  1 +
 examples/example_django/example_django/urls.py        |  1 +
 pyproject.toml                                        |  6 +++---
 src/aleph/vm/controllers/firecracker/instance.py      |  8 +++++---
 src/aleph/vm/controllers/firecracker/program.py       | 11 ++++++-----
 src/aleph/vm/controllers/qemu/cloudinit.py            |  1 +
 src/aleph/vm/hypervisors/firecracker/microvm.py       |  4 ++--
 src/aleph/vm/network/hostnetwork.py                   |  3 +--
 src/aleph/vm/network/ndp_proxy.py                     |  1 +
 .../versions/0001_bbb12a12372e_execution_records.py   |  1 +
 src/aleph/vm/orchestrator/run.py                      |  2 +-
 src/aleph/vm/orchestrator/status.py                   |  1 +
 src/aleph/vm/orchestrator/supervisor.py               |  1 +
 src/aleph/vm/storage.py                               |  1 +
 tests/supervisor/test_jwk.py                          |  6 +++---
 15 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/examples/example_django/example_django/settings.py b/examples/example_django/example_django/settings.py
index 75e53c576..18238fdd3 100644
--- a/examples/example_django/example_django/settings.py
+++ b/examples/example_django/example_django/settings.py
@@ -9,6 +9,7 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/3.2/ref/settings/
 """
+
 import os.path
 from pathlib import Path
 
diff --git a/examples/example_django/example_django/urls.py b/examples/example_django/example_django/urls.py
index 948195d46..82915b927 100644
--- a/examples/example_django/example_django/urls.py
+++ b/examples/example_django/example_django/urls.py
@@ -13,6 +13,7 @@
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
+
 from django.contrib import admin
 from django.urls import include, path
 
diff --git a/pyproject.toml b/pyproject.toml
index 49fd5b81d..bd9f1474b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -105,10 +105,10 @@ python = ["3.9", "3.10", "3.11", "3.12"]
 [tool.hatch.envs.lint]
 detached = true
 dependencies = [
-  "black==23.9.0",
+  "black==24.1.1",
   "mypy==1.6.0",
-  "ruff==0.0.292",
-  "isort==5.12.0",
+  "ruff==0.1.15",
+  "isort==5.13.2",
 ]
 [tool.hatch.envs.lint.scripts]
 typing = "mypy --install-types --non-interactive --ignore-missing-imports --explicit-package-bases {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}"
diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index 266401994..a6e7057c2 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -114,9 +114,11 @@ async def setup(self):
                 mem_size_mib=self.hardware_resources.memory,
             ),
             vsock=Vsock(),
-            network_interfaces=[NetworkInterface(iface_id="eth0", host_dev_name=self.tap_interface.device_name)]
-            if self.enable_networking
-            else [],
+            network_interfaces=(
+                [NetworkInterface(iface_id="eth0", host_dev_name=self.tap_interface.device_name)]
+                if self.enable_networking
+                else []
+            ),
         )
 
     async def wait_for_init(self) -> None:
diff --git a/src/aleph/vm/controllers/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
index 33ed763ca..d26c3ef01 100644
--- a/src/aleph/vm/controllers/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -84,8 +84,7 @@ class ProgramVmConfiguration(MsgpackSerializable):
 
 
 @dataclass
-class ConfigurationPayload(MsgpackSerializable):
-    ...
+class ConfigurationPayload(MsgpackSerializable): ...
 
 
 @dataclass
@@ -315,9 +314,11 @@ async def setup(self):
                 mem_size_mib=self.hardware_resources.memory,
             ),
             vsock=Vsock(),
-            network_interfaces=[NetworkInterface(iface_id="eth0", host_dev_name=self.tap_interface.device_name)]
-            if self.enable_networking
-            else [],
+            network_interfaces=(
+                [NetworkInterface(iface_id="eth0", host_dev_name=self.tap_interface.device_name)]
+                if self.enable_networking
+                else []
+            ),
         )
 
     async def wait_for_init(self) -> None:
diff --git a/src/aleph/vm/controllers/qemu/cloudinit.py b/src/aleph/vm/controllers/qemu/cloudinit.py
index 7a12461ef..686abac31 100644
--- a/src/aleph/vm/controllers/qemu/cloudinit.py
+++ b/src/aleph/vm/controllers/qemu/cloudinit.py
@@ -12,6 +12,7 @@
 
 See also the cloud-localds  man page (1)
 """
+
 import base64
 import json
 from pathlib import Path
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 5085cd618..3874a1acf 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -184,8 +184,8 @@ async def save_configuration_file(self, config: FirecrackerConfig) -> Path:
         with (
             NamedTemporaryFile(delete=False)
             if not self.use_jailer
-            else open(f"{self.jailer_path}/tmp/config.json", "wb") as config_file
-        ):
+            else open(f"{self.jailer_path}/tmp/config.json", "wb")
+        ) as config_file:
             config_file.write(config.json(by_alias=True, exclude_none=True, indent=4).encode())
             config_file.flush()
             config_file_path = Path(config_file.name)
diff --git a/src/aleph/vm/network/hostnetwork.py b/src/aleph/vm/network/hostnetwork.py
index f938f5e3c..0503efdc6 100644
--- a/src/aleph/vm/network/hostnetwork.py
+++ b/src/aleph/vm/network/hostnetwork.py
@@ -31,8 +31,7 @@ def get_ipv6_forwarding_state() -> int:
 
 
 class IPv6Allocator(Protocol):
-    def allocate_vm_ipv6_subnet(self, vm_id: int, vm_hash: ItemHash, vm_type: VmType) -> IPv6Network:
-        ...
+    def allocate_vm_ipv6_subnet(self, vm_id: int, vm_hash: ItemHash, vm_type: VmType) -> IPv6Network: ...
 
 
 class StaticIPv6Allocator(IPv6Allocator):
diff --git a/src/aleph/vm/network/ndp_proxy.py b/src/aleph/vm/network/ndp_proxy.py
index 1cd6b29b4..0af97b7d4 100644
--- a/src/aleph/vm/network/ndp_proxy.py
+++ b/src/aleph/vm/network/ndp_proxy.py
@@ -9,6 +9,7 @@
 To achieve this, we use ndppd. Each time an update is required, we overwrite /etc/ndppd.conf
 and restart the service.
 """
+
 import logging
 from dataclasses import dataclass
 from ipaddress import IPv6Network
diff --git a/src/aleph/vm/orchestrator/migrations/versions/0001_bbb12a12372e_execution_records.py b/src/aleph/vm/orchestrator/migrations/versions/0001_bbb12a12372e_execution_records.py
index 84e2011e1..b10e8477d 100644
--- a/src/aleph/vm/orchestrator/migrations/versions/0001_bbb12a12372e_execution_records.py
+++ b/src/aleph/vm/orchestrator/migrations/versions/0001_bbb12a12372e_execution_records.py
@@ -5,6 +5,7 @@
 Create Date: 2022-09-28 18:52:16.431200
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index b3c380634..ea09f7985 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -187,7 +187,7 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques
         headers.update(
             {
                 "Aleph-Program-ItemHash": execution.vm_hash,
-                "Aleph-Program-Code-Ref": execution.message.code.ref
+                "Aleph-Program-Code-Ref": execution.message.code.ref,
                 # "Aleph-Compute-Vm-Id": str(execution.vm.vm_id),
             }
         )
diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py
index 4aadd4612..bd09257fa 100644
--- a/src/aleph/vm/orchestrator/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -2,6 +2,7 @@
 Used to check that the example_fastapi program works as expected
 in a deployed supervisor.
 """
+
 import logging
 from typing import Any
 
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index d8611670b..a2c5eb1fd 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -5,6 +5,7 @@
 At it's core, it is currently an asynchronous HTTP server using aiohttp, but this may
 evolve in the future.
 """
+
 import asyncio
 import logging
 from collections.abc import Awaitable
diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index 976666d60..350414a0e 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -4,6 +4,7 @@
 In this prototype, it returns a hardcoded example.
 In the future, it should connect to an Aleph node and retrieve the code from there.
 """
+
 import asyncio
 import json
 import logging
diff --git a/tests/supervisor/test_jwk.py b/tests/supervisor/test_jwk.py
index 4bf8c0f2b..cce33a664 100644
--- a/tests/supervisor/test_jwk.py
+++ b/tests/supervisor/test_jwk.py
@@ -30,9 +30,9 @@ async def test_valid_signature(valid_jwk_headers: Dict[str, Any], mocker):
 
 @pytest.mark.asyncio
 async def test_invalid_signature(valid_jwk_headers: Dict[str, Any], mocker):
-    valid_jwk_headers[
-        "X-SignedOperation"
-    ] = '{"time":"2023-07-14T22:14:14.132Z","signature":"96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0ade"}'
+    valid_jwk_headers["X-SignedOperation"] = (
+        '{"time":"2023-07-14T22:14:14.132Z","signature":"96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0ade"}'
+    )
     request = mocker.AsyncMock()
     request.headers = valid_jwk_headers
 

From f46cbaa6697283a4f5bd47244d98b47771d01dc0 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 2 Feb 2024 12:43:35 +0100
Subject: [PATCH 650/990] Fix: Connectivity errors crashed the endpoint

Problem:
The endpoint `/status/check/host` should return a data structure with details about the networking that works or does not.

When IPv6 connectivity failed due to a connection error, the endpoint crashed with an error 500 instead of returning the expected value `false` for that type of connectivity.

Solution: Wrap connectivity errors with a try-catch.
---
 src/aleph/vm/orchestrator/views/host_status.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/host_status.py b/src/aleph/vm/orchestrator/views/host_status.py
index 6ef41c0ee..5ab80bebc 100644
--- a/src/aleph/vm/orchestrator/views/host_status.py
+++ b/src/aleph/vm/orchestrator/views/host_status.py
@@ -23,11 +23,14 @@ async def wrapper(*args: Any, **kwargs: Any) -> bool:
 async def check_ip_connectivity(url: str, socket_family: socket.AddressFamily = socket.AF_UNSPEC) -> bool:
     timeout = aiohttp.ClientTimeout(total=5)
     async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(family=socket_family), timeout=timeout) as session:
-        async with session.get(url) as resp:
-            # We expect the Quad9 endpoints to return a 404 error, but other endpoints may return a 200
-            if resp.status not in (200, 404):
-                resp.raise_for_status()
-            return True
+        try:
+            async with session.get(url) as resp:
+                # We expect the Quad9 endpoints to return a 404 error, but other endpoints may return a 200
+                if resp.status not in (200, 404):
+                    resp.raise_for_status()
+                return True
+        except aiohttp.ClientConnectorError:
+            return False
 
 
 @return_false_on_timeout

From 55c49fa4d9046945a286b0757c71cddef3899a8a Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 2 Feb 2024 15:50:38 +0100
Subject: [PATCH 651/990] Fix: AttributeError when `self.vm` == None

Fixes the following error, reported by node operators.

```
task:  exception=AttributeError("'NoneType' object has no attribute 'to_dict'")>
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/aiohttp/web.py", line 431, in _run_app
    await asyncio.sleep(delay)
  File "/usr/lib/python3.10/asyncio/tasks.py", line 605, in sleep
    return await future
asyncio.exceptions.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/aiohttp/web.py", line 433, in _run_app
    await runner.cleanup()
  File "/opt/aleph-vm/aleph/vm/models.py", line 234, in stop
    await self.record_usage()
  File "/opt/aleph-vm/aleph/vm/models.py", line 275, in record_usage
    pid_info = self.vm.to_dict()
AttributeError: 'NoneType' object has no attribute 'to_dict'
```
---
 src/aleph/vm/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 846b633ea..98e0b50b9 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -330,7 +330,7 @@ async def all_runs_complete(self):
             await self.runs_done_event.wait()
 
     async def save(self):
-        pid_info = self.vm.to_dict()
+        pid_info = self.vm.to_dict() if self.vm else None
         # Handle cases when the process cannot be accessed
         if not self.persistent and pid_info and pid_info.get("process"):
             await save_record(

From af1bb037a4219642cb76be67adcca870f5c28d58 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 2 Feb 2024 16:03:10 +0100
Subject: [PATCH 652/990] Fix: Logs of invalid messages were too verbose

This reduces the log level to warning and does not print the stacktrace.

We also add the `item_hash` in the logs for diagnostic as it was missing.

Previously, logs looked like:
```
2024-02-02 10:16:59,290 | ERROR | Invalid Aleph message:
  [
  {
    "loc": [
      "content",
      "payment"
    ],
    "msg": "extra fields not permitted",
    "type": "value_error.extra"
  }
]
  [ErrorWrapper(exc=ValidationError(model='InstanceContent', errors=[{'loc': ('payment',), 'msg': 'extra fields not permitted', 'type': 'value_error.extra'}]), loc=('content',))]
Traceback (most recent call last):
  File "/opt/aleph-vm/aleph/vm/orchestrator/tasks.py", line 63, in subscribe_via_ws
    yield parse_message(data)
  File "/opt/aleph-vm/aleph_message/models/__init__.py", line 366, in parse_message
    return message_class.parse_obj(message_dict)
  File "pydantic/main.py", line 526, in pydantic.main.BaseModel.parse_obj
    return cls(**obj)
  File "pydantic/main.py", line 341, in pydantic.main.BaseModel.__init__
    raise validation_error
pydantic.error_wrappers.ValidationError: 1 validation error for InstanceMessage
content -> payment
  extra fields not permitted (type=value_error.extra)
```
---
 src/aleph/vm/orchestrator/tasks.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 1dcab2345..8fb4dbcd2 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -74,9 +74,10 @@ async def subscribe_via_ws(url) -> AsyncIterable[AlephMessage]:
                     try:
                         yield parse_message(data)
                     except pydantic.error_wrappers.ValidationError as error:
-                        logger.error(
-                            f"Invalid Aleph message: \n  {error.json()}\n  {error.raw_errors}",
-                            exc_info=True,
+                        item_hash = data.get("item_hash", "ITEM_HASH_NOT_FOUND")
+                        logger.warning(
+                            f"Invalid Aleph message: {item_hash} \n  {error.json()}\n  {error.raw_errors}",
+                            exc_info=False,
                         )
                         continue
                     except KeyError:

From ea8c8494bb2831c8e8f341abad8ba1cbb6e4687b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 2 Feb 2024 13:00:57 +0100
Subject: [PATCH 653/990] Fix: Status check using FastAPI used hardcoded VM ID

Problem: The code was hardcoded to use the same VM ID from the settings for all checks. This did not allow running the same checks on a different VM ID, for example the legacy diagnostic VM with Debian 11, used by existing users.

Solution: Add an argument `vm_id` as optional and default to the value defined in the settings.
---
 src/aleph/vm/orchestrator/status.py         | 68 +++++++++++----------
 src/aleph/vm/orchestrator/views/__init__.py | 32 ++++++----
 2 files changed, 57 insertions(+), 43 deletions(-)

diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py
index bd09257fa..8c9c8064a 100644
--- a/src/aleph/vm/orchestrator/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -8,41 +8,45 @@
 
 from aiohttp import ClientResponseError, ClientSession
 from aiohttp.web_exceptions import HTTPBadGateway, HTTPInternalServerError, HTTPOk
+from aleph_message.models import ItemHash
 
 from aleph.vm.conf import settings
 
 logger = logging.getLogger(__name__)
 
-CHECK_VM_URL = f"http://{settings.SUPERVISOR_HOST}:{settings.SUPERVISOR_PORT}/vm/{settings.CHECK_FASTAPI_VM_ID}"
 
+def make_check_vm_url(vm_id: ItemHash) -> str:
+    return f"http://{settings.SUPERVISOR_HOST}:{settings.SUPERVISOR_PORT}/vm/{vm_id}"
 
-async def get_json_from_vm(session: ClientSession, suffix: str) -> Any:
-    url = f"{CHECK_VM_URL}{suffix}"
+
+async def get_json_from_vm(session: ClientSession, vm_id: ItemHash, suffix: str) -> Any:
+    vm_url = make_check_vm_url(vm_id)
+    url = f"{vm_url}{suffix}"
     async with session.get(url) as resp:
         resp.raise_for_status()
         return await resp.json()
 
 
-async def check_index(session: ClientSession) -> bool:
+async def check_index(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
-        result: dict = await get_json_from_vm(session, "/")
+        result: dict = await get_json_from_vm(session, vm_id, "/")
         assert result["Example"] == "example_fastapi"
         return True
     except ClientResponseError:
         return False
 
 
-async def check_lifespan(session: ClientSession) -> bool:
+async def check_lifespan(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
-        result: dict = await get_json_from_vm(session, "/lifespan")
+        result: dict = await get_json_from_vm(session, vm_id, "/lifespan")
         return result["Lifespan"] is True
     except ClientResponseError:
         return False
 
 
-async def check_environ(session: ClientSession) -> bool:
+async def check_environ(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
-        result: dict = await get_json_from_vm(session, "/environ")
+        result: dict = await get_json_from_vm(session, vm_id, "/environ")
         assert "ALEPH_API_HOST" in result
         assert "ALEPH_API_UNIX_SOCKET" in result
         assert "ALEPH_REMOTE_CRYPTO_HOST" in result
@@ -53,9 +57,9 @@ async def check_environ(session: ClientSession) -> bool:
         return False
 
 
-async def check_messages(session: ClientSession) -> bool:
+async def check_messages(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
-        result: dict = await get_json_from_vm(session, "/messages")
+        result: dict = await get_json_from_vm(session, vm_id, "/messages")
         assert "Messages" in result
         assert "messages" in result["Messages"]
         assert "item_hash" in result["Messages"]["messages"][0]
@@ -64,9 +68,9 @@ async def check_messages(session: ClientSession) -> bool:
         return False
 
 
-async def check_dns(session: ClientSession) -> bool:
+async def check_dns(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
-        result: dict = await get_json_from_vm(session, "/dns")
+        result: dict = await get_json_from_vm(session, vm_id, "/dns")
         assert result["ipv4"]
         assert result["ipv6"]
         return True
@@ -74,18 +78,18 @@ async def check_dns(session: ClientSession) -> bool:
         return False
 
 
-async def check_ipv4(session: ClientSession) -> bool:
+async def check_ipv4(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
-        result: dict = await get_json_from_vm(session, "/ip/4")
+        result: dict = await get_json_from_vm(session, vm_id, "/ip/4")
         assert result["result"] is True
         return True
     except ClientResponseError:
         return False
 
 
-async def check_ipv6(session: ClientSession) -> bool:
+async def check_ipv6(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
-        result: dict = await get_json_from_vm(session, "/ip/6")
+        result: dict = await get_json_from_vm(session, vm_id, "/ip/6")
         assert result["result"] is True
         assert "headers" in result
         return True
@@ -93,9 +97,9 @@ async def check_ipv6(session: ClientSession) -> bool:
         return False
 
 
-async def check_internet(session: ClientSession) -> bool:
+async def check_internet(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
-        result: dict = await get_json_from_vm(session, "/internet")
+        result: dict = await get_json_from_vm(session, vm_id, "/internet")
         assert result["result"] == HTTPOk.status_code
         assert "Server" in result["headers"]
         return True
@@ -103,24 +107,24 @@ async def check_internet(session: ClientSession) -> bool:
         return False
 
 
-async def check_cache(session: ClientSession) -> bool:
+async def check_cache(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
-        result1: bool = await get_json_from_vm(session, "/cache/set/a/42")
+        result1: bool = await get_json_from_vm(session, vm_id, "/cache/set/a/42")
         assert result1 is True
-        result2: int = await get_json_from_vm(session, "/cache/get/a")
+        result2: int = await get_json_from_vm(session, vm_id, "/cache/get/a")
         assert result2 == "42"
-        keys: list[str] = await get_json_from_vm(session, "/cache/keys")
+        keys: list[str] = await get_json_from_vm(session, vm_id, "/cache/keys")
         assert "a" in keys
         return True
     except ClientResponseError:
         return False
 
 
-async def check_persistent_storage(session: ClientSession) -> bool:
+async def check_persistent_storage(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
-        result: dict = await get_json_from_vm(session, "/state/increment")
+        result: dict = await get_json_from_vm(session, vm_id, "/state/increment")
         counter = result["counter"]
-        result_2: dict = await get_json_from_vm(session, "/state/increment")
+        result_2: dict = await get_json_from_vm(session, vm_id, "/state/increment")
         counter_2 = result_2["counter"]
         # Use >= to handle potential concurrency
         assert counter_2 >= counter + 1
@@ -129,24 +133,26 @@ async def check_persistent_storage(session: ClientSession) -> bool:
         return False
 
 
-async def check_error_raised(session: ClientSession) -> bool:
+async def check_error_raised(session: ClientSession, vm_id: ItemHash) -> bool:
+    vm_url = make_check_vm_url(vm_id)
     try:
-        async with session.get(f"{CHECK_VM_URL}/raise") as resp:
+        async with session.get(f"{vm_url}/raise") as resp:
             text = await resp.text()
             return resp.status == HTTPInternalServerError.status_code and "Traceback" in text
     except ClientResponseError:
         return False
 
 
-async def check_crash_and_restart(session: ClientSession) -> bool:
+async def check_crash_and_restart(session: ClientSession, vm_id: ItemHash) -> bool:
     # Crash the VM init.
-    async with session.get(f"{CHECK_VM_URL}/crash") as resp:
+    vm_url = make_check_vm_url(vm_id)
+    async with session.get(f"{vm_url}/crash") as resp:
         if resp.status != HTTPBadGateway.status_code:
             return False
 
     # Try loading the index page. A new execution should be created.
     try:
-        result: dict = await get_json_from_vm(session, "/")
+        result: dict = await get_json_from_vm(session, vm_id, "/")
         assert result["Example"] == "example_fastapi"
         return True
 
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 6dadb3ff6..70309e3ea 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -174,25 +174,33 @@ async def index(request: web.Request):
     return web.Response(content_type="text/html", body=body)
 
 
-async def status_check_fastapi(request: web.Request):
-    retro_compatibility: bool = request.rel_url.query.get("retro-compatibility", "false") == "true"
+async def status_check_fastapi(request: web.Request, vm_id: Optional[ItemHash] = None):
+    """Check that the FastAPI diagnostic VM runs correctly"""
+
+    # Retro-compatibility mode ignores some of the newer checks. It is used to check the status of legacy VMs.
+    retro_compatibility: bool = (
+        vm_id == settings.LEGACY_CHECK_FASTAPI_VM_ID
+        or request.rel_url.query.get("retro-compatibility", "false") == "true"
+    )
+    # Default to the value in the settings.
+    fastapi_vm_id: ItemHash = vm_id or ItemHash(settings.CHECK_FASTAPI_VM_ID)
 
     async with aiohttp.ClientSession() as session:
         result = {
-            "index": await status.check_index(session),
-            "environ": await status.check_environ(session),
-            "messages": await status.check_messages(session),
-            "dns": await status.check_dns(session),
-            "ipv4": await status.check_ipv4(session),
-            "internet": await status.check_internet(session),
-            "cache": await status.check_cache(session),
-            "persistent_storage": await status.check_persistent_storage(session),
-            "error_handling": await status.check_error_raised(session),
+            "index": await status.check_index(session, fastapi_vm_id),
+            "environ": await status.check_environ(session, fastapi_vm_id),
+            "messages": await status.check_messages(session, fastapi_vm_id),
+            "dns": await status.check_dns(session, fastapi_vm_id),
+            "ipv4": await status.check_ipv4(session, fastapi_vm_id),
+            "internet": await status.check_internet(session, fastapi_vm_id),
+            "cache": await status.check_cache(session, fastapi_vm_id),
+            "persistent_storage": await status.check_persistent_storage(session, fastapi_vm_id),
+            "error_handling": await status.check_error_raised(session, fastapi_vm_id),
         }
         if not retro_compatibility:
             # These fields were added in the runtime running Debian 12.
             result = result | {
-                "lifespan": await status.check_lifespan(session),
+                "lifespan": await status.check_lifespan(session, fastapi_vm_id),
                 # IPv6 requires extra work from node operators and is not required yet.
                 # "ipv6": await status.check_ipv6(session),
             }

From 932eba63385c15bb99e2a997102adb0e69a203a8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 2 Feb 2024 13:02:27 +0100
Subject: [PATCH 654/990] Fix: Compatibility with the legacy VM could not be
 checked

Solution: Add a dedicated endpoint that checks that the legacy VM can run on the node, and display it on the index/diagnostic page.
---
 src/aleph/vm/orchestrator/supervisor.py       |  2 +
 src/aleph/vm/orchestrator/views/__init__.py   |  5 +++
 .../vm/orchestrator/views/static/helpers.js   |  7 +++-
 .../orchestrator/views/templates/index.html   | 39 +++++++++++++++++++
 4 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index a2c5eb1fd..e541539a5 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -37,6 +37,7 @@
     run_code_from_hostname,
     run_code_from_path,
     status_check_fastapi,
+    status_check_fastapi_legacy,
     status_check_host,
     status_check_ipv6,
     status_check_version,
@@ -102,6 +103,7 @@ async def allow_cors_on_endpoint(request: web.Request):
         web.post("/control/machine/{ref}/reboot", operate_reboot),
         # /status APIs are used to check that the VM Orchestrator is running properly
         web.get("/status/check/fastapi", status_check_fastapi),
+        web.get("/status/check/fastapi/legacy", status_check_fastapi_legacy),
         web.get("/status/check/host", status_check_host),
         web.get("/status/check/version", status_check_version),
         web.get("/status/check/ipv6", status_check_ipv6),
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 70309e3ea..99eed785e 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -210,6 +210,11 @@ async def status_check_fastapi(request: web.Request, vm_id: Optional[ItemHash] =
         )
 
 
+async def status_check_fastapi_legacy(request: web.Request):
+    """Check that the legacy FastAPI VM runs correctly"""
+    return await status_check_fastapi(request, vm_id=ItemHash(settings.LEGACY_CHECK_FASTAPI_VM_ID))
+
+
 async def status_check_host(request: web.Request):
     """Check that the platform is supported and configured correctly"""
 
diff --git a/src/aleph/vm/orchestrator/views/static/helpers.js b/src/aleph/vm/orchestrator/views/static/helpers.js
index 46d12e4b6..8644a11aa 100644
--- a/src/aleph/vm/orchestrator/views/static/helpers.js
+++ b/src/aleph/vm/orchestrator/views/static/helpers.js
@@ -1,5 +1,8 @@
-async function fetchFastapiCheckStatus () {
-    const q = await fetch('/status/check/fastapi');
+
+// Add optional "legacy" argument to this function
+async function fetchFastapiCheckStatus (legacy = false) {
+    const path = legacy ? '/status/check/fastapi/legacy' : '/status/check/fastapi';
+    const q = await fetch(path);
     let res = {
         status: q.status,
         details: []
diff --git a/src/aleph/vm/orchestrator/views/templates/index.html b/src/aleph/vm/orchestrator/views/templates/index.html
index d7b449f21..96d229864 100644
--- a/src/aleph/vm/orchestrator/views/templates/index.html
+++ b/src/aleph/vm/orchestrator/views/templates/index.html
@@ -69,6 +69,31 @@ 
            Virtualization
            
             
            
         
 
+        
            
+            
            Virtualization (legacy)
            
+            
            
+                Virtualization
+                
+                ...
+                    
+                        
+                        
+                        
+                    
+                
+            
            
+            
            
+                
              
+            
              
+            
+        
              
+
         
              
             
              Host connectivity
              
             
              
@@ -208,6 +233,20 @@ 
              Version
              
             }
         })();
 
+        (async () => {
+            try {
+                const { status, details } = await fetchFastapiCheckStatus(legacy=true);
+                document.getElementById('virtualization-legacy-check').innerHTML = status;
+                if(Object.keys(details).length > 0){
+                    const detailsDiv = document.querySelector("#virtualization-legacy-checks .details ul");
+                    detailsDiv.innerHTML = objectToString(details);
+                    document.querySelector("#virtualization-legacy-checks .help").style.display = "block";
+                }
+            } catch (err) {
+                console.error('Could not fetch api status', err);
+            }
+        })();
+
         (async () => {
             try {
                 const { status, details } = await fetchHostCheckStatus();

From 2b0bdcdb20c256d386be54d8e3d682a5170d0193 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 8 Feb 2024 13:30:46 +0100
Subject: [PATCH 655/990] Problem: Now the operator API just allow to reboot
 well ephemeral VMs, not the persistent ones.

Solution: Use the VM Pool to control reboot, stop and erase operations.
---
 src/aleph/vm/orchestrator/views/operator.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index bc8153a60..052368a9c 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -149,8 +149,7 @@ async def operate_stop(request: web.Request, authenticated_sender: str) -> web.R
 
     if execution.is_running:
         logger.info(f"Stopping {execution.vm_hash}")
-        await execution.stop()
-        execution.persistent = False
+        await pool.stop_vm(execution.vm_hash)
         return web.Response(status=200, body=f"Stopped VM with ref {vm_hash}")
     else:
         return web.Response(status=200, body="Already stopped, nothing to do")
@@ -170,10 +169,13 @@ async def operate_reboot(request: web.Request, authenticated_sender: str) -> web
 
     if execution.is_running:
         logger.info(f"Rebooting {execution.vm_hash}")
-        await pool.stop_vm(vm_hash)
-        await pool.forget_vm(vm_hash)
+        if execution.persistent:
+            await pool.systemd_manager.restart(execution.controller_service)
+        else:
+            await pool.stop_vm(vm_hash)
+            pool.forget_vm(vm_hash)
 
-        await create_vm_execution(vm_hash=vm_hash, pool=pool)
+            await create_vm_execution(vm_hash=vm_hash, pool=pool)
         return web.Response(status=200, body=f"Rebooted VM with ref {vm_hash}")
     else:
         return web.Response(status=200, body="Starting VM (was not running) with ref {vm_hash}")
@@ -194,7 +196,7 @@ async def operate_erase(request: web.Request, authenticated_sender: str) -> web.
     logger.info(f"Erasing {execution.vm_hash}")
 
     # Stop the VM
-    await execution.stop()
+    await pool.stop_vm(execution.vm_hash)
     await pool.forget_vm(execution.vm_hash)
 
     # Delete all data

From 23eeda47bf496de5228a86d79cf9b327f3e47085 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 14 Feb 2024 10:51:26 +0100
Subject: [PATCH 656/990] Fix: Argument `vm_id` was missing.

---
 src/aleph/vm/orchestrator/views/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 99eed785e..b3f9ffad7 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -239,7 +239,7 @@ async def status_check_ipv6(request: web.Request):
     timeout = aiohttp.ClientTimeout(total=2)
     async with aiohttp.ClientSession(timeout=timeout) as session:
         try:
-            vm_ipv6 = await status.check_ipv6(session)
+            vm_ipv6 = await status.check_ipv6(session, vm_id=ItemHash(settings.CHECK_FASTAPI_VM_ID))
         except aiohttp.ClientTimeout:
             vm_ipv6 = False
 

From 5bf3b5d1f98097b47a52ffe86c1c34a2ac672b45 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 13 Feb 2024 18:11:53 +0100
Subject: [PATCH 657/990] Fix: Status check fastapi view could crash due to
 ServerDisconnectedError.

Solution: Catch the ServerDisconnectedError and return an appropriate error message instead.
---
 src/aleph/vm/orchestrator/views/__init__.py | 45 ++++++++++++---------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index b3f9ffad7..414df6ad4 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -185,28 +185,33 @@ async def status_check_fastapi(request: web.Request, vm_id: Optional[ItemHash] =
     # Default to the value in the settings.
     fastapi_vm_id: ItemHash = vm_id or ItemHash(settings.CHECK_FASTAPI_VM_ID)
 
-    async with aiohttp.ClientSession() as session:
-        result = {
-            "index": await status.check_index(session, fastapi_vm_id),
-            "environ": await status.check_environ(session, fastapi_vm_id),
-            "messages": await status.check_messages(session, fastapi_vm_id),
-            "dns": await status.check_dns(session, fastapi_vm_id),
-            "ipv4": await status.check_ipv4(session, fastapi_vm_id),
-            "internet": await status.check_internet(session, fastapi_vm_id),
-            "cache": await status.check_cache(session, fastapi_vm_id),
-            "persistent_storage": await status.check_persistent_storage(session, fastapi_vm_id),
-            "error_handling": await status.check_error_raised(session, fastapi_vm_id),
-        }
-        if not retro_compatibility:
-            # These fields were added in the runtime running Debian 12.
-            result = result | {
-                "lifespan": await status.check_lifespan(session, fastapi_vm_id),
-                # IPv6 requires extra work from node operators and is not required yet.
-                # "ipv6": await status.check_ipv6(session),
+    try:
+        async with aiohttp.ClientSession() as session:
+            result = {
+                "index": await status.check_index(session, fastapi_vm_id),
+                "environ": await status.check_environ(session, fastapi_vm_id),
+                "messages": await status.check_messages(session, fastapi_vm_id),
+                "dns": await status.check_dns(session, fastapi_vm_id),
+                "ipv4": await status.check_ipv4(session, fastapi_vm_id),
+                "internet": await status.check_internet(session, fastapi_vm_id),
+                "cache": await status.check_cache(session, fastapi_vm_id),
+                "persistent_storage": await status.check_persistent_storage(session, fastapi_vm_id),
+                "error_handling": await status.check_error_raised(session, fastapi_vm_id),
             }
-
+            if not retro_compatibility:
+                # These fields were added in the runtime running Debian 12.
+                result = result | {
+                    "lifespan": await status.check_lifespan(session, fastapi_vm_id),
+                    # IPv6 requires extra work from node operators and is not required yet.
+                    # "ipv6": await status.check_ipv6(session),
+                }
+
+            return web.json_response(
+                result, status=200 if all(result.values()) else 503, headers={"Access-Control-Allow-Origin": "*"}
+            )
+    except aiohttp.ServerDisconnectedError as error:
         return web.json_response(
-            result, status=200 if all(result.values()) else 503, headers={"Access-Control-Allow-Origin": "*"}
+            {"error": f"Server disconnected: {error}"}, status=503, headers={"Access-Control-Allow-Origin": "*"}
         )
 
 

From a150a2d87c4a8fda80823d5a61a1d3788b861396 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 13 Feb 2024 18:09:46 +0100
Subject: [PATCH 658/990] Fix: An IndexError was raised if no execution was
 left for this sender.

Solution: Catch and handle the IndexError.
---
 src/aleph/vm/orchestrator/tasks.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 8fb4dbcd2..8b17737d3 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -174,7 +174,11 @@ async def monitor_payments(app: web.Application):
                 logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
                 # Stop executions until the required stream is reached
                 while stream < (required_stream + settings.PAYMENT_BUFFER):
-                    last_execution = executions.pop(-1)
+                    try:
+                        last_execution = executions.pop(-1)
+                    except IndexError:  # Empty list
+                        logger.debug("No execution can be maintained due to insufficient stream")
+                        break
                     logger.debug(f"Stopping {last_execution} due to insufficient stream")
                     await pool.stop_vm(last_execution.vm_hash)
                     required_stream = await compute_required_flow(executions)

From 76e8adc0b66d5e36095cde9a3db5bc78bcd0d80c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 13 Feb 2024 18:07:40 +0100
Subject: [PATCH 659/990] Fix: TypeError: catching classes that do not inherit
 from BaseException is not allowed

This code used the wrong class for the timeout exception.
---
 src/aleph/vm/orchestrator/views/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 414df6ad4..7d516d928 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -245,7 +245,7 @@ async def status_check_ipv6(request: web.Request):
     async with aiohttp.ClientSession(timeout=timeout) as session:
         try:
             vm_ipv6 = await status.check_ipv6(session, vm_id=ItemHash(settings.CHECK_FASTAPI_VM_ID))
-        except aiohttp.ClientTimeout:
+        except TimeoutError:
             vm_ipv6 = False
 
     result = {"host": await check_host_egress_ipv6(), "vm": vm_ipv6}

From a04783d22bf5121ac58555faf0802f94e745a980 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 14 Feb 2024 12:10:17 +0100
Subject: [PATCH 660/990] Fix: `socket.getaddrinfo` does not always return 2
 values

A user reported that this function crashed on his host due to `ValueError: too many values to unpack (expected 2)`.

Solution: Process each returned tuple instead of assuming that two of them are always returned.

This fixes the issue both in the orchestrator and in the diagnostic VM.
---
 examples/example_fastapi/main.py              | 35 ++++++++++++++-----
 .../vm/orchestrator/views/host_status.py      | 29 ++++++++++++---
 2 files changed, 51 insertions(+), 13 deletions(-)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index c7c5fc161..b6861b307 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -7,13 +7,14 @@
 from datetime import datetime
 from os import listdir
 from pathlib import Path
-from typing import Dict
+from typing import Dict, Optional
 
 import aiohttp
 from fastapi import FastAPI
 from fastapi.responses import PlainTextResponse
 from pip._internal.operations.freeze import freeze
 from pydantic import BaseModel
+from starlette.responses import JSONResponse
 
 from aleph.sdk.chains.remote import RemoteAccount
 from aleph.sdk.client import AlephClient, AuthenticatedAlephClient
@@ -91,13 +92,31 @@ async def read_aleph_messages():
 @app.get("/dns")
 async def resolve_dns_hostname():
     """Check if DNS resolution is working."""
-    info_inet, info_inet6 = socket.getaddrinfo("example.org", 80, proto=socket.IPPROTO_TCP)
-    ipv4 = info_inet[4][0]
-    ipv6 = info_inet6[4][0]
-    return {
-        "ipv4": ipv4,
-        "ipv6": ipv6,
-    }
+    hostname = "example.org"
+    ipv4: Optional[str] = None
+    ipv6: Optional[str] = None
+
+    info = socket.getaddrinfo(hostname, 80, proto=socket.IPPROTO_TCP)
+    if not info:
+        logger.error("DNS resolution failed")
+
+    # Iterate over the results to find the IPv4 and IPv6 addresses they may not all be present.
+    # The function returns a list of 5-tuples with the following structure:
+    # (family, type, proto, canonname, sockaddr)
+    for info_tuple in info:
+        if info_tuple[0] == socket.AF_INET:
+            ipv4 = info_tuple[4][0]
+        elif info_tuple[0] == socket.AF_INET6:
+            ipv6 = info_tuple[4][0]
+
+    if ipv4 and not ipv6:
+        logger.warning(f"DNS resolution for {hostname} returned only an IPv4 address")
+    elif ipv6 and not ipv4:
+        logger.warning(f"DNS resolution for {hostname} returned only an IPv6 address")
+
+    result = {"ipv4": ipv4, "ipv6": ipv6}
+    status_code = 200 if len(info) > 1 else 503
+    return JSONResponse(content=result, status_code=status_code)
 
 
 @app.get("/ip/address")
diff --git a/src/aleph/vm/orchestrator/views/host_status.py b/src/aleph/vm/orchestrator/views/host_status.py
index 5ab80bebc..b429a1e2d 100644
--- a/src/aleph/vm/orchestrator/views/host_status.py
+++ b/src/aleph/vm/orchestrator/views/host_status.py
@@ -1,6 +1,6 @@
 import logging
 import socket
-from typing import Any, Awaitable, Callable, Tuple
+from typing import Any, Awaitable, Callable, Optional, Tuple
 
 import aiohttp
 
@@ -45,10 +45,29 @@ async def check_host_egress_ipv6() -> bool:
     return await check_ip_connectivity(settings.CONNECTIVITY_IPV6_URL)
 
 
-async def resolve_dns(hostname: str) -> Tuple[str, str]:
-    info_inet, info_inet6 = socket.getaddrinfo(hostname, 80, proto=socket.IPPROTO_TCP)
-    ipv4 = info_inet[4][0]
-    ipv6 = info_inet6[4][0]
+async def resolve_dns(hostname: str) -> Tuple[Optional[str], Optional[str]]:
+    """Resolve a hostname to an IPv4 and IPv6 address."""
+    ipv4: Optional[str] = None
+    ipv6: Optional[str] = None
+
+    info = socket.getaddrinfo(hostname, 80, proto=socket.IPPROTO_TCP)
+    if not info:
+        logger.error("DNS resolution failed")
+
+    # Iterate over the results to find the IPv4 and IPv6 addresses they may not all be present.
+    # The function returns a list of 5-tuples with the following structure:
+    # (family, type, proto, canonname, sockaddr)
+    for info_tuple in info:
+        if info_tuple[0] == socket.AF_INET:
+            ipv4 = info_tuple[4][0]
+        elif info_tuple[0] == socket.AF_INET6:
+            ipv6 = info_tuple[4][0]
+
+    if ipv4 and not ipv6:
+        logger.warning(f"DNS resolution for {hostname} returned only an IPv4 address")
+    elif ipv6 and not ipv4:
+        logger.warning(f"DNS resolution for {hostname} returned only an IPv6 address")
+
     return ipv4, ipv6
 
 

From efb5b304c8df1a0da36bcf2c98f026f29b70ce7d Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Tue, 20 Feb 2024 15:05:17 +0100
Subject: [PATCH 661/990] Added CORS support on supervisor endpoints (#542)

* Fix: Implemented CORS support on supervisor endpoints.

* Fix: Apply code review suggestions.

* Fix: Apply same new decorator to operator endpoints.
---
 packaging/Makefile                          |  2 +-
 pyproject.toml                              |  3 ++-
 src/aleph/vm/orchestrator/supervisor.py     | 16 ++++------------
 src/aleph/vm/orchestrator/views/__init__.py | 20 ++++++++++++++++----
 src/aleph/vm/orchestrator/views/operator.py |  7 +++++++
 src/aleph/vm/utils.py                       | 12 ++++++++++++
 6 files changed, 42 insertions(+), 18 deletions(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 0226808c9..a1d44026e 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.2' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]' 'aiosqlite==0.19.0' 'alembic==1.13.1'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.2' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo
diff --git a/pyproject.toml b/pyproject.toml
index bd9f1474b..41d2198bf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,7 +48,8 @@ dependencies = [
   "superfluid~=0.2.1",
   "sqlalchemy[asyncio]",
   "aiosqlite==0.19.0",
-  "alembic==1.13.1"
+  "alembic==1.13.1",
+  "aiohttp_cors~=0.7.0",
 ]
 
 [project.urls]
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index e541539a5..82f6979cb 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -13,6 +13,7 @@
 from secrets import token_urlsafe
 from typing import Callable
 
+import aiohttp_cors
 from aiohttp import web
 
 from aleph.vm.conf import settings
@@ -68,9 +69,6 @@ async def server_version_middleware(
     return resp
 
 
-app = web.Application(middlewares=[server_version_middleware])
-
-
 async def allow_cors_on_endpoint(request: web.Request):
     """Allow CORS on endpoints that VM owners use to control their machine."""
     return web.Response(
@@ -84,6 +82,9 @@ async def allow_cors_on_endpoint(request: web.Request):
     )
 
 
+app = web.Application(middlewares=[server_version_middleware])
+cors = aiohttp_cors.setup(app)
+
 app.add_routes(
     [
         # /about APIs return information about the VM Orchestrator
@@ -108,15 +109,6 @@ async def allow_cors_on_endpoint(request: web.Request):
         web.get("/status/check/version", status_check_version),
         web.get("/status/check/ipv6", status_check_ipv6),
         web.get("/status/config", status_public_config),
-        # Allow CORS on endpoints expected to be called from a web browser
-        web.options("/about/executions/list", allow_cors_on_endpoint),
-        web.options("/about/usage/system", allow_cors_on_endpoint),
-        web.options("/control/allocation/notify", allow_cors_on_endpoint),
-        web.options(
-            "/control/machine/{ref}/{view:.*}",
-            allow_cors_on_endpoint,
-        ),
-        web.options("/status/check/ipv6", allow_cors_on_endpoint),
         # Raise an HTTP Error 404 if attempting to access an unknown URL within these paths.
         web.get("/about/{suffix:.*}", lambda _: web.HTTPNotFound()),
         web.get("/control/{suffix:.*}", lambda _: web.HTTPNotFound()),
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 7d516d928..a60423605 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -5,6 +5,7 @@
 from hashlib import sha256
 from json import JSONDecodeError
 from pathlib import Path
+from secrets import compare_digest
 from string import Template
 from typing import Optional
 
@@ -46,6 +47,7 @@
 from aleph.vm.utils import (
     HostNotFoundError,
     b32_to_b16,
+    cors_allow_all,
     dumps_for_json,
     get_ref_from_dns,
 )
@@ -110,9 +112,10 @@ def authenticate_request(request: web.Request) -> None:
         raise web.HTTPUnauthorized(reason="Invalid token", text="401 Invalid token")
 
 
+@cors_allow_all
 async def about_login(request: web.Request) -> web.Response:
     token = request.query.get("token")
-    if token == request.app["secret_token"]:
+    if compare_digest(token, request.app["secret_token"]):
         response = web.HTTPFound("/about/config")
         response.cookies["token"] = token
         return response
@@ -120,6 +123,7 @@ async def about_login(request: web.Request) -> web.Response:
         return web.json_response({"success": False}, status=401)
 
 
+@cors_allow_all
 async def about_executions(request: web.Request) -> web.Response:
     authenticate_request(request)
     pool: VmPool = request.app["vm_pool"]
@@ -129,6 +133,7 @@ async def about_executions(request: web.Request) -> web.Response:
     )
 
 
+@cors_allow_all
 async def list_executions(request: web.Request) -> web.Response:
     pool: VmPool = request.app["vm_pool"]
     return web.json_response(
@@ -143,10 +148,10 @@ async def list_executions(request: web.Request) -> web.Response:
             if execution.is_running
         },
         dumps=dumps_for_json,
-        headers={"Access-Control-Allow-Origin": "*"},
     )
 
 
+@cors_allow_all
 async def about_config(request: web.Request) -> web.Response:
     authenticate_request(request)
     return web.json_response(
@@ -155,9 +160,10 @@ async def about_config(request: web.Request) -> web.Response:
     )
 
 
+@cors_allow_all
 async def about_execution_records(_: web.Request):
     records = await get_execution_records()
-    return web.json_response(records, dumps=dumps_for_json, headers={"Access-Control-Allow-Origin": "*"})
+    return web.json_response(records, dumps=dumps_for_json)
 
 
 async def index(request: web.Request):
@@ -174,6 +180,7 @@ async def index(request: web.Request):
     return web.Response(content_type="text/html", body=body)
 
 
+@cors_allow_all
 async def status_check_fastapi(request: web.Request, vm_id: Optional[ItemHash] = None):
     """Check that the FastAPI diagnostic VM runs correctly"""
 
@@ -215,11 +222,13 @@ async def status_check_fastapi(request: web.Request, vm_id: Optional[ItemHash] =
         )
 
 
+@cors_allow_all
 async def status_check_fastapi_legacy(request: web.Request):
     """Check that the legacy FastAPI VM runs correctly"""
     return await status_check_fastapi(request, vm_id=ItemHash(settings.LEGACY_CHECK_FASTAPI_VM_ID))
 
 
+@cors_allow_all
 async def status_check_host(request: web.Request):
     """Check that the platform is supported and configured correctly"""
 
@@ -239,6 +248,7 @@ async def status_check_host(request: web.Request):
     return web.json_response(result, status=result_status, headers={"Access-Control-Allow-Origin": "*"})
 
 
+@cors_allow_all
 async def status_check_ipv6(request: web.Request):
     """Check that the platform has IPv6 egress connectivity"""
     timeout = aiohttp.ClientTimeout(total=2)
@@ -252,6 +262,7 @@ async def status_check_ipv6(request: web.Request):
     return web.json_response(result, headers={"Access-Control-Allow-Origin": "*"})
 
 
+@cors_allow_all
 async def status_check_version(request: web.Request):
     """Check if the software is running a version equal or newer than the given one"""
     reference_str: Optional[str] = request.query.get("reference")
@@ -277,6 +288,7 @@ async def status_check_version(request: web.Request):
         return web.HTTPForbidden(text=f"Outdated: version {current} < {reference}")
 
 
+@cors_allow_all
 async def status_public_config(request: web.Request):
     """Expose the public fields from the configuration"""
     return web.json_response(
@@ -414,6 +426,7 @@ async def update_allocations(request: web.Request):
     )
 
 
+@cors_allow_all
 async def notify_allocation(request: web.Request):
     """Notify instance allocation, only used for Pay as you Go feature"""
     try:
@@ -501,5 +514,4 @@ async def notify_allocation(request: web.Request):
             "errors": {vm_hash: repr(error) for vm_hash, error in scheduling_errors.items()},
         },
         status=status_code,
-        headers={"Access-Control-Allow-Origin": "*"},
     )
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 052368a9c..726886fd6 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -5,6 +5,7 @@
 import aiohttp.web_exceptions
 from aiohttp import web
 from aiohttp.web_urldispatcher import UrlMappingMatchInfo
+from aiohttp_cors import ResourceOptions, custom_cors
 from aleph_message.exceptions import UnknownHashError
 from aleph_message.models import ItemHash
 from aleph_message.models.execution import BaseExecutableContent
@@ -17,6 +18,7 @@
     require_jwk_authentication,
 )
 from aleph.vm.pool import VmPool
+from aleph.vm.utils import cors_allow_all
 
 logger = logging.getLogger(__name__)
 
@@ -50,6 +52,7 @@ def is_sender_authorized(authenticated_sender: str, message: BaseExecutableConte
         return False
 
 
+@cors_allow_all
 async def stream_logs(request: web.Request) -> web.StreamResponse:
     """Stream the logs of a VM.
 
@@ -105,6 +108,7 @@ async def authenticate_for_vm_or_403(execution, request, vm_hash, ws):
     raise web.HTTPForbidden(body="Unauthorized sender")
 
 
+@cors_allow_all
 @require_jwk_authentication
 async def operate_expire(request: web.Request, authenticated_sender: str) -> web.Response:
     """Stop the virtual machine, smoothly if possible.
@@ -131,6 +135,7 @@ async def operate_expire(request: web.Request, authenticated_sender: str) -> web
     return web.Response(status=200, body=f"Expiring VM with ref {vm_hash} in {timeout} seconds")
 
 
+@cors_allow_all
 @require_jwk_authentication
 async def operate_stop(request: web.Request, authenticated_sender: str) -> web.Response:
     """Stop the virtual machine, smoothly if possible."""
@@ -155,6 +160,7 @@ async def operate_stop(request: web.Request, authenticated_sender: str) -> web.R
         return web.Response(status=200, body="Already stopped, nothing to do")
 
 
+@cors_allow_all
 @require_jwk_authentication
 async def operate_reboot(request: web.Request, authenticated_sender: str) -> web.Response:
     """
@@ -181,6 +187,7 @@ async def operate_reboot(request: web.Request, authenticated_sender: str) -> web
         return web.Response(status=200, body="Starting VM (was not running) with ref {vm_hash}")
 
 
+@cors_allow_all
 @require_jwk_authentication
 async def operate_erase(request: web.Request, authenticated_sender: str) -> web.Response:
     """Delete all data stored by a virtual machine.
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index f190dceea..0da684386 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -14,6 +14,7 @@
 
 import aiodns
 import msgpack
+from aiohttp_cors import ResourceOptions, custom_cors
 from aleph_message.models import ExecutableContent, InstanceContent, ProgramContent
 from aleph_message.models.execution.base import MachineType
 from eth_typing import HexAddress, HexStr
@@ -31,6 +32,17 @@ def get_message_executable_content(message_dict: Dict) -> ExecutableContent:
         raise ValueError(f"Unknown message type {message_dict['type']}")
 
 
+def cors_allow_all(function):
+    default_config = {
+        "*": ResourceOptions(
+            allow_credentials=True,
+            allow_headers="*",
+            expose_headers="*",
+        )
+    }
+    return custom_cors(config=default_config)(function)
+
+
 class MsgpackSerializable:
     def __post_init__(self, *args, **kwargs):
         if not is_dataclass(self):

From a470f4e9bc8bbaf4ff7f1816fd62396fb6ff008d Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Tue, 20 Feb 2024 16:17:56 +0100
Subject: [PATCH 662/990] Detect already running Persistent VMs (#541)

Problem: Persistent VMs running were not detected after the orchestrator reboot.

Solution: Don't delete the entire table on the start process.

* Fix: Avoid to wait so long time to stop guest_api process. Put a timeout of 10 seconds.
* Fix: Avoid final `cannot unpack non-iterable VmExecution object` errors giving an empty list instead None value.
* Fix: If the execution already exist, only continue, not break the loop.

---------

Co-authored-by: Andres D. Molins 
---
 src/aleph/vm/controllers/firecracker/executable.py |  5 ++++-
 src/aleph/vm/orchestrator/metrics.py               |  1 -
 src/aleph/vm/orchestrator/views/operator.py        |  4 ++--
 src/aleph/vm/pool.py                               | 12 +++++++-----
 src/aleph/vm/utils.py                              |  7 ++-----
 5 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index 358524d36..b04675b37 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -306,8 +306,11 @@ async def start_guest_api(self):
         logger.debug(f"started guest API for {self.vm_id}")
 
     async def stop_guest_api(self):
-        if self.guest_api_process and self.guest_api_process._popen:
+        if self.guest_api_process and self.guest_api_process.is_alive():
             self.guest_api_process.terminate()
+            await asyncio.sleep(5)
+            if self.guest_api_process.is_alive():
+                self.guest_api_process.kill()
 
     async def teardown(self):
         if self.fvm:
diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index 0578845a8..35563f0d2 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -42,7 +42,6 @@ def setup_engine():
 
 async def create_tables(engine: Engine):
     async with engine.begin() as conn:
-        await conn.run_sync(Base.metadata.drop_all)
         await conn.run_sync(Base.metadata.create_all)
 
 
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 726886fd6..876415d78 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -176,7 +176,7 @@ async def operate_reboot(request: web.Request, authenticated_sender: str) -> web
     if execution.is_running:
         logger.info(f"Rebooting {execution.vm_hash}")
         if execution.persistent:
-            await pool.systemd_manager.restart(execution.controller_service)
+            pool.systemd_manager.restart(execution.controller_service)
         else:
             await pool.stop_vm(vm_hash)
             pool.forget_vm(vm_hash)
@@ -204,7 +204,7 @@ async def operate_erase(request: web.Request, authenticated_sender: str) -> web.
 
     # Stop the VM
     await pool.stop_vm(execution.vm_hash)
-    await pool.forget_vm(execution.vm_hash)
+    pool.forget_vm(execution.vm_hash)
 
     # Delete all data
     if execution.resources is not None:
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index f61b4dbde..eb9d7ec48 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -189,7 +189,6 @@ async def stop_persistent_execution(self, execution):
         assert execution.persistent, "Execution isn't persistent"
         self.systemd_manager.stop_and_disable(execution.controller_service)
         await execution.stop()
-        execution.persistent = False
 
     def forget_vm(self, vm_hash: ItemHash) -> None:
         """Remove a VM from the executions pool.
@@ -209,7 +208,7 @@ async def _load_persistent_executions(self):
         for saved_execution in saved_executions:
             # Prevent to load the same execution twice
             if self.executions.get(saved_execution.vm_hash):
-                break
+                continue
 
             vm_id = saved_execution.vm_id
             message_dict = json.loads(saved_execution.message)
@@ -249,25 +248,28 @@ async def stop(self):
         await asyncio.gather(*(execution.stop() for vm_hash, execution in self.get_ephemeral_executions()))
 
     def get_ephemeral_executions(self) -> Iterable[VmExecution]:
-        return (
+        executions = (
             execution
             for _vm_hash, execution in self.executions.items()
             if execution.is_running and not execution.persistent
         )
+        return executions or []
 
     def get_persistent_executions(self) -> Iterable[VmExecution]:
-        return (
+        executions = (
             execution
             for _vm_hash, execution in self.executions.items()
             if execution.is_running and execution.persistent
         )
+        return executions or []
 
     def get_instance_executions(self) -> Iterable[VmExecution]:
-        return (
+        executions = (
             execution
             for _vm_hash, execution in self.executions.items()
             if execution.is_running and execution.is_instance
         )
+        return executions or []
 
     def get_executions_by_sender(self, payment_type: PaymentType) -> Dict[str, Dict[str, list[VmExecution]]]:
         """Return all executions of the given type, grouped by sender and by chain."""
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index 0da684386..1a0daf0ae 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -16,7 +16,6 @@
 import msgpack
 from aiohttp_cors import ResourceOptions, custom_cors
 from aleph_message.models import ExecutableContent, InstanceContent, ProgramContent
-from aleph_message.models.execution.base import MachineType
 from eth_typing import HexAddress, HexStr
 from eth_utils import hexstr_if_str, is_address, to_hex
 
@@ -24,12 +23,10 @@
 
 
 def get_message_executable_content(message_dict: Dict) -> ExecutableContent:
-    if message_dict["type"] == MachineType.vm_function:
+    try:
         return ProgramContent.parse_obj(message_dict)
-    elif message_dict["type"] == MachineType.vm_instance:
+    except ValueError as error:
         return InstanceContent.parse_obj(message_dict)
-    else:
-        raise ValueError(f"Unknown message type {message_dict['type']}")
 
 
 def cors_allow_all(function):

From e37bb6f4761648e6b138f1745905e0757ffedd31 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 20 Feb 2024 17:52:34 +0100
Subject: [PATCH 663/990] Cleanup: Minor code cleanup and refactoring (#546)

---
 src/aleph/vm/orchestrator/views/operator.py |  4 ++-
 src/aleph/vm/pool.py                        | 38 ++++++++++++++-------
 2 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 876415d78..7e9482883 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -204,7 +204,9 @@ async def operate_erase(request: web.Request, authenticated_sender: str) -> web.
 
     # Stop the VM
     await pool.stop_vm(execution.vm_hash)
-    pool.forget_vm(execution.vm_hash)
+    if execution.vm_hash in pool.executions:
+        logger.warning(f"VM {execution.vm_hash} was not stopped properly, forgetting it anyway")
+        pool.forget_vm(execution.vm_hash)
 
     # Delete all data
     if execution.resources is not None:
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index eb9d7ec48..0c1673cee 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import json
 import logging
@@ -128,11 +130,7 @@ async def create_a_vm(
             self.forget_vm(vm_hash)
             raise
 
-        async def forget_on_stop(stop_event: asyncio.Event):
-            await stop_event.wait()
-            self.forget_vm(vm_hash)
-
-        asyncio.create_task(forget_on_stop(stop_event=execution.stop_event))
+        self._schedule_forget_on_stop(execution)
 
         return execution
 
@@ -184,7 +182,7 @@ async def stop_vm(self, vm_hash: ItemHash) -> Optional[VmExecution]:
         else:
             return None
 
-    async def stop_persistent_execution(self, execution):
+    async def stop_persistent_execution(self, execution: VmExecution):
         """Stop persistent VMs in the pool."""
         assert execution.persistent, "Execution isn't persistent"
         self.systemd_manager.stop_and_disable(execution.controller_service)
@@ -202,31 +200,45 @@ def forget_vm(self, vm_hash: ItemHash) -> None:
         except KeyError:
             pass
 
+    def _schedule_forget_on_stop(self, execution: VmExecution):
+        """Create a task that will remove the VM from the pool after it stops."""
+
+        async def forget_on_stop(stop_event: asyncio.Event):
+            await stop_event.wait()
+            self.forget_vm(execution.vm_hash)
+
+        _ = asyncio.create_task(forget_on_stop(stop_event=execution.stop_event))
+
     async def _load_persistent_executions(self):
         """Load persistent executions from the database."""
         saved_executions = await get_execution_records()
         for saved_execution in saved_executions:
-            # Prevent to load the same execution twice
-            if self.executions.get(saved_execution.vm_hash):
+            vm_hash = ItemHash(saved_execution.vm_hash)
+
+            if vm_hash in self.executions:
+                # The execution is already loaded, skip it
                 continue
 
             vm_id = saved_execution.vm_id
+
             message_dict = json.loads(saved_execution.message)
             original_dict = json.loads(saved_execution.original_message)
+
             execution = VmExecution(
-                vm_hash=saved_execution.vm_hash,
+                vm_hash=vm_hash,
                 message=get_message_executable_content(message_dict),
-                original=get_message_executable_content(message_dict),
+                original=get_message_executable_content(original_dict),
                 snapshot_manager=self.snapshot_manager,
                 systemd_manager=self.systemd_manager,
                 persistent=saved_execution.persistent,
             )
+
             if execution.is_running:
                 # TODO: Improve the way that we re-create running execution
                 await execution.prepare()
                 if self.network:
                     vm_type = VmType.from_message_content(execution.message)
-                    tap_interface = await self.network.prepare_tap(vm_id, execution.vm_hash, vm_type)
+                    tap_interface = await self.network.prepare_tap(vm_id, vm_hash, vm_type)
                 else:
                     tap_interface = None
 
@@ -235,7 +247,9 @@ async def _load_persistent_executions(self):
                 execution.ready_event.set()
                 execution.times.started_at = datetime.now(tz=timezone.utc)
 
-                self.executions[execution.vm_hash] = execution
+                self._schedule_forget_on_stop(execution)
+
+                self.executions[vm_hash] = execution
             else:
                 execution.uuid = saved_execution.uuid
                 await execution.record_usage()

From 83b4f9bbdbebc960875b57f2cf4df851891d34e6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 20 Feb 2024 17:57:15 +0100
Subject: [PATCH 664/990] Fix: Linux kernel from package could not be on a
 different device

If the Linux kernel provided is was another device than the execution root, creating a hardlink within the jailer directory would fail.

Solution: Copy the kernel to the execution root during setup.
---
 src/aleph/vm/conf.py  | 15 ++++++++++++++-
 src/aleph/vm/utils.py | 18 +++++++++++++++++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 5fcbcbdd0..141f790dc 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -2,6 +2,7 @@
 import logging
 import os
 import re
+import shutil
 from collections.abc import Iterable
 from decimal import Decimal
 from enum import Enum
@@ -15,7 +16,7 @@
 from pydantic.env_settings import DotenvType, env_file_sentinel
 from pydantic.typing import StrPath
 
-from aleph.vm.utils import is_command_available
+from aleph.vm.utils import file_hashes_differ, is_command_available
 
 logger = logging.getLogger(__name__)
 
@@ -356,6 +357,18 @@ def setup(self):
 
         os.makedirs(self.EXECUTION_ROOT, exist_ok=True)
 
+        # If the Linux kernel provided is on another device than the execution root,
+        # copy it to the execution root to allow hardlink creation within jailer directories.
+        if os.stat(self.LINUX_PATH).st_dev != os.stat(self.EXECUTION_ROOT).st_dev:
+            logger.info("The Linux kernel is on another device than the execution root. Creating a copy.")
+            linux_path_on_device = self.EXECUTION_ROOT / "vmlinux.bin"
+
+            # Only copy if the hash of the file differ.
+            if file_hashes_differ(self.LINUX_PATH, linux_path_on_device):
+                shutil.copy(self.LINUX_PATH, linux_path_on_device)
+
+            self.LINUX_PATH = linux_path_on_device
+
         os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True)
         os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True)
 
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index 1a0daf0ae..43ed8e306 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -10,7 +10,7 @@
 from dataclasses import is_dataclass
 from pathlib import Path
 from shutil import disk_usage
-from typing import Any, Dict, Optional
+from typing import Any, Callable, Dict, Optional
 
 import aiodns
 import msgpack
@@ -195,3 +195,19 @@ def to_normalized_address(value: str) -> HexAddress:
         return HexAddress(HexStr(hex_address))
     else:
         raise ValueError("Unknown format {}, attempted to normalize to {}".format(value, hex_address))
+
+
+def md5sum(file_path: Path) -> str:
+    """Calculate the MD5 hash of a file. Externalize to the `md5sum` command for better performance."""
+    return subprocess.check_output(["md5sum", file_path], text=True).split()[0]
+
+
+def file_hashes_differ(source: Path, destination: Path, checksum: Callable[[Path], str] = md5sum) -> bool:
+    """Check if the MD5 hash of two files differ."""
+    if not source.exists():
+        raise FileNotFoundError("Source file does not exist: {}".format(source))
+
+    if not destination.exists():
+        return True
+
+    return checksum(source) != checksum(destination)

From 87fc8d7144ec59198d9d4677ebf97265b1690c62 Mon Sep 17 00:00:00 2001
From: aliel 
Date: Wed, 21 Feb 2024 17:24:16 +0100
Subject: [PATCH 665/990] FIX

* add missing packages (nftables)
---
 .github/workflows/code-quality.yml  | 2 +-
 docker/vm_supervisor-dev.dockerfile | 2 +-
 packaging/aleph-vm/DEBIAN/control   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
index d1a25d9d1..b476f6c6a 100644
--- a/.github/workflows/code-quality.yml
+++ b/.github/workflows/code-quality.yml
@@ -16,7 +16,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get -y upgrade
-          sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema
+          sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables
           pip install --upgrade typing-extensions types-PyYAML
 
       - name: Install required Python packages
diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 07f0e6d50..7e1cbaed4 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -5,7 +5,7 @@ FROM debian:bullseye
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     sudo acl curl squashfs-tools git \
     python3 python3-aiohttp python3-alembic python3-msgpack python3-pip python3-aiodns python3-aioredis\
-    python3-nftables python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging python3-cpuinfo ndppd \
+    python3-nftables python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging python3-cpuinfo ndppd nftables \
     && rm -rf /var/lib/apt/lists/*
 
 RUN useradd jailman
diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index b91a89fa3..45aa6bd65 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils,python3-systemd,python3-dbus,btrfs-progs
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils,python3-systemd,python3-dbus,btrfs-progs,nftables
 Section: aleph-im
 Priority: Extra

From 0453d1419efb01b4f020090a2056e9abe83153ce Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 29 Feb 2024 12:26:05 +0100
Subject: [PATCH 666/990] Problem: Diagnostic VM API isn't accessible through
 frontend.

Solution: Implement CORS policy headers on diagnostic VM endpoints.
---
 examples/example_fastapi/main.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index b6861b307..0f9195f7e 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -11,6 +11,7 @@
 
 import aiohttp
 from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import PlainTextResponse
 from pip._internal.operations.freeze import freeze
 from pydantic import BaseModel
@@ -27,6 +28,13 @@
 
 http_app = FastAPI()
 app = AlephApp(http_app=http_app)
+app.add_middleware(
+    CORSMiddleware,
+    allow_credentials=True,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
 cache = VmCache()
 
 startup_lifespan_executed: bool = False

From a20fc4981b11734d5f5b62d954fd23a0fca3616a Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Wed, 28 Feb 2024 15:54:49 +0100
Subject: [PATCH 667/990] Fix: Check request token exists before compare it to
 secret token of app.

---
 src/aleph/vm/orchestrator/views/__init__.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index a60423605..91ad43340 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -114,10 +114,12 @@ def authenticate_request(request: web.Request) -> None:
 
 @cors_allow_all
 async def about_login(request: web.Request) -> web.Response:
-    token = request.query.get("token")
-    if compare_digest(token, request.app["secret_token"]):
+    secret_token = request.app["secret_token"]
+    request_token = request.query.get("token")
+
+    if request_token and secret_token and compare_digest(request_token, secret_token):
         response = web.HTTPFound("/about/config")
-        response.cookies["token"] = token
+        response.cookies["token"] = request_token
         return response
     else:
         return web.json_response({"success": False}, status=401)

From 61f0b8645f9dd13be14fb96a200e11642e819438 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 20 Feb 2024 14:48:00 +0100
Subject: [PATCH 668/990] Fix: Invalid references in unit tests failed imports

Due to previous refactoring.
---
 tests/supervisor/test_jwk.py                    | 4 ++--
 tests/supervisor/test_resolvectl_dns_servers.py | 9 +++------
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/tests/supervisor/test_jwk.py b/tests/supervisor/test_jwk.py
index cce33a664..b34a78ae0 100644
--- a/tests/supervisor/test_jwk.py
+++ b/tests/supervisor/test_jwk.py
@@ -14,7 +14,7 @@
 
 @pytest.fixture
 def valid_jwk_headers(mocker):
-    mocker.patch("aleph.vm.orchestrator.views.operator.is_token_still_valid", lambda timestamp: True)
+    mocker.patch("aleph.vm.orchestrator.views.authentication.is_token_still_valid", lambda timestamp: True)
     return {
         "X-SignedPubKey": '{"payload":"7b227075626b6579223a7b22616c67223a224553323536222c22637276223a22502d323536222c22657874223a747275652c226b65795f6f7073223a5b22766572696679225d2c226b7479223a224543222c2278223a224b65763844614d7356454673365a6b4679525a4272796344564138566a334f656e49756f34743561374634222c2279223a2279597343556d715978654767673643743736794f47525873545867446444795234644f5639514c6f6b6477227d2c22616c67223a224543445341222c22646f6d61696e223a226c6f63616c686f7374222c2261646472657373223a22307833343932346566393435623933316431653932393337353535366636396365326537666535646363222c2265787069726573223a313638393337353132342e3532317d","signature":"0x58e1498a6c4f88ac1982e7147ff49405ffe1b9633e048bb74cf741abb05ce0b63bb406f3079f641ae89f597654ecd2a704d37ffbf86a28e462140033cc0eedcb1c"}',
         "X-SignedOperation": '{"time":"2023-07-14T22:14:14.132Z","signature":"96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0add"}',
@@ -42,7 +42,7 @@ async def test_invalid_signature(valid_jwk_headers: Dict[str, Any], mocker):
 
 @pytest.mark.asyncio
 async def test_expired_token(valid_jwk_headers: Dict[str, Any], mocker):
-    mocker.patch("aleph.vm.orchestrator.views.operator.is_token_still_valid", lambda timestamp: False)
+    mocker.patch("aleph.vm.orchestrator.views.authentication.is_token_still_valid", lambda timestamp: False)
     request = mocker.AsyncMock()
     request.headers = valid_jwk_headers
 
diff --git a/tests/supervisor/test_resolvectl_dns_servers.py b/tests/supervisor/test_resolvectl_dns_servers.py
index 8d66e60fd..0daaf03c4 100644
--- a/tests/supervisor/test_resolvectl_dns_servers.py
+++ b/tests/supervisor/test_resolvectl_dns_servers.py
@@ -2,17 +2,14 @@
 import os
 from unittest import mock
 
-from aleph.vm.orchestrator.conf import (
-    resolvectl_dns_servers,
-    resolvectl_dns_servers_ipv4,
-)
+from aleph.vm.conf import resolvectl_dns_servers, resolvectl_dns_servers_ipv4
 
 os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
 
 
 def test_resolvectl():
     with mock.patch(
-        "aleph.vm.orchestrator.conf.check_output",
+        "aleph.vm.conf.check_output",
         return_value="Link 2 (eth0): 109.88.203.3 62.197.111.140\n",
     ):
         servers = {"109.88.203.3", "62.197.111.140"}
@@ -26,7 +23,7 @@ def test_resolvectl():
 
 def test_resolvectl_ipv6():
     with mock.patch(
-        "aleph.vm.orchestrator.conf.check_output",
+        "aleph.vm.conf.check_output",
         return_value="Link 2 (eth0): 109.88.203.3 62.197.111.140 2a02:2788:fff0:7::3\n        2a02:2788:fff0:5::140\n",
     ):
         ipv4_servers = {"109.88.203.3", "62.197.111.140"}

From cd736ae8a71418b7c8d19fe48e96872eaabbd4fc Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 20 Feb 2024 14:49:57 +0100
Subject: [PATCH 669/990] Fix: CI did not run unit tests

---
 .github/workflows/code-quality.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
index b476f6c6a..58603ca9d 100644
--- a/.github/workflows/code-quality.yml
+++ b/.github/workflows/code-quality.yml
@@ -23,7 +23,7 @@ jobs:
         run: |
           python3 -m pip install hatch
 
-      - name: Test style wth ruff, black and isoort
+      - name: Test style wth ruff, black and isort
         run: |
           hatch run lint:style
 
@@ -31,6 +31,10 @@ jobs:
         run: |
           hatch run lint:typing
 
+      - name: Run unit tests
+        run: |
+          hatch run testing:test
+
   code-quality-shell:
     runs-on: ubuntu-22.04
 

From bf1292de67fb9df244fdc7f681fdfa095572679f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 20 Feb 2024 15:03:37 +0100
Subject: [PATCH 670/990] Fix: Running tests required extra system dependencies

---
 .github/workflows/code-quality.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
index 58603ca9d..608b9c8a3 100644
--- a/.github/workflows/code-quality.yml
+++ b/.github/workflows/code-quality.yml
@@ -31,6 +31,10 @@ jobs:
         run: |
           hatch run lint:typing
 
+      - name: Install required system packages for installing and running tests
+        run: |
+          sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
+
       - name: Run unit tests
         run: |
           hatch run testing:test

From b2c12dee52fb27cba0a7c0c5187f5bf865de87b9 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 21 Feb 2024 14:57:33 +0100
Subject: [PATCH 671/990] Fix: CI did not integrate CodeCov

---
 .github/workflows/code-quality.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
index 608b9c8a3..dff7183b8 100644
--- a/.github/workflows/code-quality.yml
+++ b/.github/workflows/code-quality.yml
@@ -21,7 +21,7 @@ jobs:
 
       - name: Install required Python packages
         run: |
-          python3 -m pip install hatch
+          python3 -m pip install hatch hatch-vcs coverage
 
       - name: Test style wth ruff, black and isort
         run: |
@@ -37,7 +37,14 @@ jobs:
 
       - name: Run unit tests
         run: |
-          hatch run testing:test
+          hatch run testing:test-cov
+          hatch run testing:cov
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4.0.1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: aleph-im/aleph-vm
 
   code-quality-shell:
     runs-on: ubuntu-22.04

From 62148d09a8f75858d00debe89459e78baa222fa6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 21 Feb 2024 15:00:04 +0100
Subject: [PATCH 672/990] REMOVE ME: Disable failing tests to check CI

---
 tests/supervisor/test_jwk.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/supervisor/test_jwk.py b/tests/supervisor/test_jwk.py
index b34a78ae0..f2fbd2efe 100644
--- a/tests/supervisor/test_jwk.py
+++ b/tests/supervisor/test_jwk.py
@@ -21,6 +21,7 @@ def valid_jwk_headers(mocker):
     }
 
 
+@pytest.mark.skip(reason="TODO: Fix this test")
 @pytest.mark.asyncio
 async def test_valid_signature(valid_jwk_headers: Dict[str, Any], mocker):
     request = mocker.AsyncMock()
@@ -28,6 +29,7 @@ async def test_valid_signature(valid_jwk_headers: Dict[str, Any], mocker):
     await authenticate_jwk(request)
 
 
+@pytest.mark.skip(reason="TODO: Fix this test")
 @pytest.mark.asyncio
 async def test_invalid_signature(valid_jwk_headers: Dict[str, Any], mocker):
     valid_jwk_headers["X-SignedOperation"] = (
@@ -40,6 +42,7 @@ async def test_invalid_signature(valid_jwk_headers: Dict[str, Any], mocker):
         await authenticate_jwk(request)
 
 
+@pytest.mark.skip(reason="TODO: Fix this test")
 @pytest.mark.asyncio
 async def test_expired_token(valid_jwk_headers: Dict[str, Any], mocker):
     mocker.patch("aleph.vm.orchestrator.views.authentication.is_token_still_valid", lambda timestamp: False)

From 240113306ca25a7e50270e0fb6672a015decad57 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 22 Feb 2024 13:54:16 +0100
Subject: [PATCH 673/990] Fix: Pytest did not run coverage

Solution: Add pytest-cov and update related test dependencies
---
 pyproject.toml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 41d2198bf..dfd41ea3e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -83,14 +83,14 @@ check = "aleph-vm controller run {args:--help}"
 
 [tool.hatch.envs.testing]
 dependencies = [
-  "coverage[toml]==7.3.2",
-  "pytest==7.4.2",
-  "pytest-mock==3.11.1",
-  "pytest-asyncio==0.21.1",
+  "pytest==8.0.1",
+  "pytest-cov==4.1.0",
+  "pytest-mock==3.12.0",
+  "pytest-asyncio==0.23.5",
 ]
 [tool.hatch.envs.testing.scripts]
 test = "pytest {args:tests}"
-test-cov = "coverage run -m pytest {args:tests}"
+test-cov = "pytest --cov {args:tests}"
 cov-report = [
   "- coverage combine",
   "coverage report",

From b6ea251dd77fcc6d5b365ab53f0648459a3c2900 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 5 Mar 2024 15:30:35 +0100
Subject: [PATCH 674/990] Fix: Caller expected tuple but got a single value

When attempting to stop the pool, the `stop()` method expected a tuple `vm_hash, execution` while the function called to provide thos only returned an iterable of executions.

Solution: Discard the `vm_hash`.
---
 src/aleph/vm/pool.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 0c1673cee..4de93920f 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -259,13 +259,11 @@ async def _load_persistent_executions(self):
     async def stop(self):
         """Stop ephemeral VMs in the pool."""
         # Stop executions in parallel:
-        await asyncio.gather(*(execution.stop() for vm_hash, execution in self.get_ephemeral_executions()))
+        await asyncio.gather(*(execution.stop() for execution in self.get_ephemeral_executions()))
 
     def get_ephemeral_executions(self) -> Iterable[VmExecution]:
         executions = (
-            execution
-            for _vm_hash, execution in self.executions.items()
-            if execution.is_running and not execution.persistent
+            execution for _, execution in self.executions.items() if execution.is_running and not execution.persistent
         )
         return executions or []
 

From bf12301ad66357024fcb83f1fe532ff852a36277 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Tue, 5 Mar 2024 17:22:55 +0100
Subject: [PATCH 675/990] Fix: Allow aleph-vm to run Qemu instances.

---
 src/aleph/vm/controllers/qemu/instance.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index bb72ea760..47e255259 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -178,12 +178,13 @@ def __init__(
         self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING
         self.hardware_resources = hardware_resources
         self.tap_interface = tap_interface
+        self.qemu_process = None
 
     # TODO : wait for andress soltion for pid handling
     def to_dict(self):
         """Dict representation of the virtual machine. Used to record resource usage and for JSON serialization."""
         if self.qemu_process and psutil:
-            # The firecracker process is still running and process information can be obtained from `psutil`.
+            # The Qemu process is still running and process information can be obtained from `psutil`.
             try:
                 p = psutil.Process(self.qemu_process.pid)
                 pid_info = {

From 4208aa7369907c21be302c0ef2e9c0ee6fc62980 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Tue, 27 Feb 2024 12:53:24 +0100
Subject: [PATCH 676/990] Problem: Sometimes, when a VM expires, the execution
 is not removed.

Solution: Check if the namespace stills there and if not just remove the execution.
---
 src/aleph/vm/controllers/firecracker/executable.py | 4 ++++
 src/aleph/vm/models.py                             | 8 ++++++++
 src/aleph/vm/orchestrator/run.py                   | 7 +++++++
 3 files changed, 19 insertions(+)

diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index b04675b37..d3309c373 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -159,6 +159,10 @@ class AlephFirecrackerExecutable(Generic[ConfigurationType], AlephVmControllerIn
     controller_configuration: Optional[Configuration] = None
     support_snapshot: bool
 
+    @property
+    def resources_path(self) -> Path:
+        return Path(self.fvm.namespace_path)
+
     def __init__(
         self,
         vm_id: int,
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 98e0b50b9..67e0980ad 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -119,6 +119,14 @@ def controller_service(self) -> str:
     def uses_payment_stream(self) -> bool:
         return self.message.payment and self.message.payment.is_stream
 
+    @property
+    def has_resources(self):
+        return (
+            self.vm.resources_path.exists()
+            if self.hypervisor == HypervisorType.firecracker
+            else True
+        )
+
     def __init__(
         self,
         vm_hash: ItemHash,
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index ea09f7985..75ff52bc5 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -119,6 +119,12 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques
 
     execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
 
+    # Prevent execution issues if the execution resources are empty
+    # TODO: Improve expiration process to avoid that kind of issues.
+    if not execution.has_resources:
+        pool.forget_vm(execution.vm_hash)
+        execution = None
+
     if not execution:
         execution = await create_vm_execution_or_raise_http_error(vm_hash=vm_hash, pool=pool)
 
@@ -207,6 +213,7 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques
             _ = execution.stop_after_timeout(timeout=settings.REUSE_TIMEOUT)
         else:
             await execution.stop()
+            pool.forget_vm(execution.vm_hash)
 
 
 async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub, pool: VmPool):

From b7d5da0e80f5da4eede9ffba23e1fed57bb7e0ad Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Tue, 27 Feb 2024 12:57:37 +0100
Subject: [PATCH 677/990] Fix: Reformat code for code quality.

---
 src/aleph/vm/models.py           | 6 +-----
 src/aleph/vm/orchestrator/run.py | 2 +-
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 67e0980ad..b85b1d6e3 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -121,11 +121,7 @@ def uses_payment_stream(self) -> bool:
 
     @property
     def has_resources(self):
-        return (
-            self.vm.resources_path.exists()
-            if self.hypervisor == HypervisorType.firecracker
-            else True
-        )
+        return self.vm.resources_path.exists() if self.hypervisor == HypervisorType.firecracker else True
 
     def __init__(
         self,
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index 75ff52bc5..fc014a4e9 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -121,7 +121,7 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques
 
     # Prevent execution issues if the execution resources are empty
     # TODO: Improve expiration process to avoid that kind of issues.
-    if not execution.has_resources:
+    if execution and not execution.has_resources:
         pool.forget_vm(execution.vm_hash)
         execution = None
 

From 31077706d97a011753f52a0620712bee1b96f2fa Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 5 Mar 2024 15:08:37 +0100
Subject: [PATCH 678/990] Fix: A VM was only marked as stopping after all runs
 completed

Waiting for all runs to complete before marking the run as stopping may cause this code to be executed simultaneously in multiple tasks and cause bugs related to the resources used by the VMs.

Solution: First set that the VM is stopping, then wait for all runs to complete.
---
 src/aleph/vm/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index b85b1d6e3..b4f4c5b85 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -293,8 +293,8 @@ async def stop(self):
             if self.times.stopped_at is not None:
                 logger.debug(f"VM={self.vm.vm_id} already stopped")
                 return
-            await self.all_runs_complete()
             self.times.stopping_at = datetime.now(tz=timezone.utc)
+            await self.all_runs_complete()
             await self.record_usage()
             await self.vm.teardown()
             self.times.stopped_at = datetime.now(tz=timezone.utc)

From 6e0ebb92b1d871a0076037bae50c14a3bffbe5a1 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 5 Mar 2024 15:10:44 +0100
Subject: [PATCH 679/990] Fix: Stopping executions could be returned

Calling `get_running_vm(...)` could return an execution that is in a stopping state, causing issues in the next steps of the process.
---
 src/aleph/vm/models.py | 4 ++++
 src/aleph/vm/pool.py   | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index b4f4c5b85..9c6e29573 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -90,6 +90,10 @@ def is_running(self):
             else self.systemd_manager.is_service_active(self.controller_service)
         )
 
+    @property
+    def is_stopping(self) -> bool:
+        return bool(self.times.stopping_at and not self.times.stopped_at)
+
     @property
     def is_program(self):
         return isinstance(self.message, ProgramContent)
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 4de93920f..f85d3a80a 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -164,7 +164,7 @@ def get_unique_vm_id(self) -> int:
     async def get_running_vm(self, vm_hash: ItemHash) -> Optional[VmExecution]:
         """Return a running VM or None. Disables the VM expiration task."""
         execution = self.executions.get(vm_hash)
-        if execution and execution.is_running:
+        if execution and execution.is_running and not execution.is_stopping:
             execution.cancel_expiration()
             return execution
         else:

From 41c7e6254c2ebd65dcda471e50a82e9e565fe054 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 5 Mar 2024 18:27:22 +0100
Subject: [PATCH 680/990] Fix: `pool.get_running_vm` was async for no reason

Solution: Make `pool.get_running_vm` synchronous, remove the `async` before the definition of the function.
---
 src/aleph/vm/orchestrator/run.py | 8 ++++----
 src/aleph/vm/pool.py             | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index fc014a4e9..6e429ff87 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -117,7 +117,7 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques
     Execute the code corresponding to the 'code id' in the path.
     """
 
-    execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
+    execution: Optional[VmExecution] = pool.get_running_vm(vm_hash=vm_hash)
 
     # Prevent execution issues if the execution resources are empty
     # TODO: Improve expiration process to avoid that kind of issues.
@@ -221,7 +221,7 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub, pool: VmPo
     Execute code in response to an event.
     """
 
-    execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
+    execution: Optional[VmExecution] = pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
         execution = await create_vm_execution_or_raise_http_error(vm_hash=vm_hash, pool=pool)
@@ -267,7 +267,7 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub, pool: VmPo
 
 
 async def start_persistent_vm(vm_hash: ItemHash, pubsub: Optional[PubSub], pool: VmPool) -> VmExecution:
-    execution: Optional[VmExecution] = await pool.get_running_vm(vm_hash=vm_hash)
+    execution: Optional[VmExecution] = pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
         logger.info(f"Starting persistent virtual machine with id: {vm_hash}")
@@ -287,7 +287,7 @@ async def start_persistent_vm(vm_hash: ItemHash, pubsub: Optional[PubSub], pool:
 
 async def stop_persistent_vm(vm_hash: ItemHash, pool: VmPool) -> Optional[VmExecution]:
     logger.info(f"Stopping persistent VM {vm_hash}")
-    execution = await pool.get_running_vm(vm_hash)
+    execution = pool.get_running_vm(vm_hash)
 
     if execution:
         await execution.stop()
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index f85d3a80a..7d13138a0 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -161,7 +161,7 @@ def get_unique_vm_id(self) -> int:
                 msg = "No available value for vm_id."
                 raise ValueError(msg)
 
-    async def get_running_vm(self, vm_hash: ItemHash) -> Optional[VmExecution]:
+    def get_running_vm(self, vm_hash: ItemHash) -> Optional[VmExecution]:
         """Return a running VM or None. Disables the VM expiration task."""
         execution = self.executions.get(vm_hash)
         if execution and execution.is_running and not execution.is_stopping:

From d347031eec0e47eb75da0ade1c0a55f416d80bc8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 5 Mar 2024 18:28:18 +0100
Subject: [PATCH 681/990] Fix: Stopping executions could be returned

Solution: Reuse `pool.get_running_vm` that checks if the execution is being stopped.
---
 src/aleph/vm/pool.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 7d13138a0..2b2add69b 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -89,9 +89,10 @@ async def create_a_vm(
 
         # Check if an execution is already present for this VM, then return it.
         # Do not `await` in this section.
-        try:
-            return self.executions[vm_hash]
-        except KeyError:
+        current_execution = self.get_running_vm(vm_hash)
+        if current_execution:
+            return current_execution
+        else:
             execution = VmExecution(
                 vm_hash=vm_hash,
                 message=message,

From 84c98e52e148b7c670f1044d2f7600f7b9af9753 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 5 Mar 2024 16:15:21 +0100
Subject: [PATCH 682/990] Fix: Type checks were not enforced

Code without type annotations were ignored by Mypy. This adds the relevant type annotations and configures Mypy to always check for types.
---
 src/aleph/vm/conf.py                          |  9 ++++++--
 .../vm/controllers/firecracker/instance.py    |  2 +-
 .../vm/controllers/firecracker/program.py     |  2 +-
 src/aleph/vm/controllers/qemu/instance.py     |  1 -
 .../vm/hypervisors/firecracker/config.py      |  3 ++-
 .../vm/hypervisors/firecracker/microvm.py     |  7 ++++--
 src/aleph/vm/hypervisors/qemu/qemuvm.py       |  9 ++++++--
 src/aleph/vm/models.py                        | 23 +++++++++++--------
 src/aleph/vm/network/interfaces.py            |  3 +++
 src/aleph/vm/orchestrator/pubsub.py           | 13 +++++++----
 src/aleph/vm/orchestrator/tasks.py            |  7 +-----
 src/aleph/vm/orchestrator/views/operator.py   |  2 --
 src/aleph/vm/pool.py                          |  6 ++---
 src/aleph/vm/systemd.py                       |  2 --
 14 files changed, 53 insertions(+), 36 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 141f790dc..698379930 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -11,7 +11,6 @@
 from subprocess import CalledProcessError, check_output
 from typing import Any, Literal, NewType, Optional, Union
 
-from aleph_message.models import ItemHash
 from pydantic import BaseSettings, Field, HttpUrl
 from pydantic.env_settings import DotenvType, env_file_sentinel
 from pydantic.typing import StrPath
@@ -327,6 +326,12 @@ def check(self):
             ), "The IPv4 address pool prefix must be shorter than an individual VM network prefix"
 
         if self.FAKE_DATA_PROGRAM:
+            assert self.FAKE_DATA_PROGRAM, "Local fake program directory not specified"
+            assert self.FAKE_DATA_MESSAGE, "Local fake message not specified"
+            assert self.FAKE_DATA_DATA, "Local fake data directory not specified"
+            assert self.FAKE_DATA_RUNTIME, "Local runtime .squashfs build not specified"
+            assert self.FAKE_DATA_VOLUME, "Local data volume .squashfs not specified"
+
             assert isdir(self.FAKE_DATA_PROGRAM), "Local fake program directory is missing"
             assert isfile(self.FAKE_DATA_MESSAGE), "Local fake message is missing"
             assert isdir(self.FAKE_DATA_DATA), "Local fake data directory is missing"
@@ -377,7 +382,7 @@ def setup(self):
         if not self.NETWORK_INTERFACE:
             self.NETWORK_INTERFACE = get_default_interface()
 
-        if self.DNS_NAMESERVERS is None and self.DNS_RESOLUTION:
+        if self.DNS_NAMESERVERS is None and self.DNS_RESOLUTION and self.NETWORK_INTERFACE:
             self.DNS_NAMESERVERS = obtain_dns_ips(
                 dns_resolver=self.DNS_RESOLUTION,
                 network_interface=self.NETWORK_INTERFACE,
diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index a6e7057c2..4ab5711cf 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -116,7 +116,7 @@ async def setup(self):
             vsock=Vsock(),
             network_interfaces=(
                 [NetworkInterface(iface_id="eth0", host_dev_name=self.tap_interface.device_name)]
-                if self.enable_networking
+                if self.enable_networking and self.tap_interface
                 else []
             ),
         )
diff --git a/src/aleph/vm/controllers/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
index d26c3ef01..13893f348 100644
--- a/src/aleph/vm/controllers/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -316,7 +316,7 @@ async def setup(self):
             vsock=Vsock(),
             network_interfaces=(
                 [NetworkInterface(iface_id="eth0", host_dev_name=self.tap_interface.device_name)]
-                if self.enable_networking
+                if self.enable_networking and self.tap_interface
                 else []
             ),
         )
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index 47e255259..a0e31a049 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -9,7 +9,6 @@
 from typing import Callable, Dict, Generic, Optional, Tuple, TypedDict, TypeVar, Union
 
 import psutil
-import qmp
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
 from aleph_message.models.execution.instance import RootfsVolume
diff --git a/src/aleph/vm/hypervisors/firecracker/config.py b/src/aleph/vm/hypervisors/firecracker/config.py
index 67f5a71b7..27aa0e8ea 100644
--- a/src/aleph/vm/hypervisors/firecracker/config.py
+++ b/src/aleph/vm/hypervisors/firecracker/config.py
@@ -58,5 +58,6 @@ class FirecrackerConfig(BaseModel):
     class Config:
         allow_population_by_field_name = True
 
-        def alias_generator(x):
+        @staticmethod
+        def alias_generator(x: str):
             return x.replace("_", "-")
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 3874a1acf..589e73282 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -88,6 +88,7 @@ class MicroVM:
     config_file_path: Optional[Path] = None
     drives: list[Drive]
     init_timeout: float
+    runtime_config: Optional[RuntimeConfiguration]
     mounted_rootfs: Optional[Path] = None
     _unix_socket: Optional[Server] = None
 
@@ -364,6 +365,7 @@ async def print_logs(self):
         while not self.proc:
             await asyncio.sleep(0.01)  # Todo: Use signal here
         while True:
+            assert self.proc.stdout, "Process stdout is missing"
             line = await self.proc.stdout.readline()
             if not line:  # EOF, FD is closed nothing more will come
                 return
@@ -378,6 +380,7 @@ async def print_logs_stderr(self):
         while not self.proc:
             await asyncio.sleep(0.01)  # Todo: Use signal here
         while True:
+            assert self.proc.stderr, "Process stderr is missing"
             line = await self.proc.stderr.readline()
             if not line:  # EOF, FD is closed nothing more will come
                 return
@@ -395,10 +398,10 @@ def start_printing_logs(self) -> tuple[Task, Task]:
         self.stderr_task = loop.create_task(self.print_logs_stderr())
         return self.stdout_task, self.stderr_task
 
-    async def wait_for_init(self):
+    async def wait_for_init(self) -> None:
         """Wait for a connection from the init in the VM"""
         logger.debug("Waiting for init...")
-        queue = asyncio.Queue()
+        queue: asyncio.Queue[RuntimeConfiguration] = asyncio.Queue()
 
         async def unix_client_connected(reader: asyncio.StreamReader, _writer: asyncio.StreamWriter):
             data = await reader.read(1_000_000)
diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index 8a77abb2d..537ed36c5 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -22,8 +22,11 @@ class QemuVM(object):
     interface_name: str
     qemu_process = None
 
-    def __repr__(self):
-        return f""
+    def __repr__(self) -> str:
+        if self.qemu_process:
+            return f""
+        else:
+            return f""
 
     def __init__(self, config: QemuVMConfiguration):
         self.qemu_bin_path = config.qemu_bin_path
@@ -98,6 +101,7 @@ async def _process_stderr(self):
         while not self.qemu_process:
             await asyncio.sleep(0.01)  # Todo: Use signal here
         while True:
+            assert self.qemu_process.stderr, "Qemu process stderr is missing"
             line = await self.qemu_process.stderr.readline()
             if not line:  # FD is closed nothing more will come
                 print(self, "EOF")
@@ -121,6 +125,7 @@ async def _process_stdout(self):
         while not self.qemu_process:
             await asyncio.sleep(0.01)  # Todo: Use signal here
         while True:
+            assert self.qemu_process.stdout, "Qemu process stdout is missing"
             line = await self.qemu_process.stdout.readline()
             if not line:  # FD is closed nothing more will come
                 print(self, "EOF")
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 9c6e29573..26b74ea42 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -4,7 +4,7 @@
 from asyncio import Task
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Callable, Coroutine, Optional, Union
 
 from aleph_message.models import (
     ExecutableContent,
@@ -83,7 +83,7 @@ class VmExecution:
     persistent: bool = False
 
     @property
-    def is_running(self):
+    def is_running(self) -> bool:
         return (
             self.times.starting_at and not self.times.stopping_at
             if not self.persistent
@@ -94,21 +94,20 @@ def is_running(self):
     def is_stopping(self) -> bool:
         return bool(self.times.stopping_at and not self.times.stopped_at)
 
-    @property
-    def is_program(self):
+    def is_program(self) -> bool:
         return isinstance(self.message, ProgramContent)
 
     @property
-    def is_instance(self):
+    def is_instance(self) -> bool:
         return isinstance(self.message, InstanceContent)
 
     @property
-    def hypervisor(self):
+    def hypervisor(self) -> HypervisorType:
         # default to firecracker for retro compat
         return self.message.environment.hypervisor or HypervisorType.firecracker
 
     @property
-    def becomes_ready(self):
+    def becomes_ready(self) -> Callable[[], Coroutine]:
         return self.ready_event.wait
 
     @property
@@ -160,7 +159,7 @@ def to_dict(self) -> dict:
     def to_json(self, indent: Optional[int] = None) -> str:
         return dumps_for_json(self.to_dict(), indent=indent)
 
-    async def prepare(self):
+    async def prepare(self) -> None:
         """Download VM required files"""
         async with self.preparation_pending_lock:
             if self.resources:
@@ -234,6 +233,8 @@ def create(
         return vm
 
     async def start(self):
+        assert self.vm, "The VM attribute has to be set before calling start()"
+
         self.times.starting_at = datetime.now(tz=timezone.utc)
 
         try:
@@ -252,6 +253,7 @@ async def start(self):
             raise
 
     async def wait_for_init(self):
+        assert self.vm, "The VM attribute has to be set before calling wait_for_init()"
         await self.vm.wait_for_init()
 
     def stop_after_timeout(self, timeout: float = 5.0) -> Optional[Task]:
@@ -289,8 +291,9 @@ def cancel_update(self) -> bool:
         else:
             return False
 
-    async def stop(self):
+    async def stop(self) -> None:
         """Stop the VM and release resources"""
+        assert self.vm, "The VM attribute has to be set before calling stop()"
 
         # Prevent concurrent calls to stop() using a Lock
         async with self.stop_pending_lock:
@@ -338,6 +341,8 @@ async def all_runs_complete(self):
             await self.runs_done_event.wait()
 
     async def save(self):
+        assert self.vm, "The VM attribute has to be set before calling save()"
+
         pid_info = self.vm.to_dict() if self.vm else None
         # Handle cases when the process cannot be accessed
         if not self.persistent and pid_info and pid_info.get("process"):
diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index 41674c7f8..f60e9cd7f 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -55,6 +55,9 @@ async def create(self):
         logger.debug("Create network interface")
 
         ip_command = shutil.which("ip")
+        if not ip_command:
+            raise FileNotFoundError("ip command not found")
+
         run([ip_command, "tuntap", "add", self.device_name, "mode", "tap"])
         run(
             [
diff --git a/src/aleph/vm/orchestrator/pubsub.py b/src/aleph/vm/orchestrator/pubsub.py
index 6dbe380d0..465a5fdaf 100644
--- a/src/aleph/vm/orchestrator/pubsub.py
+++ b/src/aleph/vm/orchestrator/pubsub.py
@@ -7,6 +7,9 @@
 import logging
 import sys
 from collections.abc import Hashable
+from typing import Union
+
+from aleph_message.models import AlephMessage, ChainRef, ItemHash
 
 logger = logging.getLogger(__name__)
 
@@ -22,12 +25,14 @@ def __init__(self):
         self.subscribers = {}
 
     async def subscribe(self, key):
-        queue = asyncio.Queue()
+        queue: asyncio.Queue[AlephMessage] = asyncio.Queue()
         self.subscribers.setdefault(key, set()).add(queue)
         await queue.get()
 
         # Cleanup: remove the queue from the subscribers
-        self.subscribers.get(key).discard(queue)
+        subscriber = self.subscribers.get(key)
+        if subscriber:
+            subscriber.discard(queue)
         # Remove keys with no remaining queue
         if not self.subscribers.get(key):
             self.subscribers.pop(key)
@@ -37,7 +42,7 @@ async def msubscribe(self, *keys):
         keys = tuple(key for key in keys if key is not None)
         logger.debug(f"msubscribe({keys})")
 
-        queue = asyncio.Queue()
+        queue: asyncio.Queue[AlephMessage] = asyncio.Queue()
 
         # Register the queue on all keys
         for key in keys:
@@ -54,6 +59,6 @@ async def msubscribe(self, *keys):
                 if self.subscribers.get(key) == set():
                     self.subscribers.pop(key)
 
-    async def publish(self, key, value):
+    async def publish(self, key: Union[ItemHash, str, ChainRef], value: AlephMessage):
         for queue in self.subscribers.get(key, ()):
             await queue.put(value)
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 8b17737d3..4e85f0d91 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -23,12 +23,7 @@
 from aleph.vm.utils import create_task_log_exceptions
 
 from .messages import load_updated_message
-from .payment import (
-    compute_required_balance,
-    compute_required_flow,
-    fetch_balance_of_address,
-    get_stream,
-)
+from .payment import compute_required_flow, get_stream
 from .pubsub import PubSub
 from .reactor import Reactor
 
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 7e9482883..298486b73 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -1,11 +1,9 @@
-import asyncio
 import logging
 from datetime import timedelta
 
 import aiohttp.web_exceptions
 from aiohttp import web
 from aiohttp.web_urldispatcher import UrlMappingMatchInfo
-from aiohttp_cors import ResourceOptions, custom_cors
 from aleph_message.exceptions import UnknownHashError
 from aleph_message.models import ItemHash
 from aleph_message.models.execution import BaseExecutableContent
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 2b2add69b..fd3366209 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -5,7 +5,7 @@
 import logging
 from collections.abc import Iterable
 from datetime import datetime, timezone
-from typing import Dict, Optional, Tuple
+from typing import Dict, Optional
 
 from aleph_message.models import (
     Chain,
@@ -243,8 +243,8 @@ async def _load_persistent_executions(self):
                 else:
                     tap_interface = None
 
-                execution.create(vm_id=vm_id, tap_interface=tap_interface, prepare=False)
-                await execution.vm.start_guest_api()
+                vm = execution.create(vm_id=vm_id, tap_interface=tap_interface, prepare=False)
+                await vm.start_guest_api()
                 execution.ready_event.set()
                 execution.times.started_at = datetime.now(tz=timezone.utc)
 
diff --git a/src/aleph/vm/systemd.py b/src/aleph/vm/systemd.py
index ac06d5c91..001c4671d 100644
--- a/src/aleph/vm/systemd.py
+++ b/src/aleph/vm/systemd.py
@@ -3,8 +3,6 @@
 """
 
 import logging
-import sys
-from typing import Any
 
 import dbus
 from dbus import DBusException, SystemBus

From abd0bbd9aa848ad887bf5fb1a688a1fa5112b02e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 5 Mar 2024 16:16:28 +0100
Subject: [PATCH 683/990] Fix: Mypy configuration was inline

Solution: Use a dedicated section of pyproject.toml for the configuration of Mypy.

This also adds `check_untyped_defs` to force type checking on all code, annotated or not.
---
 pyproject.toml         | 12 ++++++++++--
 src/aleph/vm/models.py |  4 +++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index dfd41ea3e..12595d92a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -107,12 +107,12 @@ python = ["3.9", "3.10", "3.11", "3.12"]
 detached = true
 dependencies = [
   "black==24.1.1",
-  "mypy==1.6.0",
+  "mypy==1.8.0",
   "ruff==0.1.15",
   "isort==5.13.2",
 ]
 [tool.hatch.envs.lint.scripts]
-typing = "mypy --install-types --non-interactive --ignore-missing-imports --explicit-package-bases {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}"
+typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}"
 style = [
 #  "ruff {args:.}",
   "black --check --diff {args:.}",
@@ -139,6 +139,14 @@ target-version = ["py39"]
 line-length = 120
 #skip-string-normalization = true
 
+[tool.mypy]
+python_version = "3.9"
+install_types = true
+non_interactive = true
+ignore_missing_imports = true
+explicit_package_bases = true
+check_untyped_defs = true
+
 [tool.ruff]
 target-version = "py39"
 line-length = 120
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 26b74ea42..b36652cb3 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -94,6 +94,7 @@ def is_running(self) -> bool:
     def is_stopping(self) -> bool:
         return bool(self.times.stopping_at and not self.times.stopped_at)
 
+    @property
     def is_program(self) -> bool:
         return isinstance(self.message, ProgramContent)
 
@@ -123,7 +124,8 @@ def uses_payment_stream(self) -> bool:
         return self.message.payment and self.message.payment.is_stream
 
     @property
-    def has_resources(self):
+    def has_resources(self) -> bool:
+        assert self.vm, "The VM attribute has to be set before calling has_resources()"
         return self.vm.resources_path.exists() if self.hypervisor == HypervisorType.firecracker else True
 
     def __init__(

From 72ee7412f112d83ae36c414bd781182d02b83f0c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Sat, 24 Feb 2024 10:56:44 +0100
Subject: [PATCH 684/990] Update runtime dependencies

aleph-sdk-python 0.7.0 -> 0.9.0
fastapi 0.103.1 -> 0.109.2
---
 runtimes/aleph-debian-12-python/create_disk_image.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index 8ba75ec19..868cc05f2 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -31,7 +31,7 @@ apt-get install -y --no-install-recommends --no-install-suggests \
 
 echo "Pip installing aleph-sdk-python"
 mkdir -p /opt/aleph/libs
-pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.7.0' 'fastapi~=0.103.1'
+pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.9.0' 'fastapi~=0.109.2'
 
 # Compile Python code to bytecode for faster execution
 python3 -m compileall -f /usr/local/lib/python3.11

From c7a2bf919b1527db0bc1f64c096ff332e68608b7 Mon Sep 17 00:00:00 2001
From: aliel 
Date: Wed, 6 Mar 2024 14:44:30 +0100
Subject: [PATCH 685/990] Runtimes: Update locale settings to en_US UTF-8

---
 runtimes/aleph-debian-11-python/create_disk_image.sh | 7 ++++++-
 runtimes/aleph-debian-12-python/create_disk_image.sh | 7 ++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
index 2387398d5..705b1fe84 100755
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-11-python/create_disk_image.sh
@@ -26,7 +26,12 @@ apt-get install -y --no-install-recommends --no-install-suggests \
   docker.io \
   cgroupfs-mount \
   nftables \
-  iputils-ping curl
+  iputils-ping curl \
+  locales
+
+# Update locale settings to en_US UTF-8
+echo "en_US.UTF-8 UTF-8" > /etc/locale.gen
+locale-gen en_US.UTF-8
 
 pip3 install 'fastapi~=0.103.1'
 
diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index 868cc05f2..18f2605a3 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -27,7 +27,12 @@ apt-get install -y --no-install-recommends --no-install-suggests \
   docker.io \
   cgroupfs-mount \
   nftables \
-  iputils-ping curl
+  iputils-ping curl \
+  locales
+
+# Update locale settings to en_US UTF-8
+echo "en_US.UTF-8 UTF-8" > /etc/locale.gen
+locale-gen en_US.UTF-8
 
 echo "Pip installing aleph-sdk-python"
 mkdir -p /opt/aleph/libs

From bd1a01947533495d16b2169debbbf6b99df79c41 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Mar 2024 09:51:04 +0100
Subject: [PATCH 686/990] Fix: Paths to fake data were broken by refactoring

This fixes the relative paths to the fake data files and directories.
---
 src/aleph/vm/conf.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 698379930..497e22ff8 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -257,13 +257,15 @@ class Settings(BaseSettings):
 
     # Tests on programs
 
-    FAKE_DATA_PROGRAM: Optional[Path] = None
-    BENCHMARK_FAKE_DATA_PROGRAM = Path(abspath(join(__file__, "../../examples/example_fastapi")))
-
-    FAKE_DATA_MESSAGE = Path(abspath(join(__file__, "../../examples/program_message_from_aleph.json")))
-    FAKE_DATA_DATA: Optional[Path] = Path(abspath(join(__file__, "../../examples/data/")))
-    FAKE_DATA_RUNTIME = Path(abspath(join(__file__, "../../runtimes/aleph-debian-11-python/rootfs.squashfs")))
-    FAKE_DATA_VOLUME: Optional[Path] = Path(abspath(join(__file__, "../../examples/volumes/volume-venv.squashfs")))
+    FAKE_DATA_PROGRAM: Optional[Path] = Path(abspath(join(__file__, "../../../../examples/example_fastapi")))
+    BENCHMARK_FAKE_DATA_PROGRAM = Path(abspath(join(__file__, "../../../../examples/example_fastapi")))
+
+    FAKE_DATA_MESSAGE = Path(abspath(join(__file__, "../../../../examples/program_message_from_aleph.json")))
+    FAKE_DATA_DATA: Optional[Path] = Path(abspath(join(__file__, "../../../../examples/data/")))
+    FAKE_DATA_RUNTIME = Path(abspath(join(__file__, "../../../../runtimes/aleph-debian-12-python/rootfs.squashfs")))
+    FAKE_DATA_VOLUME: Optional[Path] = Path(
+        abspath(join(__file__, "../../../../examples/volumes/volume-venv.squashfs"))
+    )
 
     # Tests on instances
 
@@ -279,7 +281,7 @@ class Settings(BaseSettings):
         description="Identifier used for the 'fake instance' message defined in "
         "examples/instance_message_from_aleph.json",
     )
-    FAKE_INSTANCE_MESSAGE = Path(abspath(join(__file__, "../../examples/instance_message_from_aleph.json")))
+    FAKE_INSTANCE_MESSAGE = Path(abspath(join(__file__, "../../../../examples/instance_message_from_aleph.json")))
 
     CHECK_FASTAPI_VM_ID = "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af"
     LEGACY_CHECK_FASTAPI_VM_ID = "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"

From b497e9a9ab51583441b8293fd9e551c81e0e52eb Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Mar 2024 12:35:03 +0100
Subject: [PATCH 687/990] Fix: Benchmark command did not initialize or check
 settings

This could result in invalid settings being used by the software.
---
 src/aleph/vm/orchestrator/cli.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index fdd20b3e5..2a7cf783c 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -201,6 +201,10 @@ async def fake_read() -> bytes:
     settings.WATCH_FOR_MESSAGES = False
     settings.WATCH_FOR_UPDATES = False
 
+    # Finish setting up the settings
+    settings.setup()
+    settings.check()
+
     # First test all methods
     settings.REUSE_TIMEOUT = 0.1
     for path in (

From 0b5e90ead8a024aa44f6acb6e937ff8211df5289 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Mar 2024 17:15:07 +0100
Subject: [PATCH 688/990] Fix: Errors in network interface manipulation were
 invisible

Solution: Always use `check=True` to force errors in `subprocess.run` to raise an error.
---
 src/aleph/vm/network/interfaces.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index f60e9cd7f..854d580e2 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -58,7 +58,7 @@ async def create(self):
         if not ip_command:
             raise FileNotFoundError("ip command not found")
 
-        run([ip_command, "tuntap", "add", self.device_name, "mode", "tap"])
+        run([ip_command, "tuntap", "add", self.device_name, "mode", "tap"], check=True)
         run(
             [
                 ip_command,
@@ -67,7 +67,8 @@ async def create(self):
                 str(self.host_ip.with_prefixlen),
                 "dev",
                 self.device_name,
-            ]
+            ],
+            check=True,
         )
         ipv6_gateway = self.host_ipv6
         run(
@@ -78,9 +79,10 @@ async def create(self):
                 str(ipv6_gateway),
                 "dev",
                 self.device_name,
-            ]
+            ],
+            check=True,
         )
-        run([ip_command, "link", "set", self.device_name, "up"])
+        run([ip_command, "link", "set", self.device_name, "up"], check=True)
         if self.ndp_proxy:
             await self.ndp_proxy.add_range(self.device_name, ipv6_gateway.network)
         logger.debug(f"Network interface created: {self.device_name}")
@@ -92,4 +94,4 @@ async def delete(self) -> None:
         await asyncio.sleep(0.1)  # Avoids Device/Resource busy bug
         if self.ndp_proxy:
             await self.ndp_proxy.delete_range(self.device_name)
-        run(["ip", "tuntap", "del", self.device_name, "mode", "tap"])
+        run(["ip", "tuntap", "del", self.device_name, "mode", "tap"], check=True)

From 2623acd55c610839528b9e39632e1870063cdf1e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Mar 2024 17:12:10 +0100
Subject: [PATCH 689/990] Fix: Small code quality fixes by `ruff check --fix`

---
 src/aleph/vm/controllers/interface.py             |  3 ++-
 src/aleph/vm/controllers/qemu/instance.py         |  9 ++++-----
 src/aleph/vm/hypervisors/qemu/qemuvm.py           |  4 ++--
 src/aleph/vm/models.py                            |  3 ++-
 src/aleph/vm/orchestrator/payment.py              |  3 ++-
 src/aleph/vm/orchestrator/views/authentication.py |  3 ++-
 src/aleph/vm/orchestrator/views/host_status.py    |  5 +++--
 src/aleph/vm/pool.py                              |  6 +++---
 src/aleph/vm/utils.py                             | 12 ++++++------
 9 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/src/aleph/vm/controllers/interface.py b/src/aleph/vm/controllers/interface.py
index 915fda2aa..b7afb32bf 100644
--- a/src/aleph/vm/controllers/interface.py
+++ b/src/aleph/vm/controllers/interface.py
@@ -2,7 +2,8 @@
 import logging
 from abc import ABC
 from asyncio.subprocess import Process
-from typing import Any, Coroutine, Optional
+from collections.abc import Coroutine
+from typing import Any, Optional
 
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index a0e31a049..a8b9cde4e 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -6,7 +6,7 @@
 from asyncio import Task
 from asyncio.subprocess import Process
 from pathlib import Path
-from typing import Callable, Dict, Generic, Optional, Tuple, TypedDict, TypeVar, Union
+from typing import Callable, Generic, Optional, TypedDict, TypeVar, Union
 
 import psutil
 from aleph_message.models import ItemHash
@@ -37,11 +37,10 @@
 
 
 class AlephQemuResources(AlephFirecrackerResources):
-    async def download_all(self):
+    async def download_all(self) -> None:
         volume = self.message_content.rootfs
         parent_image_path = await get_rootfs_base_path(volume.parent.ref)
         self.rootfs_path = await self.make_writable_volume(parent_image_path, volume)
-        return
 
     async def make_writable_volume(self, parent_image_path, volume: Union[PersistentVolume, RootfsVolume]):
         """Create a new qcow2 image file based on the passed one, that we give to the VM to write onto"""
@@ -90,7 +89,7 @@ class EntryDict(TypedDict):
     MESSAGE: str
 
 
-def make_logs_queue(stdout_identifier, stderr_identifier, skip_past=True) -> Tuple[asyncio.Queue, Callable[[], None]]:
+def make_logs_queue(stdout_identifier, stderr_identifier, skip_past=True) -> tuple[asyncio.Queue, Callable[[], None]]:
     """Create a queue which streams the logs for the process.
 
     @param stdout_identifier: journald identifier for process stdout
@@ -149,7 +148,7 @@ class AlephQemuInstance(Generic[ConfigurationType], CloudInitMixin, AlephVmContr
     support_snapshot = False
     qmp_socket_path = None
     persistent = True
-    _queue_cancellers: Dict[asyncio.Queue, Callable] = {}
+    _queue_cancellers: dict[asyncio.Queue, Callable] = {}
     controller_configuration: Configuration
 
     def __repr__(self):
diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index 537ed36c5..b4f6bb1f5 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -11,7 +11,7 @@
 from aleph.vm.controllers.qemu.instance import logger
 
 
-class QemuVM(object):
+class QemuVM:
     qemu_bin_path: str
     cloud_init_drive_path: Optional[str]
     image_path: str
@@ -26,7 +26,7 @@ def __repr__(self) -> str:
         if self.qemu_process:
             return f""
         else:
-            return f""
+            return ""
 
     def __init__(self, config: QemuVMConfiguration):
         self.qemu_bin_path = config.qemu_bin_path
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index b36652cb3..4dacc8abf 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -2,9 +2,10 @@
 import logging
 import uuid
 from asyncio import Task
+from collections.abc import Coroutine
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Callable, Coroutine, Optional, Union
+from typing import TYPE_CHECKING, Callable, Optional, Union
 
 from aleph_message.models import (
     ExecutableContent,
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index a34d382bf..b93a90e45 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -1,7 +1,8 @@
 import asyncio
 import logging
+from collections.abc import Iterable
 from decimal import Decimal
-from typing import Iterable, Optional
+from typing import Optional
 
 import aiohttp
 from aleph_message.models import ItemHash, PaymentType
diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index 532ce49ee..70aed4186 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -1,8 +1,9 @@
 import functools
 import json
 import logging
+from collections.abc import Awaitable, Coroutine
 from datetime import datetime, timedelta, timezone
-from typing import Any, Awaitable, Callable, Coroutine, Literal, Union
+from typing import Any, Callable, Literal, Union
 
 import pydantic
 from aiohttp import web
diff --git a/src/aleph/vm/orchestrator/views/host_status.py b/src/aleph/vm/orchestrator/views/host_status.py
index b429a1e2d..15c37dbe7 100644
--- a/src/aleph/vm/orchestrator/views/host_status.py
+++ b/src/aleph/vm/orchestrator/views/host_status.py
@@ -1,6 +1,7 @@
 import logging
 import socket
-from typing import Any, Awaitable, Callable, Optional, Tuple
+from collections.abc import Awaitable
+from typing import Any, Callable, Optional
 
 import aiohttp
 
@@ -45,7 +46,7 @@ async def check_host_egress_ipv6() -> bool:
     return await check_ip_connectivity(settings.CONNECTIVITY_IPV6_URL)
 
 
-async def resolve_dns(hostname: str) -> Tuple[Optional[str], Optional[str]]:
+async def resolve_dns(hostname: str) -> tuple[Optional[str], Optional[str]]:
     """Resolve a hostname to an IPv4 and IPv6 address."""
     ipv4: Optional[str] = None
     ipv6: Optional[str] = None
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index fd3366209..6192a124c 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -5,7 +5,7 @@
 import logging
 from collections.abc import Iterable
 from datetime import datetime, timezone
-from typing import Dict, Optional
+from typing import Optional
 
 from aleph_message.models import (
     Chain,
@@ -284,9 +284,9 @@ def get_instance_executions(self) -> Iterable[VmExecution]:
         )
         return executions or []
 
-    def get_executions_by_sender(self, payment_type: PaymentType) -> Dict[str, Dict[str, list[VmExecution]]]:
+    def get_executions_by_sender(self, payment_type: PaymentType) -> dict[str, dict[str, list[VmExecution]]]:
         """Return all executions of the given type, grouped by sender and by chain."""
-        executions_by_sender: Dict[str, Dict[str, list[VmExecution]]] = {}
+        executions_by_sender: dict[str, dict[str, list[VmExecution]]] = {}
         for vm_hash, execution in self.executions.items():
             if execution.vm_hash in (settings.CHECK_FASTAPI_VM_ID, settings.LEGACY_CHECK_FASTAPI_VM_ID):
                 # Ignore Diagnostic VM execution
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index 43ed8e306..63ce18253 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -10,7 +10,7 @@
 from dataclasses import is_dataclass
 from pathlib import Path
 from shutil import disk_usage
-from typing import Any, Callable, Dict, Optional
+from typing import Any, Callable, Optional
 
 import aiodns
 import msgpack
@@ -22,10 +22,10 @@
 logger = logging.getLogger(__name__)
 
 
-def get_message_executable_content(message_dict: Dict) -> ExecutableContent:
+def get_message_executable_content(message_dict: dict) -> ExecutableContent:
     try:
         return ProgramContent.parse_obj(message_dict)
-    except ValueError as error:
+    except ValueError:
         return InstanceContent.parse_obj(message_dict)
 
 
@@ -190,11 +190,11 @@ def to_normalized_address(value: str) -> HexAddress:
     try:
         hex_address = hexstr_if_str(to_hex, value).lower()
     except AttributeError:
-        raise TypeError("Value must be any string, instead got type {}".format(type(value)))
+        raise TypeError(f"Value must be any string, instead got type {type(value)}")
     if is_address(hex_address):
         return HexAddress(HexStr(hex_address))
     else:
-        raise ValueError("Unknown format {}, attempted to normalize to {}".format(value, hex_address))
+        raise ValueError(f"Unknown format {value}, attempted to normalize to {hex_address}")
 
 
 def md5sum(file_path: Path) -> str:
@@ -205,7 +205,7 @@ def md5sum(file_path: Path) -> str:
 def file_hashes_differ(source: Path, destination: Path, checksum: Callable[[Path], str] = md5sum) -> bool:
     """Check if the MD5 hash of two files differ."""
     if not source.exists():
-        raise FileNotFoundError("Source file does not exist: {}".format(source))
+        raise FileNotFoundError(f"Source file does not exist: {source}")
 
     if not destination.exists():
         return True

From 07d1f6b81f384e566e47e91715b1ea9490fddb78 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Mar 2024 16:48:46 +0100
Subject: [PATCH 690/990] Fix: Firecracker would not start without DNS servers,
 even when networking is disabled.

---
 src/aleph/vm/controllers/firecracker/program.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/controllers/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
index 13893f348..2a7c0082f 100644
--- a/src/aleph/vm/controllers/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -356,7 +356,7 @@ async def _setup_configuration(
         ipv6 = self.get_ipv6()
         ipv6_gateway = self.get_ipv6_gateway()
 
-        if not settings.DNS_NAMESERVERS:
+        if settings.ALLOW_VM_NETWORKING and not settings.DNS_NAMESERVERS:
             msg = "Invalid configuration: DNS nameservers missing"
             raise ValueError(msg)
 

From d6025f5209206617f8b32662f39fc120a1c72823 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Mar 2024 18:14:37 +0100
Subject: [PATCH 691/990] Fix: Workflows used deprecated actions

---
 .github/workflows/build-deb-package.yml         | 6 +++---
 .github/workflows/codeql-analysis.yml           | 6 +++---
 .github/workflows/test-new-runtime-examples.yml | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index aaac94502..825116667 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -30,7 +30,7 @@ jobs:
           cd packaging && make ${{ matrix.make_target }} && cd ..
           ls packaging/target
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.artifact_name }}
           path: packaging/target/${{ matrix.artifact_name }}
@@ -58,7 +58,7 @@ jobs:
           sudo apt install -y debootstrap
           cd runtimes/aleph-${{ matrix.os }}-python && sudo ./create_disk_image.sh && cd ../..
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.artifact_name }}
           path: runtimes/aleph-${{ matrix.os }}-python/rootfs.squashfs
@@ -74,7 +74,7 @@ jobs:
           docker build -t aleph-vm-build-squashfs -f examples/volumes/Dockerfile examples/volumes
           docker run --rm -v "$(pwd)":/mnt aleph-vm-build-squashfs
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: example-volume-venv.squashfs
           path: volume-venv.squashfs
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index d5a5fff40..d928b37c0 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -43,7 +43,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
diff --git a/.github/workflows/test-new-runtime-examples.yml b/.github/workflows/test-new-runtime-examples.yml
index a8e62d0be..f48ae2ac4 100644
--- a/.github/workflows/test-new-runtime-examples.yml
+++ b/.github/workflows/test-new-runtime-examples.yml
@@ -49,7 +49,7 @@ jobs:
           sudo apt install -y debootstrap
           cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: aleph-debian-12-python.squashfs
           path: runtimes/aleph-debian-12-python/rootfs.squashfs

From 1f6b7b89c5bb31f6f4a78fc7e128f3965912ae9d Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Mar 2024 18:33:53 +0100
Subject: [PATCH 692/990] Fix: Tests on Droplets ran even when tests failed

Solution: Only run integration tests on Droplets if code quality and pytest succeed.
---
 .github/workflows/code-quality.yml            | 65 ------------------
 .github/workflows/test-on-droplets-matrix.yml | 66 +++++++++++++++++++
 2 files changed, 66 insertions(+), 65 deletions(-)
 delete mode 100644 .github/workflows/code-quality.yml

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
deleted file mode 100644
index dff7183b8..000000000
--- a/.github/workflows/code-quality.yml
+++ /dev/null
@@ -1,65 +0,0 @@
-name: Test code quality
-
-on: push
-
-jobs:
-  code-quality-python:
-    runs-on: ubuntu-22.04
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
-        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
-
-      - name: Install required system packages only for Ubuntu Linux
-        run: |
-          sudo apt-get update
-          sudo apt-get -y upgrade
-          sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables
-          pip install --upgrade typing-extensions types-PyYAML
-
-      - name: Install required Python packages
-        run: |
-          python3 -m pip install hatch hatch-vcs coverage
-
-      - name: Test style wth ruff, black and isort
-        run: |
-          hatch run lint:style
-
-      - name: Test typing with Mypy
-        run: |
-          hatch run lint:typing
-
-      - name: Install required system packages for installing and running tests
-        run: |
-          sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
-
-      - name: Run unit tests
-        run: |
-          hatch run testing:test-cov
-          hatch run testing:cov
-
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4.0.1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          slug: aleph-im/aleph-vm
-
-  code-quality-shell:
-    runs-on: ubuntu-22.04
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
-        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
-
-      - name: Install required system packages only for Ubuntu Linux
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y shellcheck
-
-      - name: Run Shellcheck on all shell scripts
-        run: |
-          find ./ -type f -name "*.sh" -exec shellcheck {} \;
diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index a457495c4..4ffd95d38 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -4,11 +4,77 @@ on:
   push
 
 jobs:
+  tests-python:
+    name: "Test Python code"
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
+      - name: Install required system packages only for Ubuntu Linux
+        run: |
+          sudo apt-get update
+          sudo apt-get -y upgrade
+          sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables
+          pip install --upgrade typing-extensions types-PyYAML
+
+      - name: Install required Python packages
+        run: |
+          python3 -m pip install hatch hatch-vcs coverage
+
+      - name: Test style wth ruff, black and isort
+        run: |
+          hatch run lint:style
+
+      - name: Test typing with Mypy
+        run: |
+          hatch run lint:typing
+
+      - name: Install required system packages for installing and running tests
+        run: |
+          sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
+
+      - name: Run unit tests
+        run: |
+          hatch run testing:test-cov
+          hatch run testing:cov
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4.0.1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: aleph-im/aleph-vm
+
+  code-quality-shell:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
+      - name: Install required system packages only for Ubuntu Linux
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Run Shellcheck on all shell scripts
+        run: |
+          find ./ -type f -name "*.sh" -exec shellcheck {} \;
+  
+
   run_on_droplet:
     name: "Test Droplet with ${{ matrix.os_config.os_name }}-${{ matrix.check_vm.alias }}"
     runs-on: ubuntu-latest
     concurrency: "${{ matrix.os_config.concurrency_group }}-${{ matrix.check_vm.alias }}"
     timeout-minutes: 10
+    needs:
+      - tests-python
+      - code-quality-shell
 
     strategy:
       matrix:

From 867da3062138a8d0792060cab8dc632e64525b51 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Mar 2024 12:28:15 +0100
Subject: [PATCH 693/990] Fix: asyncio.run was sometimes used within a
 coroutine

`asyncio.run` was called when initializing a pool object using `VmPool.__init__(...)`.

This caused two issues:
  1. The pool was sometimes created from within a coroutine in the context of tests, and this would raise an error.
  2. Having side effects inside the `__init__` method makes objects more difficult to manipulate and test.
  3. Tests should not load persistent executions automatically.
  4. The network was configured after loading persistent executions, which could cause networking issues.

A related issue is the snapshot manager being started when initializing the `VmPool`, while this is not always desirable.

Solution proposed:
  1. Explicitly load the persistent executions using `pool.load_persistent_executions()` from the `supervisor.run()` function. This is now called after `VmPool.setup()` and therefore after the networking of the host has been configured.
  2. The snapshot manager is now started by `VmPool.setup()` instead of `VmPool.__init__`. This function is almost always called just after initializing the pool.
  3. Configuring `settings.SNAPSHOT_FREQUENCY` to zero now disables the snapshot manager.
  4. `SnapshotManager.run_snapshots` is renamed `SnapshotManager.run_in_thread` to make its behaviour more explicit.
---
 src/aleph/vm/conf.py                           |  3 ++-
 .../firecracker/snapshot_manager.py            |  2 +-
 src/aleph/vm/orchestrator/supervisor.py        |  4 ++++
 src/aleph/vm/pool.py                           | 18 +++++++++---------
 4 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 497e22ff8..db238f690 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -242,7 +242,8 @@ class Settings(BaseSettings):
 
     SNAPSHOT_FREQUENCY: int = Field(
         default=60,
-        description="Snapshot frequency interval in minutes. It will create a VM snapshot every X minutes.",
+        description="Snapshot frequency interval in minutes. It will create a VM snapshot every X minutes. "
+        "If set to zero, snapshots are disabled.",
     )
 
     SNAPSHOT_COMPRESSION_ALGORITHM: SnapshotCompressionAlgorithm = Field(
diff --git a/src/aleph/vm/controllers/firecracker/snapshot_manager.py b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
index 2a42774e0..d5f81d1eb 100644
--- a/src/aleph/vm/controllers/firecracker/snapshot_manager.py
+++ b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
@@ -86,7 +86,7 @@ def __init__(self):
         self.executions = {}
         self._scheduler = Scheduler()
 
-    def run_snapshots(self) -> None:
+    def run_in_thread(self) -> None:
         job_thread = threading.Thread(
             target=infinite_run_scheduler_jobs,
             args=[self._scheduler],
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 82f6979cb..f40031d48 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -156,6 +156,10 @@ def run():
             app.on_cleanup.append(stop_balances_monitoring_task)
             app.on_cleanup.append(stop_all_vms)
 
+        logger.info("Loading existing executions ...")
+        asyncio.run(pool.load_persistent_executions())
+
+        logger.info(f"Starting the web server on http://{settings.SUPERVISOR_HOST}:{settings.SUPERVISOR_PORT}")
         web.run_app(app, host=settings.SUPERVISOR_HOST, port=settings.SUPERVISOR_PORT)
     except OSError as e:
         if e.errno == 98:
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 6192a124c..331ab7989 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -41,7 +41,7 @@ class VmPool:
     executions: dict[ItemHash, VmExecution]
     message_cache: dict[str, ExecutableMessage] = {}
     network: Optional[Network]
-    snapshot_manager: SnapshotManager
+    snapshot_manager: Optional[SnapshotManager] = None
     systemd_manager: SystemDManager
 
     def __init__(self):
@@ -65,18 +65,18 @@ def __init__(self):
             else None
         )
         self.systemd_manager = SystemDManager()
-        self.snapshot_manager = SnapshotManager()
-        logger.debug("Initializing SnapshotManager ...")
-        self.snapshot_manager.run_snapshots()
-
-        logger.debug("Loading existing executions ...")
-        asyncio.run(self._load_persistent_executions())
+        if settings.SNAPSHOT_FREQUENCY > 0:
+            self.snapshot_manager = SnapshotManager()
 
     def setup(self) -> None:
         """Set up the VM pool and the network."""
         if self.network:
             self.network.setup()
 
+        if self.snapshot_manager:
+            logger.debug("Initializing SnapshotManager ...")
+            self.snapshot_manager.run_in_thread()
+
     def teardown(self) -> None:
         """Stop the VM pool and the network properly."""
         if self.network:
@@ -124,7 +124,7 @@ async def create_a_vm(
                 if execution.is_program and execution.vm:
                     await execution.vm.load_configuration()
 
-            if execution.vm and execution.vm.support_snapshot:
+            if execution.vm and execution.vm.support_snapshot and self.snapshot_manager:
                 await self.snapshot_manager.start_for(vm=execution.vm)
         except Exception:
             # ensure the VM is removed from the pool on creation error
@@ -210,7 +210,7 @@ async def forget_on_stop(stop_event: asyncio.Event):
 
         _ = asyncio.create_task(forget_on_stop(stop_event=execution.stop_event))
 
-    async def _load_persistent_executions(self):
+    async def load_persistent_executions(self):
         """Load persistent executions from the database."""
         saved_executions = await get_execution_records()
         for saved_execution in saved_executions:

From 7b89ae108ff53661d50be7b51890bb786e04f404 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Mar 2024 17:42:48 +0100
Subject: [PATCH 694/990] fixup! Fix: asyncio.run was sometimes used within a
 coroutine

Schedule snapshots for loaded persistent executions.
---
 src/aleph/vm/pool.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 331ab7989..8b20442e6 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -250,6 +250,9 @@ async def load_persistent_executions(self):
 
                 self._schedule_forget_on_stop(execution)
 
+                # Start the snapshot manager for the VM
+                await self.snapshot_manager.start_for(vm=execution.vm)
+
                 self.executions[vm_hash] = execution
             else:
                 execution.uuid = saved_execution.uuid

From 9fff68f454a9bb9f98d1d29ce28161d04fada931 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 7 Mar 2024 09:41:08 +0100
Subject: [PATCH 695/990] Update src/aleph/vm/pool.py

Co-authored-by: nesitor 
---
 src/aleph/vm/pool.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 8b20442e6..1dc52ba0c 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -251,7 +251,8 @@ async def load_persistent_executions(self):
                 self._schedule_forget_on_stop(execution)
 
                 # Start the snapshot manager for the VM
-                await self.snapshot_manager.start_for(vm=execution.vm)
+                if execution.vm.support_snapshot and self.snapshot_manager:
+                    await self.snapshot_manager.start_for(vm=execution.vm)
 
                 self.executions[vm_hash] = execution
             else:

From 80c22ffbd28d65ffe2aefa50c886fbb5f989faef Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 7 Mar 2024 16:16:41 +0100
Subject: [PATCH 696/990] fixup! Update src/aleph/vm/pool.py

---
 src/aleph/vm/pool.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 1dc52ba0c..190dde1f6 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -251,7 +251,7 @@ async def load_persistent_executions(self):
                 self._schedule_forget_on_stop(execution)
 
                 # Start the snapshot manager for the VM
-                if execution.vm.support_snapshot and self.snapshot_manager:
+                if vm.support_snapshot and self.snapshot_manager:
                     await self.snapshot_manager.start_for(vm=execution.vm)
 
                 self.executions[vm_hash] = execution

From 1ef12ee4c189e3a187e3907e40ff83d3176078c2 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 11 Mar 2024 14:18:17 +0100
Subject: [PATCH 697/990] Fix: Depencency aleph-message was outdated

---
 docker/vm_supervisor-dev.dockerfile | 2 +-
 examples/volumes/Dockerfile         | 2 +-
 packaging/Makefile                  | 2 +-
 pyproject.toml                      | 2 +-
 src/aleph/vm/orchestrator/README.md | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 7e1cbaed4..da730aca8 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -19,7 +19,7 @@ RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://s3.amazonaws.com/spec.ccf
 RUN ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker
 RUN ln /opt/firecracker/release-*/jailer-v* /opt/firecracker/jailer
 
-RUN pip3 install typing-extensions 'aleph-message==0.4.2'
+RUN pip3 install typing-extensions 'aleph-message==0.4.4'
 
 RUN mkdir -p /var/lib/aleph/vm/jailer
 
diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile
index 4d9a54150..c5e67993f 100644
--- a/examples/volumes/Dockerfile
+++ b/examples/volumes/Dockerfile
@@ -6,6 +6,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 RUN python3 -m venv /opt/venv
-RUN /opt/venv/bin/pip install 'aleph-message==0.4.2'
+RUN /opt/venv/bin/pip install 'aleph-message==0.4.4'
 
 CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs
diff --git a/packaging/Makefile b/packaging/Makefile
index a1d44026e..673b8f6d3 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.2' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.4' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo
diff --git a/pyproject.toml b/pyproject.toml
index 12595d92a..67bbe1cd2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,7 +29,7 @@ dependencies = [
   "aiodns==3.1.0",
   "setproctitle==1.3.3",
   "pyyaml==6.0.1",
-  "aleph-message==0.4.2",
+  "aleph-message==0.4.4",
   "jwskate==0.8.0",
   "eth-account==0.9.0",
   "sentry-sdk==1.31.0",
diff --git a/src/aleph/vm/orchestrator/README.md b/src/aleph/vm/orchestrator/README.md
index ba39e2cfb..215b5e853 100644
--- a/src/aleph/vm/orchestrator/README.md
+++ b/src/aleph/vm/orchestrator/README.md
@@ -87,7 +87,7 @@ is used to parse and validate Aleph messages.
 ```shell
 apt install -y --no-install-recommends --no-install-suggests python3-pip
 pip3 install pydantic[dotenv]
-pip3 install 'aleph-message==0.4.2'
+pip3 install 'aleph-message==0.4.4'
 ```
 
 ### 2.f. Create the jailer working directory:

From 3c56ddceb28746724241eb8833ff2196f4b1e5e6 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 12 Mar 2024 17:28:52 +0100
Subject: [PATCH 698/990] Fix: Network errors were hard to inspect

Context: We recently added the argument `check=True` to `subprocess.run(command)` to ensure we do not ignore errors in the setup or teardown of network interfaces.

Problem: Analyzing the errors that may happen during the setup or teardown of network interfaces is difficult due to the little amount of information provided by the `ip` command. It is therefore difficult to react accordingly.

Solution: Switch to use `pyroute2`, a pure Python netlink library that provides the required functions with more fined grained error reports.

The library `Pyroute2` is available in Debian 12 [1], however that version (0.7.2-2) is not documented anymore. It is absent from Debian 11. We therefore use the latest stable version from PyPI.

[1] https://packages.debian.org/bookworm/python3-pyroute2
---
 .github/workflows/test-on-droplets-matrix.yml |  6 +-
 packaging/Makefile                            |  2 +-
 pyproject.toml                                |  1 +
 src/aleph/vm/network/interfaces.py            | 86 +++++++++++++------
 tests/supervisor/test_interfaces.py           | 69 +++++++++++++++
 5 files changed, 134 insertions(+), 30 deletions(-)
 create mode 100644 tests/supervisor/test_interfaces.py

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index 4ffd95d38..e8e4edfb8 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -37,10 +37,12 @@ jobs:
         run: |
           sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
 
+        # Unit tests create and delete network interfaces, and therefore require to run as root
       - name: Run unit tests
         run: |
-          hatch run testing:test-cov
-          hatch run testing:cov
+          sudo python3 -m pip install hatch hatch-vcs coverage
+          sudo hatch run testing:test-cov
+          sudo hatch run testing:cov
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4.0.1
diff --git a/packaging/Makefile b/packaging/Makefile
index 673b8f6d3..43d8a0017 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.4' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.4' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo
diff --git a/pyproject.toml b/pyproject.toml
index 67bbe1cd2..dbfecee92 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -50,6 +50,7 @@ dependencies = [
   "aiosqlite==0.19.0",
   "alembic==1.13.1",
   "aiohttp_cors~=0.7.0",
+  "pyroute2==0.7.12",
 ]
 
 [project.urls]
diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index 854d580e2..1a79892cc 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -1,9 +1,11 @@
 import asyncio
+import errno
 import logging
 import shutil
 from ipaddress import IPv4Interface, IPv6Interface, IPv6Network
-from subprocess import run
-from typing import Optional
+from typing import Optional, Union
+
+from pyroute2 import IPRoute, NetlinkError
 
 from .ipaddresses import IPv4NetworkWithInterfaces
 from .ndp_proxy import NdpProxy
@@ -11,6 +13,52 @@
 logger = logging.getLogger(__name__)
 
 
+class InterfaceBusyError(Exception):
+    """The interface is busy."""
+
+    pass
+
+
+def create_tap_interface(ipr: IPRoute, device_name: str):
+    """Create a TAP interface with the given name. If the interface already exists, which should not happen, a warning
+    is logged and the function returns without error."""
+    try:
+        ipr.link("add", ifname=device_name, kind="tuntap", mode="tap")
+    except NetlinkError as error:
+        if error.code == 17:
+            logger.warning(f"Interface {device_name} already exists")
+        elif error.code == 16:
+            raise InterfaceBusyError(
+                f"Interface {device_name} is busy - is there another process using it ?"
+            ) from error
+        else:
+            raise
+    except OSError as error:
+        if error.errno == errno.EBUSY:
+            raise InterfaceBusyError(f"Interface {device_name} is busy. Is another process using it ?") from error
+
+
+def add_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6Interface]):
+    """Add an IP address to the given interface. If the address already exists, a warning is logged and the function
+    returns without error."""
+    try:
+        ipr.addr("add", index=ipr.link_lookup(ifname=device_name)[0], address=str(ip.ip), mask=ip.network.prefixlen)
+    except NetlinkError as e:
+        if e.code == 17:
+            logger.warning(f"Address {ip} already exists")
+        else:
+            raise
+
+
+def set_link_up(ipr: IPRoute, device_name: str):
+    """Set the given interface up."""
+    ipr.link("set", index=ipr.link_lookup(ifname=device_name)[0], state="up")
+
+
+def delete_tap_interface(ipr: IPRoute, device_name: str):
+    ipr.link("del", index=ipr.link_lookup(ifname=device_name)[0])
+
+
 class TapInterface:
     device_name: str
     ip_network: IPv4NetworkWithInterfaces
@@ -58,31 +106,14 @@ async def create(self):
         if not ip_command:
             raise FileNotFoundError("ip command not found")
 
-        run([ip_command, "tuntap", "add", self.device_name, "mode", "tap"], check=True)
-        run(
-            [
-                ip_command,
-                "addr",
-                "add",
-                str(self.host_ip.with_prefixlen),
-                "dev",
-                self.device_name,
-            ],
-            check=True,
-        )
         ipv6_gateway = self.host_ipv6
-        run(
-            [
-                ip_command,
-                "addr",
-                "add",
-                str(ipv6_gateway),
-                "dev",
-                self.device_name,
-            ],
-            check=True,
-        )
-        run([ip_command, "link", "set", self.device_name, "up"], check=True)
+
+        with IPRoute() as ipr:
+            create_tap_interface(ipr, self.device_name)
+            add_ip_address(ipr, self.device_name, self.host_ip)
+            add_ip_address(ipr, self.device_name, self.host_ipv6)
+            set_link_up(ipr, self.device_name)
+
         if self.ndp_proxy:
             await self.ndp_proxy.add_range(self.device_name, ipv6_gateway.network)
         logger.debug(f"Network interface created: {self.device_name}")
@@ -94,4 +125,5 @@ async def delete(self) -> None:
         await asyncio.sleep(0.1)  # Avoids Device/Resource busy bug
         if self.ndp_proxy:
             await self.ndp_proxy.delete_range(self.device_name)
-        run(["ip", "tuntap", "del", self.device_name, "mode", "tap"], check=True)
+        with IPRoute() as ipr:
+            delete_tap_interface(ipr, self.device_name)
diff --git a/tests/supervisor/test_interfaces.py b/tests/supervisor/test_interfaces.py
new file mode 100644
index 000000000..a309e8796
--- /dev/null
+++ b/tests/supervisor/test_interfaces.py
@@ -0,0 +1,69 @@
+from ipaddress import IPv4Interface
+from subprocess import run
+
+import pytest
+from pyroute2 import IPRoute
+
+from aleph.vm.network.interfaces import (
+    add_ip_address,
+    create_tap_interface,
+    delete_tap_interface,
+    set_link_up,
+)
+
+
+def test_create_tap_interface():
+    """Test the creation of a TAP interface and related error handling."""
+    test_device_name = "test_tap"
+    try:
+        with IPRoute() as ipr:
+            create_tap_interface(ipr, test_device_name)
+            # Check that the interface was created
+            assert run(["ip", "link", "show", test_device_name], check=False).returncode == 0
+            # Create the interface a second time, which should be ignored
+            create_tap_interface(ipr, test_device_name)
+    finally:
+        run(["ip", "tuntap", "del", test_device_name, "mode", "tap"], check=False)
+
+
+def test_add_ip_address():
+    """Test the addition of an IP address to an interface."""
+    test_device_name = "test_tap"
+    test_ipv4 = IPv4Interface(("10.10.10.10", 24))
+    try:
+        with IPRoute() as ipr:
+            # We need an interface to add an address to
+            create_tap_interface(ipr, test_device_name)
+            # Add an IP address to the interface
+            add_ip_address(ipr, test_device_name, test_ipv4)
+            # Check that the address was added
+            assert run(["ip", "address", "show", test_device_name], check=False).returncode == 0
+            # Add the same address again, which should be ignored
+            add_ip_address(ipr, test_device_name, test_ipv4)
+    finally:
+        # Delete the interface, ignoring any errors
+        run(["ip", "tuntap", "del", test_device_name, "mode", "tap"], check=False)
+
+    # Without an interface, the function should raise an error
+    with pytest.raises(IndexError):
+        add_ip_address(IPRoute(), test_device_name, test_ipv4)
+
+
+def test_link_up_down():
+    """Test the addition of an IP address to an interface."""
+    test_device_name = "test_tap"
+    try:
+        with IPRoute() as ipr:
+            # We need an interface to set the link up
+            create_tap_interface(ipr, test_device_name)
+
+            set_link_up(ipr, test_device_name)
+            # Check that the interface is up
+            assert run(["ip", "link", "show", test_device_name], check=False).returncode == 0
+            # Delete the interface
+            delete_tap_interface(ipr, test_device_name)
+            # Check that the interface is down
+            assert run(["ip", "link", "show", test_device_name], check=False).returncode != 0
+    finally:
+        # Delete the interface, ignoring any errors
+        run(["ip", "tuntap", "del", test_device_name, "mode", "tap"], check=False)

From fc2eae73ac1c18342434aaebc7f08ab4d2be42cc Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Wed, 13 Mar 2024 14:34:40 +0100
Subject: [PATCH 699/990] Fix: If the `FAKE_DATA_PROGRAM` variable have a
 default value, when we start the controllers, it take that default value
 instead to avoid it, and it fails on the checks.

Solution: Put `FAKE_DATA_PROGRAM` variable to None by default.
---
 src/aleph/vm/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index db238f690..ee7ec99f8 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -258,7 +258,7 @@ class Settings(BaseSettings):
 
     # Tests on programs
 
-    FAKE_DATA_PROGRAM: Optional[Path] = Path(abspath(join(__file__, "../../../../examples/example_fastapi")))
+    FAKE_DATA_PROGRAM: Optional[Path] = None
     BENCHMARK_FAKE_DATA_PROGRAM = Path(abspath(join(__file__, "../../../../examples/example_fastapi")))
 
     FAKE_DATA_MESSAGE = Path(abspath(join(__file__, "../../../../examples/program_message_from_aleph.json")))

From 1c0761dde99910d546612d6f3f0d42419ff2dfbb Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Wed, 13 Mar 2024 16:48:00 +0100
Subject: [PATCH 700/990] Implement default hypervisor options on settings
 (#573)

Problem: The unique way to set the default hypervisor to use is hardcoded on the `models.py` class, so we can't select another one by default.

Solution: Implement `INSTANCE_DEFAULT_HYPERVISOR` field on the orchestrator configuration variables.
---
 src/aleph/vm/conf.py   | 9 +++++++++
 src/aleph/vm/models.py | 7 +++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index ee7ec99f8..4c44bd6cc 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -11,6 +11,7 @@
 from subprocess import CalledProcessError, check_output
 from typing import Any, Literal, NewType, Optional, Union
 
+from aleph_message.models.execution.environment import HypervisorType
 from pydantic import BaseSettings, Field, HttpUrl
 from pydantic.env_settings import DotenvType, env_file_sentinel
 from pydantic.typing import StrPath
@@ -255,6 +256,10 @@ class Settings(BaseSettings):
     ALLOCATION_TOKEN_HASH = "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda"
 
     ENABLE_QEMU_SUPPORT: bool = Field(default=False)
+    INSTANCE_DEFAULT_HYPERVISOR: Optional[HypervisorType] = Field(
+        default=HypervisorType.firecracker,  # User Firecracker
+        description="Default hypervisor to use on running instances, can be Firecracker or QEmu",
+    )
 
     # Tests on programs
 
@@ -391,6 +396,10 @@ def setup(self):
                 network_interface=self.NETWORK_INTERFACE,
             )
 
+        if not settings.ENABLE_QEMU_SUPPORT:
+            # If QEmu is not supported, ignore the setting and use Firecracker by default
+            settings.INSTANCE_DEFAULT_HYPERVISOR = HypervisorType.firecracker
+
     def display(self) -> str:
         attributes: dict[str, Any] = {}
 
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 4dacc8abf..1a39655da 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -105,8 +105,11 @@ def is_instance(self) -> bool:
 
     @property
     def hypervisor(self) -> HypervisorType:
-        # default to firecracker for retro compat
-        return self.message.environment.hypervisor or HypervisorType.firecracker
+        if self.is_program:
+            return HypervisorType.firecracker
+
+        # Hypervisor setting is only used for instances
+        return self.message.environment.hypervisor or settings.INSTANCE_DEFAULT_HYPERVISOR
 
     @property
     def becomes_ready(self) -> Callable[[], Coroutine]:

From 3bc73a21acb765b3bc6155ba35613905d9e2b056 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 21:12:50 +0100
Subject: [PATCH 701/990] Fix: pyproject.toml license field must point to a
 file (#574)

---
 LICENSE        | 19 +++++++++++++++++++
 pyproject.toml |  4 ++--
 2 files changed, 21 insertions(+), 2 deletions(-)
 create mode 100644 LICENSE

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 000000000..ac3bc7b9d
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,19 @@
+Copyright (c) 2021-2024 ALEPH.IM SAS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/pyproject.toml b/pyproject.toml
index dbfecee92..4619dbd82 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,13 +8,13 @@ dynamic = ["version"]
 description = "Aleph.im VM execution engine"
 readme = "README.md"
 requires-python = ">=3.9"
-license = "MIT"
+license = {file = "LICENSE"}
 keywords = []
 authors = [
   { name="Hugo Herter", email="git@hugoherter.com" },
 ]
 classifiers = [
-  "Development Status :: 3 - Alpha",
+  "Development Status :: 4 - Beta",
   "Environment :: Console",
   "Framework :: aiohttp",
   "Intended Audience :: Information Technology",

From 0b93f6a4ac90976ce750eacbf34ded2ce40c9eab Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 15 Mar 2024 16:28:32 +0100
Subject: [PATCH 702/990] Fix: Deleting a missing interface crashed (#576)

* Fix: Deleting a missing interface crashed

Calling `delete_tap_interface` on an interface that is not present raised an error. This logs a debug message instead.

When the interface is not found for adding an IP address or setting a link up, a more explicit error is raised.

* Update src/aleph/vm/network/interfaces.py

Co-authored-by: nesitor 

---------

Co-authored-by: nesitor 
---
 src/aleph/vm/network/interfaces.py  | 22 +++++++++++++++++++---
 tests/supervisor/test_interfaces.py |  3 ++-
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index 1a79892cc..fb9d10d7f 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -13,6 +13,12 @@
 logger = logging.getLogger(__name__)
 
 
+class MissingInterfaceError(Exception):
+    """The interface is missing."""
+
+    pass
+
+
 class InterfaceBusyError(Exception):
     """The interface is busy."""
 
@@ -41,8 +47,11 @@ def create_tap_interface(ipr: IPRoute, device_name: str):
 def add_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6Interface]):
     """Add an IP address to the given interface. If the address already exists, a warning is logged and the function
     returns without error."""
+    interface_index: list[int] = ipr.link_lookup(ifname=device_name)
+    if not interface_index:
+        raise MissingInterfaceError(f"Interface {device_name} does not exist, can't add address {ip} to it.")
     try:
-        ipr.addr("add", index=ipr.link_lookup(ifname=device_name)[0], address=str(ip.ip), mask=ip.network.prefixlen)
+        ipr.addr("add", index=interface_index[0], address=str(ip.ip), mask=ip.network.prefixlen)
     except NetlinkError as e:
         if e.code == 17:
             logger.warning(f"Address {ip} already exists")
@@ -52,11 +61,18 @@ def add_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6
 
 def set_link_up(ipr: IPRoute, device_name: str):
     """Set the given interface up."""
-    ipr.link("set", index=ipr.link_lookup(ifname=device_name)[0], state="up")
+    interface_index: list[int] = ipr.link_lookup(ifname=device_name)
+    if not interface_index:
+        raise MissingInterfaceError(f"Interface {device_name} does not exist, can't set it up.")
+    ipr.link("set", index=interface_index[0], state="up")
 
 
 def delete_tap_interface(ipr: IPRoute, device_name: str):
-    ipr.link("del", index=ipr.link_lookup(ifname=device_name)[0])
+    interface_index: list[int] = ipr.link_lookup(ifname=device_name)
+    if not interface_index:
+        logger.debug(f"Interface {device_name} does not exist, won't be deleted.")
+        return
+    ipr.link("del", index=interface_index[0])
 
 
 class TapInterface:
diff --git a/tests/supervisor/test_interfaces.py b/tests/supervisor/test_interfaces.py
index a309e8796..79868b85e 100644
--- a/tests/supervisor/test_interfaces.py
+++ b/tests/supervisor/test_interfaces.py
@@ -5,6 +5,7 @@
 from pyroute2 import IPRoute
 
 from aleph.vm.network.interfaces import (
+    MissingInterfaceError,
     add_ip_address,
     create_tap_interface,
     delete_tap_interface,
@@ -45,7 +46,7 @@ def test_add_ip_address():
         run(["ip", "tuntap", "del", test_device_name, "mode", "tap"], check=False)
 
     # Without an interface, the function should raise an error
-    with pytest.raises(IndexError):
+    with pytest.raises(MissingInterfaceError):
         add_ip_address(IPRoute(), test_device_name, test_ipv4)
 
 

From e7d97fa171daaf7cc4d7aef4ff84e3fbcc8d4a39 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 19 Mar 2024 16:54:36 +0100
Subject: [PATCH 703/990] Fix: Interface being busy prevented instance creation

When attempting to schedule a Firecracker instance, the error below came in a loop.

Solution: Log a warning instead of raising an exception.

```
 2024-03-19 15:45:49,346 | ERROR |   File "", line 198, in _run_module_as_main
   File "", line 88, in _run_code
   File "/opt/aleph-vm/aleph/vm/orchestrator/__main__.py", line 4, in 
     main()
   File "/opt/aleph-vm/aleph/vm/orchestrator/cli.py", line 371, in main
     supervisor.run()
   File "/opt/aleph-vm/aleph/vm/orchestrator/supervisor.py", line 163, in run
     web.run_app(app, host=settings.SUPERVISOR_HOST, port=settings.SUPERVISOR_PORT)
   File "/opt/aleph-vm/aiohttp/web.py", line 544, in run_app
     loop.run_until_complete(main_task)
   File "/usr/lib/python3.11/asyncio/base_events.py", line 640, in run_until_complete
     self.run_forever()
   File "/usr/lib/python3.11/asyncio/base_events.py", line 607, in run_forever
     self._run_once()
   File "/usr/lib/python3.11/asyncio/base_events.py", line 1922, in _run_once
     handle._run()
   File "/usr/lib/python3.11/asyncio/events.py", line 80, in _run
     self._context.run(self._callback, *self._args)
   File "/opt/aleph-vm/aiohttp/web_protocol.py", line 452, in _handle_request
     resp = await request_handler(request)
   File "/opt/aleph-vm/sentry_sdk/integrations/aiohttp.py", line 129, in sentry_app_handle
     response = await old_handle(self, request)
   File "/opt/aleph-vm/aiohttp/web_app.py", line 543, in _handle
     resp = await handler(request)
   File "/opt/aleph-vm/aiohttp/web_middlewares.py", line 114, in impl
     return await handler(request)
   File "/opt/aleph-vm/aleph/vm/orchestrator/supervisor.py", line 65, in server_version_middleware
     resp: web.StreamResponse = await handler(request)
   File "/opt/aleph-vm/aiohttp/web_urldispatcher.py", line 200, in handler_wrapper
     result = await result
   File "/opt/aleph-vm/aleph/vm/orchestrator/run.py", line 129, in run_code_on_request
     execution = await create_vm_execution_or_raise_http_error(vm_hash=vm_hash, pool=pool)
   File "/opt/aleph-vm/aleph/vm/orchestrator/run.py", line 90, in create_vm_execution_or_raise_http_error
     return await create_vm_execution(vm_hash=vm_hash, pool=pool)
   File "/opt/aleph-vm/aleph/vm/orchestrator/run.py", line 60, in create_vm_execution
     execution = await pool.create_a_vm(
   File "/opt/aleph-vm/aleph/vm/pool.py", line 113, in create_a_vm
     await self.network.create_tap(vm_id, tap_interface)
   File "/opt/aleph-vm/aleph/vm/network/hostnetwork.py", line 221, in create_tap
     await interface.create()
   File "/opt/aleph-vm/aleph/vm/network/interfaces.py", line 128, in create
     create_tap_interface(ipr, self.device_name)
   File "/opt/aleph-vm/aleph/vm/network/interfaces.py", line 32, in create_tap_interface
     ipr.link("add", ifname=device_name, kind="tuntap", mode="tap")
   File "/opt/aleph-vm/pyroute2/iproute/linux.py", line 1696, in link
     ret = self.nlm_request(msg, msg_type=msg_type, msg_flags=msg_flags)
   File "/opt/aleph-vm/pyroute2/netlink/nlsocket.py", line 870, in nlm_request
     return tuple(self._genlm_request(*argv, **kwarg))
   File "/opt/aleph-vm/pyroute2/netlink/nlsocket.py", line 1209, in nlm_request
     self.put(msg, msg_type, msg_flags, msg_seq=msg_seq)
   File "/opt/aleph-vm/pyroute2/netlink/nlsocket.py", line 906, in put
     return self.engine.put(
   File "/opt/aleph-vm/pyroute2/netlink/nlsocket.py", line 443, in put
     self.socket.sendto_gate(msg, addr)
   File "/opt/aleph-vm/pyroute2/netlink/rtnl/iprsocket.py", line 52, in sendto_gate
     ret = self._sproxy.handle(msg)
   File "/opt/aleph-vm/pyroute2/netlink/proxy.py", line 61, in handle
     log.error(''.join(traceback.format_stack()))
 2024-03-19 15:45:49,353 | ERROR | Traceback (most recent call last):
   File "/opt/aleph-vm/pyroute2/netlink/proxy.py", line 43, in handle
     ret = plugin(msg, self.nl)
           ^^^^^^^^^^^^^^^^^^^^
   File "/opt/aleph-vm/pyroute2/netlink/rtnl/ifinfmsg/proxy.py", line 73, in proxy_newlink
     return manage_tuntap(msg)
            ^^^^^^^^^^^^^^^^^^
   File "/opt/aleph-vm/pyroute2/netlink/rtnl/ifinfmsg/sync.py", line 60, in decorated
     ret = f(msg)
           ^^^^^^
   File "/opt/aleph-vm/pyroute2/netlink/rtnl/ifinfmsg/tuntap.py", line 135, in manage_tuntap
     ioctl(fd, TUNSETIFF, ifr)
 OSError: [Errno 16] Device or resource busy
 2024-03-19 15:45:49,356 | ERROR | Interface vmtap4 is busy - is there another process using it ?
 Traceback (most recent call last):
   File "/opt/aleph-vm/aleph/vm/network/interfaces.py", line 32, in create_tap_interface
     ipr.link("add", ifname=device_name, kind="tuntap", mode="tap")
   File "/opt/aleph-vm/pyroute2/iproute/linux.py", line 1696, in link
     ret = self.nlm_request(msg, msg_type=msg_type, msg_flags=msg_flags)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   File "/opt/aleph-vm/pyroute2/netlink/nlsocket.py", line 870, in nlm_request
     return tuple(self._genlm_request(*argv, **kwarg))
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   File "/opt/aleph-vm/pyroute2/netlink/nlsocket.py", line 1214, in nlm_request
     for msg in self.get(
                ^^^^^^^^^
   File "/opt/aleph-vm/pyroute2/netlink/nlsocket.py", line 873, in get
     return tuple(self._genlm_get(*argv, **kwarg))
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   File "/opt/aleph-vm/pyroute2/netlink/nlsocket.py", line 550, in get
     raise msg['header']['error']
 pyroute2.netlink.exceptions.NetlinkError: (16, 'Device or resource busy')
 The above exception was the direct cause of the following exception:
 Traceback (most recent call last):
   File "/opt/aleph-vm/aleph/vm/orchestrator/run.py", line 90, in create_vm_execution_or_raise_http_error
     return await create_vm_execution(vm_hash=vm_hash, pool=pool)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   File "/opt/aleph-vm/aleph/vm/orchestrator/run.py", line 60, in create_vm_execution
     execution = await pool.create_a_vm(
                 ^^^^^^^^^^^^^^^^^^^^^^^
   File "/opt/aleph-vm/aleph/vm/pool.py", line 113, in create_a_vm
     await self.network.create_tap(vm_id, tap_interface)
   File "/opt/aleph-vm/aleph/vm/network/hostnetwork.py", line 221, in create_tap
     await interface.create()
   File "/opt/aleph-vm/aleph/vm/network/interfaces.py", line 128, in create
     create_tap_interface(ipr, self.device_name)
   File "/opt/aleph-vm/aleph/vm/network/interfaces.py", line 37, in create_tap_interface
     raise InterfaceBusyError(
 aleph.vm.network.interfaces.InterfaceBusyError: Interface vmtap4 is busy - is there another process using it ?
 2024-03-19 15:45:49,362 | INFO | 127.0.0.1 [19/Mar/2024:15:45:30 +0000] "GET /vm/3fc0aa9569da840c43e7bd2033c3c580abb4
 ```
---
 src/aleph/vm/network/interfaces.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index fb9d10d7f..6da33db11 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -34,14 +34,12 @@ def create_tap_interface(ipr: IPRoute, device_name: str):
         if error.code == 17:
             logger.warning(f"Interface {device_name} already exists")
         elif error.code == 16:
-            raise InterfaceBusyError(
-                f"Interface {device_name} is busy - is there another process using it ?"
-            ) from error
+            logger.warning(f"Interface {device_name} is busy - is there another process using it ?")
         else:
             raise
     except OSError as error:
         if error.errno == errno.EBUSY:
-            raise InterfaceBusyError(f"Interface {device_name} is busy. Is another process using it ?") from error
+            logger.warning(f"Interface {device_name} is busy - is there another process using it ?")
 
 
 def add_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6Interface]):

From 643004b746a42fa31c626aa0802c0a41513761d0 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 19 Mar 2024 21:41:52 +0100
Subject: [PATCH 704/990] Fix: Waiting for another tasks reached timeout

An error was reported on Sentry with the timeout waiting for another task being reached.

The `return` of the function upon success was absent, hence multiple attempts being always retried.

This refactors the download login with 10 attempts to download the file and a 30 seconds timeout waiting for another task.

If the other task ends up failing, this will now take over the download.

The raise of exceptions is also improved for better traceability.
---
 src/aleph/vm/storage.py | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index 350414a0e..cf72440a2 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -90,28 +90,37 @@ async def download_file(url: str, local_path: Path) -> None:
     # Avoid partial downloads and incomplete files by only moving the file when it's complete.
     tmp_path = Path(f"{local_path}.part")
 
-    # Ensure the file is not being downloaded by another task in parallel.
-    try:
-        tmp_path.touch(exist_ok=False)
-    except FileExistsError:
-        # Another task is already downloading the file
-        # Use `asyncio.timeout` manager after dropping support for Python 3.10
-        await asyncio.wait_for(file_downloaded_by_another_task(local_path), timeout=300)
-
     logger.debug(f"Downloading {url} -> {tmp_path}")
-    download_attempts = 3
+    download_attempts = 10
     for attempt in range(download_attempts):
+        logger.debug(f"Download attempt {attempt + 1}/{download_attempts}...")
         try:
+            # Ensure the file is not being downloaded by another task in parallel.
+            tmp_path.touch(exist_ok=False)
+
             await download_file_in_chunks(url, tmp_path)
             tmp_path.rename(local_path)
             logger.debug(f"Download complete, moved {tmp_path} -> {local_path}")
+            return
+        except FileExistsError as file_exists_error:
+            # Another task is already downloading the file.
+            # Use `asyncio.timeout` manager after dropping support for Python 3.10
+            logger.debug(f"File already being downloaded by another task: {local_path}")
+            try:
+                await asyncio.wait_for(file_downloaded_by_another_task(local_path), timeout=30)
+            except TimeoutError as error:
+                if attempt < (download_attempts - 1):
+                    logger.warning(f"Download failed, retrying attempt {attempt + 1}/{download_attempts}...")
+                    continue
+                else:
+                    raise error from file_exists_error
         except (
             aiohttp.ClientConnectionError,
             aiohttp.ClientResponseError,
             aiohttp.ClientPayloadError,
         ) as error:
             if attempt < (download_attempts - 1):
-                logger.warning(f"Download failed, retrying attempt {attempt + 1}/3...")
+                logger.warning(f"Download failed, retrying attempt {attempt + 1}/{download_attempts}...")
                 continue
             else:
                 raise error

From 4ba5c4fb59a0695ce4c8a763653aa56b48708d87 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 20 Mar 2024 17:00:47 +0100
Subject: [PATCH 705/990] Fix: Failing to restart `nddpd` caused the exception
 to escalate

The `ndppd` service is not always required and restarting it can fail.

While waiting for a better fix that prevents restarting the service too quickly, this logs an error instead of raising an exception.
---
 src/aleph/vm/network/ndp_proxy.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/network/ndp_proxy.py b/src/aleph/vm/network/ndp_proxy.py
index 0af97b7d4..60e73b287 100644
--- a/src/aleph/vm/network/ndp_proxy.py
+++ b/src/aleph/vm/network/ndp_proxy.py
@@ -14,6 +14,7 @@
 from dataclasses import dataclass
 from ipaddress import IPv6Network
 from pathlib import Path
+from subprocess import CalledProcessError
 
 from aleph.vm.utils import run_in_subprocess
 
@@ -33,7 +34,11 @@ def __init__(self, host_network_interface: str):
     @staticmethod
     async def _restart_ndppd():
         logger.debug("Restarting ndppd")
-        await run_in_subprocess(["systemctl", "restart", "ndppd"])
+        try:
+            await run_in_subprocess(["systemctl", "restart", "ndppd"])
+        except CalledProcessError as error:
+            logger.error("Failed to restart ndppd: %s", error)
+            # We do not raise the error here, since this should not crash the entire system
 
     async def _update_ndppd_conf(self):
         config = f"proxy {self.host_network_interface} {{\n"

From 79f8d0e1007fed3f04828a1357e25edef2e33295 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 20 Mar 2024 16:57:34 +0100
Subject: [PATCH 706/990] Fix: Guest API would crash due to FileNotFound

The parent directory on which the Guest APi listens did not exist in some context (probably Firecracker based instances), which crashed the process.

This ensures that the directories exist and that the Unix socket can therefore be created.
---
 src/aleph/vm/controllers/firecracker/executable.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index d3309c373..ad1a8364e 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -296,8 +296,12 @@ async def load_configuration(self):
         return
 
     async def start_guest_api(self):
-        logger.debug(f"starting guest API for {self.vm_id}")
-        vsock_path = f"{self.fvm.vsock_path}_53"
+        vsock_path = Path(f"{self.fvm.vsock_path}_53")
+
+        # Ensure that the directory where the VSOCK socket will be created exists
+        vsock_path.parent.mkdir(parents=True, exist_ok=True)
+        logger.debug(f"starting guest API for {self.vm_id} on {vsock_path}")
+
         vm_hash = self.vm_hash
         self.guest_api_process = Process(
             target=run_guest_api,

From 46fbf2d6c4e6b3e787ada2cc6c14be7dad492de0 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 21 Mar 2024 15:00:39 +0100
Subject: [PATCH 707/990] Problem: After the change of the new interface
 library to handle network interfaces, when you call 2 VMs at the same time,
 it assigns to you the same VM_ID and the network setup fails.

Solution: Implement semaphores on the execution creation to avoid create VMs at the same time.
---
 src/aleph/vm/network/interfaces.py          | 22 +++++++++++++++++----
 src/aleph/vm/orchestrator/views/__init__.py |  4 ++--
 src/aleph/vm/pool.py                        |  8 ++++++++
 3 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index 6da33db11..f51e5a732 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -36,10 +36,12 @@ def create_tap_interface(ipr: IPRoute, device_name: str):
         elif error.code == 16:
             logger.warning(f"Interface {device_name} is busy - is there another process using it ?")
         else:
-            raise
+            logger.error(f"Unknown exception while creating interface {device_name}: {error}")
     except OSError as error:
         if error.errno == errno.EBUSY:
             logger.warning(f"Interface {device_name} is busy - is there another process using it ?")
+        else:
+            logger.error(f"Unknown exception while creating interface {device_name}: {error}")
 
 
 def add_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6Interface]):
@@ -54,7 +56,9 @@ def add_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6
         if e.code == 17:
             logger.warning(f"Address {ip} already exists")
         else:
-            raise
+            logger.error(f"Unknown exception while adding address {ip} to interface {device_name}: {e}")
+    except OSError as e:
+        logger.error(f"Unknown exception while adding address {ip} to interface {device_name}: {e}")
 
 
 def set_link_up(ipr: IPRoute, device_name: str):
@@ -62,7 +66,12 @@ def set_link_up(ipr: IPRoute, device_name: str):
     interface_index: list[int] = ipr.link_lookup(ifname=device_name)
     if not interface_index:
         raise MissingInterfaceError(f"Interface {device_name} does not exist, can't set it up.")
-    ipr.link("set", index=interface_index[0], state="up")
+    try:
+        ipr.link("set", index=interface_index[0], state="up")
+    except NetlinkError as e:
+        logger.error(f"Unknown exception while setting link up to interface {device_name}: {e}")
+    except OSError as e:
+        logger.error(f"Unknown exception while setting link up to interface {device_name}: {e}")
 
 
 def delete_tap_interface(ipr: IPRoute, device_name: str):
@@ -70,7 +79,12 @@ def delete_tap_interface(ipr: IPRoute, device_name: str):
     if not interface_index:
         logger.debug(f"Interface {device_name} does not exist, won't be deleted.")
         return
-    ipr.link("del", index=interface_index[0])
+    try:
+        ipr.link("del", index=interface_index[0])
+    except NetlinkError as error:
+        logger.warning(f"Interface {device_name} cannot be deleted: {error}")
+    except OSError as error:
+        logger.warning(f"Interface {device_name} cannot be deleted: {error}")
 
 
 class TapInterface:
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 91ad43340..600c71f63 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -57,7 +57,7 @@
 logger = logging.getLogger(__name__)
 
 
-def run_code_from_path(request: web.Request) -> Awaitable[web.Response]:
+async def run_code_from_path(request: web.Request) -> web.Response:
     """Allow running an Aleph VM function from a URL path
 
     The path is expected to follow the scheme defined in `app.add_routes` below,
@@ -68,7 +68,7 @@ def run_code_from_path(request: web.Request) -> Awaitable[web.Response]:
 
     message_ref = ItemHash(request.match_info["ref"])
     pool: VmPool = request.app["vm_pool"]
-    return run_code_on_request(message_ref, path, pool, request)
+    return await run_code_on_request(message_ref, path, pool, request)
 
 
 async def run_code_from_hostname(request: web.Request) -> web.Response:
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 190dde1f6..0c35b2d7f 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -43,10 +43,12 @@ class VmPool:
     network: Optional[Network]
     snapshot_manager: Optional[SnapshotManager] = None
     systemd_manager: SystemDManager
+    creation_semaphore: asyncio.Semaphore
 
     def __init__(self):
         self.counter = settings.START_ID_INDEX
         self.executions = {}
+        self.creation_semaphore = asyncio.Semaphore(1)
 
         self.network = (
             Network(
@@ -87,6 +89,8 @@ async def create_a_vm(
     ) -> VmExecution:
         """Create a new Aleph Firecracker VM from an Aleph function message."""
 
+        await self.creation_semaphore.acquire()
+
         # Check if an execution is already present for this VM, then return it.
         # Do not `await` in this section.
         current_execution = self.get_running_vm(vm_hash)
@@ -130,9 +134,13 @@ async def create_a_vm(
             # ensure the VM is removed from the pool on creation error
             self.forget_vm(vm_hash)
             raise
+        finally:
+            self.creation_semaphore.release()
 
         self._schedule_forget_on_stop(execution)
 
+        self.creation_semaphore.release()
+
         return execution
 
     def get_unique_vm_id(self) -> int:

From 3bf413f6a3f0761a34bbcfbe00197cd1516b947f Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 21 Mar 2024 15:58:24 +0100
Subject: [PATCH 708/990] Fix: Instead using a semaphore, changed to use a
 lock.

---
 src/aleph/vm/pool.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 0c35b2d7f..eff87f40b 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -3,6 +3,7 @@
 import asyncio
 import json
 import logging
+import threading
 from collections.abc import Iterable
 from datetime import datetime, timezone
 from typing import Optional
@@ -43,12 +44,12 @@ class VmPool:
     network: Optional[Network]
     snapshot_manager: Optional[SnapshotManager] = None
     systemd_manager: SystemDManager
-    creation_semaphore: asyncio.Semaphore
+    creation_lock: threading.Lock
 
     def __init__(self):
         self.counter = settings.START_ID_INDEX
         self.executions = {}
-        self.creation_semaphore = asyncio.Semaphore(1)
+        self.creation_lock = threading.Lock()
 
         self.network = (
             Network(
@@ -89,7 +90,7 @@ async def create_a_vm(
     ) -> VmExecution:
         """Create a new Aleph Firecracker VM from an Aleph function message."""
 
-        await self.creation_semaphore.acquire()
+        self.creation_lock.acquire()
 
         # Check if an execution is already present for this VM, then return it.
         # Do not `await` in this section.
@@ -135,12 +136,10 @@ async def create_a_vm(
             self.forget_vm(vm_hash)
             raise
         finally:
-            self.creation_semaphore.release()
+            self.creation_lock.release()
 
         self._schedule_forget_on_stop(execution)
 
-        self.creation_semaphore.release()
-
         return execution
 
     def get_unique_vm_id(self) -> int:

From 73e51abd4e524e8cc9b9369ab5ebd36764999be6 Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 21 Mar 2024 17:36:41 +0100
Subject: [PATCH 709/990] Fix: Use context manager instead integrate it on try
 and catch.

---
 src/aleph/vm/pool.py | 89 +++++++++++++++++++++-----------------------
 1 file changed, 43 insertions(+), 46 deletions(-)

diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index eff87f40b..daee88424 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -90,57 +90,54 @@ async def create_a_vm(
     ) -> VmExecution:
         """Create a new Aleph Firecracker VM from an Aleph function message."""
 
-        self.creation_lock.acquire()
+        with self.creation_lock:
+            # Check if an execution is already present for this VM, then return it.
+            # Do not `await` in this section.
+            current_execution = self.get_running_vm(vm_hash)
+            if current_execution:
+                return current_execution
+            else:
+                execution = VmExecution(
+                    vm_hash=vm_hash,
+                    message=message,
+                    original=original,
+                    snapshot_manager=self.snapshot_manager,
+                    systemd_manager=self.systemd_manager,
+                    persistent=persistent,
+                )
+                self.executions[vm_hash] = execution
 
-        # Check if an execution is already present for this VM, then return it.
-        # Do not `await` in this section.
-        current_execution = self.get_running_vm(vm_hash)
-        if current_execution:
-            return current_execution
-        else:
-            execution = VmExecution(
-                vm_hash=vm_hash,
-                message=message,
-                original=original,
-                snapshot_manager=self.snapshot_manager,
-                systemd_manager=self.systemd_manager,
-                persistent=persistent,
-            )
-            self.executions[vm_hash] = execution
+            try:
+                await execution.prepare()
+                vm_id = self.get_unique_vm_id()
 
-        try:
-            await execution.prepare()
-            vm_id = self.get_unique_vm_id()
+                if self.network:
+                    vm_type = VmType.from_message_content(message)
+                    tap_interface = await self.network.prepare_tap(vm_id, vm_hash, vm_type)
+                    await self.network.create_tap(vm_id, tap_interface)
+                else:
+                    tap_interface = None
 
-            if self.network:
-                vm_type = VmType.from_message_content(message)
-                tap_interface = await self.network.prepare_tap(vm_id, vm_hash, vm_type)
-                await self.network.create_tap(vm_id, tap_interface)
-            else:
-                tap_interface = None
+                execution.create(vm_id=vm_id, tap_interface=tap_interface)
+                await execution.start()
 
-            execution.create(vm_id=vm_id, tap_interface=tap_interface)
-            await execution.start()
+                # Start VM and snapshots automatically
+                if execution.persistent:
+                    self.systemd_manager.enable_and_start(execution.controller_service)
+                    await execution.wait_for_init()
+                    if execution.is_program and execution.vm:
+                        await execution.vm.load_configuration()
 
-            # Start VM and snapshots automatically
-            if execution.persistent:
-                self.systemd_manager.enable_and_start(execution.controller_service)
-                await execution.wait_for_init()
-                if execution.is_program and execution.vm:
-                    await execution.vm.load_configuration()
-
-            if execution.vm and execution.vm.support_snapshot and self.snapshot_manager:
-                await self.snapshot_manager.start_for(vm=execution.vm)
-        except Exception:
-            # ensure the VM is removed from the pool on creation error
-            self.forget_vm(vm_hash)
-            raise
-        finally:
-            self.creation_lock.release()
-
-        self._schedule_forget_on_stop(execution)
-
-        return execution
+                if execution.vm and execution.vm.support_snapshot and self.snapshot_manager:
+                    await self.snapshot_manager.start_for(vm=execution.vm)
+            except Exception:
+                # ensure the VM is removed from the pool on creation error
+                self.forget_vm(vm_hash)
+                raise
+
+            self._schedule_forget_on_stop(execution)
+
+            return execution
 
     def get_unique_vm_id(self) -> int:
         """Get a unique identifier for the VM.

From ab29137a91de72b57d46b1d2d31e7957aadbf92c Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Thu, 21 Mar 2024 22:00:31 +0100
Subject: [PATCH 710/990] Fix: Changed threads implementation by asyncio
 implementation.

---
 src/aleph/vm/orchestrator/supervisor.py |  3 ++-
 src/aleph/vm/pool.py                    | 12 ++++++------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index f40031d48..20452df90 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -134,7 +134,8 @@ def run():
     engine = setup_engine()
     asyncio.run(create_tables(engine))
 
-    pool = VmPool()
+    loop = asyncio.new_event_loop()
+    pool = VmPool(loop)
     pool.setup()
 
     hostname = settings.DOMAIN_NAME
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index daee88424..02889f68e 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -3,7 +3,6 @@
 import asyncio
 import json
 import logging
-import threading
 from collections.abc import Iterable
 from datetime import datetime, timezone
 from typing import Optional
@@ -44,12 +43,14 @@ class VmPool:
     network: Optional[Network]
     snapshot_manager: Optional[SnapshotManager] = None
     systemd_manager: SystemDManager
-    creation_lock: threading.Lock
+    creation_lock: asyncio.Lock
 
-    def __init__(self):
+    def __init__(self, loop: asyncio.AbstractEventLoop):
         self.counter = settings.START_ID_INDEX
         self.executions = {}
-        self.creation_lock = threading.Lock()
+
+        asyncio.set_event_loop(loop)
+        self.creation_lock = asyncio.Lock()
 
         self.network = (
             Network(
@@ -89,8 +90,7 @@ async def create_a_vm(
         self, vm_hash: ItemHash, message: ExecutableContent, original: ExecutableContent, persistent: bool
     ) -> VmExecution:
         """Create a new Aleph Firecracker VM from an Aleph function message."""
-
-        with self.creation_lock:
+        async with self.creation_lock:
             # Check if an execution is already present for this VM, then return it.
             # Do not `await` in this section.
             current_execution = self.get_running_vm(vm_hash)

From b6053f617babb4fa2a45bb44b2825307389f24fc Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 18 Mar 2024 14:46:18 +0100
Subject: [PATCH 711/990] Fix: Tests were run twice

The command `hatch run testing:cov` includes running `hatch run testing:test-cov`.

This removes the duplication
---
 .github/workflows/test-on-droplets-matrix.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index e8e4edfb8..61853bf7e 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -41,7 +41,6 @@ jobs:
       - name: Run unit tests
         run: |
           sudo python3 -m pip install hatch hatch-vcs coverage
-          sudo hatch run testing:test-cov
           sudo hatch run testing:cov
 
       - name: Upload coverage reports to Codecov

From b90cb84dbccce3b6fcfc0038295adb354a9feebe Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 19 Mar 2024 22:42:58 +0100
Subject: [PATCH 712/990] Fix: Log level was DEBUG instead of WARNING in prod

Solution: Remove the `--very-verbose` argument from the SystemD service definition.
---
 .../aleph-vm/etc/systemd/system/aleph-vm-supervisor.service     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service b/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
index ab4006c2a..9cee47227 100644
--- a/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
+++ b/packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service
@@ -10,7 +10,7 @@ WorkingDirectory=/opt/aleph-vm
 Environment=PYTHONPATH=/opt/aleph-vm/:$PYTHONPATH
 Environment=PYTHONDONTWRITEBYTECODE="enabled"
 EnvironmentFile=/etc/aleph-vm/supervisor.env
-ExecStart=python3 -m aleph.vm.orchestrator --print-settings --very-verbose
+ExecStart=python3 -m aleph.vm.orchestrator --print-settings
 Restart=always
 RestartSec=10s
 

From a355428d05220928e5bbce0ce6a92d76d5feb44c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:36:33 +0100
Subject: [PATCH 713/990] Fix typing: web.HTTPBadRequest may not be raised

---
 src/aleph/vm/orchestrator/views/__init__.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 600c71f63..78cf4d61d 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -434,12 +434,12 @@ async def notify_allocation(request: web.Request):
     try:
         data = await request.json()
         vm_notification = VMNotification.parse_obj(data)
-    except JSONDecodeError as error:
-        raise web.HTTPBadRequest(reason="Body is not valid JSON") from error
+    except JSONDecodeError:
+        return web.HTTPBadRequest(reason="Body is not valid JSON")
     except ValidationError as error:
-        raise web.json_response(
+        return web.json_response(
             data=error.json(), status=web.HTTPBadRequest.status_code, headers={"Access-Control-Allow-Origin": "*"}
-        ) from error
+        )
 
     pubsub: PubSub = request.app["pubsub"]
     pool: VmPool = request.app["vm_pool"]
@@ -447,13 +447,13 @@ async def notify_allocation(request: web.Request):
     item_hash: ItemHash = vm_notification.instance
     message = await try_get_message(item_hash)
     if message.type != MessageType.instance:
-        raise web.HTTPBadRequest(reason="Message is not an instance")
+        return web.HTTPBadRequest(reason="Message is not an instance")
 
     if not message.content.payment:
-        raise web.HTTPBadRequest(reason="Message does not have payment information")
+        return web.HTTPBadRequest(reason="Message does not have payment information")
 
     if message.content.payment.receiver != settings.PAYMENT_RECEIVER_ADDRESS:
-        raise web.HTTPBadRequest(reason="Message is not for this instance")
+        return web.HTTPBadRequest(reason="Message is not for this instance")
 
     # Check that there is a payment stream for this instance
     try:
@@ -462,7 +462,7 @@ async def notify_allocation(request: web.Request):
         )
     except InvalidAddressError as error:
         logger.warning(f"Invalid address {error}", exc_info=True)
-        raise web.HTTPBadRequest(reason=f"Invalid address {error}") from error
+        return web.HTTPBadRequest(reason=f"Invalid address {error}")
 
     if not active_flow:
         raise web.HTTPPaymentRequired(reason="Empty payment stream for this instance")
@@ -472,7 +472,7 @@ async def notify_allocation(request: web.Request):
     if active_flow < required_flow:
         active_flow_per_month = active_flow * 60 * 60 * 24 * (Decimal("30.41666666666923904761904784"))
         required_flow_per_month = required_flow * 60 * 60 * 24 * Decimal("30.41666666666923904761904784")
-        raise web.HTTPPaymentRequired(
+        return web.HTTPPaymentRequired(
             reason="Insufficient payment stream",
             text="Insufficient payment stream for this instance\n\n"
             f"Required: {required_flow_per_month} / month (flow = {required_flow})\n"

From a6508f145e8c3a7d7c5be404db8943fcc81e7190 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:36:58 +0100
Subject: [PATCH 714/990] Fix typing: Wrong type in annotation

---
 src/aleph/vm/orchestrator/views/authentication.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index 70aed4186..f1acb2f01 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -218,7 +218,7 @@ async def authenticate_websocket_message(message) -> str:
 
 def require_jwk_authentication(
     handler: Callable[[web.Request, str], Coroutine[Any, Any, web.StreamResponse]]
-) -> Callable[[web.Response], Awaitable[web.StreamResponse]]:
+) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
     @functools.wraps(handler)
     async def wrapper(request):
         try:

From b65fd9ebe9fe5363b1efb7a3b3ccfb884a73a857 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:38:09 +0100
Subject: [PATCH 715/990] Fix typing: FakeRequest class was not of type Request

---
 src/aleph/vm/orchestrator/cli.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index 2a7cf783c..94b17d305 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -8,6 +8,7 @@
 from pathlib import Path
 from statistics import mean
 from typing import Callable
+from typing import Callable, Optional, cast
 
 from aiohttp.web import Request, Response
 from sqlalchemy.ext.asyncio import create_async_engine
@@ -157,6 +158,15 @@ def parse_args(args):
     return parser.parse_args(args)
 
 
+class FakeRequest:
+    headers: dict[str, str]
+    raw_headers: list[tuple[bytes, bytes]]
+    match_info: dict
+    method: str
+    query_string: str
+    read: Callable
+
+
 async def benchmark(runs: int):
     """Measure program performance by immediately running the supervisor
     with fake requests.
@@ -167,16 +177,6 @@ async def benchmark(runs: int):
     ref = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")
     settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM
 
-    FakeRequest: Request
-
-    class FakeRequest:  # type: ignore[no-redef]
-        headers: dict[str, str]
-        raw_headers: list[tuple[bytes, bytes]]
-        match_info: dict
-        method: str
-        query_string: str
-        read: Callable
-
     fake_request = FakeRequest()  # type: ignore[operator]
     fake_request.match_info = {"ref": ref, "suffix": "/"}
     fake_request.method = "GET"
@@ -219,7 +219,9 @@ async def fake_read() -> bytes:
         "/cache/keys",
     ):
         fake_request.match_info["suffix"] = path
-        response: Response = await run_code_on_request(vm_hash=ref, path=path, pool=pool, request=fake_request)
+        response: Response = await run_code_on_request(
+            vm_hash=ref, path=path, pool=pool, request=cast(Request, fake_request)
+        )
         assert response.status == 200
 
     # Disable VM timeout to exit benchmark properly
@@ -228,7 +230,9 @@ async def fake_read() -> bytes:
     for _run in range(runs):
         t0 = time.time()
         fake_request.match_info["suffix"] = path
-        response2: Response = await run_code_on_request(vm_hash=ref, path=path, pool=pool, request=fake_request)
+        response2: Response = await run_code_on_request(
+            vm_hash=ref, path=path, pool=pool, request=cast(Request, fake_request)
+        )
         assert response2.status == 200
         bench.append(time.time() - t0)
 

From 1b278e08a2a683f987487ff7d4b7f8240edf8ab7 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:39:01 +0100
Subject: [PATCH 716/990] Fix typing: Invalid type annotations

---
 src/aleph/vm/orchestrator/cli.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index 94b17d305..29b527def 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -7,7 +7,6 @@
 import time
 from pathlib import Path
 from statistics import mean
-from typing import Callable
 from typing import Callable, Optional, cast
 
 from aiohttp.web import Request, Response
@@ -239,8 +238,7 @@ async def fake_read() -> bytes:
     logger.info(f"BENCHMARK: n={len(bench)} avg={mean(bench):03f} min={min(bench):03f} max={max(bench):03f}")
     logger.info(bench)
 
-    event = None
-    result = await run_code_on_event(vm_hash=ref, event=event, pubsub=PubSub(), pool=pool)
+    result = await run_code_on_event(vm_hash=ref, event=None, pubsub=PubSub(), pool=pool)
     print("Event result", result)
 
 
@@ -251,7 +249,7 @@ async def start_instance(item_hash: ItemHash) -> None:
     # The main program uses a singleton pubsub instance in order to watch for updates.
     # We create another instance here since that singleton is not initialized yet.
     # Watching for updates on this instance will therefore not work.
-    pubsub = None
+    pubsub: Optional[PubSub] = None
 
     await start_persistent_vm(item_hash, pubsub, pool)
 

From 19b0916a2d8397fe9b1e465c18c35a2b0898e3a8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:41:32 +0100
Subject: [PATCH 717/990] Fix typing: Replace Dict->dict, List->list

---
 examples/example_fastapi/main.py         |  4 +-
 runtimes/aleph-debian-11-python/init1.py | 50 ++++++++++--------------
 tests/supervisor/test_jwk.py             | 10 ++---
 vm_connector/main.py                     |  8 ++--
 4 files changed, 31 insertions(+), 41 deletions(-)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index 0f9195f7e..516d667a8 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -7,7 +7,7 @@
 from datetime import datetime
 from os import listdir
 from pathlib import Path
-from typing import Dict, Optional
+from typing import Optional
 
 import aiohttp
 from fastapi import FastAPI
@@ -84,7 +84,7 @@ async def check_lifespan():
 
 
 @app.get("/environ")
-async def environ() -> Dict[str, str]:
+async def environ() -> dict[str, str]:
     """List environment variables"""
     return dict(os.environ)
 
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index beb7060ea..f41128a8b 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -18,24 +18,14 @@
 import subprocess
 import sys
 import traceback
+from collections.abc import AsyncIterable
 from contextlib import redirect_stdout
 from dataclasses import dataclass, field
 from enum import Enum
 from io import StringIO
 from os import system
 from shutil import make_archive
-from typing import (
-    Any,
-    AsyncIterable,
-    Dict,
-    List,
-    Literal,
-    NewType,
-    Optional,
-    Tuple,
-    Union,
-    cast,
-)
+from typing import Any, Literal, NewType, Optional, Union, cast
 
 import aiohttp
 import msgpack
@@ -80,15 +70,15 @@ class ConfigurationPayload:
     ipv6: Optional[str] = None
     route: Optional[str] = None
     ipv6_gateway: Optional[str] = None
-    dns_servers: List[str] = field(default_factory=list)
-    volumes: List[Volume] = field(default_factory=list)
-    variables: Optional[Dict[str, str]] = None
-    authorized_keys: Optional[List[str]] = None
+    dns_servers: list[str] = field(default_factory=list)
+    volumes: list[Volume] = field(default_factory=list)
+    variables: Optional[dict[str, str]] = None
+    authorized_keys: Optional[list[str]] = None
 
 
 @dataclass
 class RunCodePayload:
-    scope: Dict
+    scope: dict
 
 
 # Open a socket to receive instructions from the host
@@ -117,7 +107,7 @@ def setup_hostname(hostname: str):
     system(f"hostname {hostname}")
 
 
-def setup_variables(variables: Optional[Dict[str, str]]):
+def setup_variables(variables: Optional[dict[str, str]]):
     if variables is None:
         return
     for key, value in variables.items():
@@ -129,7 +119,7 @@ def setup_network(
     ipv6: Optional[str],
     ipv4_gateway: Optional[str],
     ipv6_gateway: Optional[str],
-    dns_servers: Optional[List[str]] = None,
+    dns_servers: Optional[list[str]] = None,
 ):
     """Setup the system with info from the host."""
     dns_servers = dns_servers or []
@@ -180,13 +170,13 @@ def setup_input_data(input_data: bytes):
             os.system("unzip -q /opt/input.zip -d /data")
 
 
-def setup_authorized_keys(authorized_keys: List[str]) -> None:
+def setup_authorized_keys(authorized_keys: list[str]) -> None:
     path = Path("/root/.ssh/authorized_keys")
     path.parent.mkdir(exist_ok=True)
     path.write_text("\n".join(key for key in authorized_keys))
 
 
-def setup_volumes(volumes: List[Volume]):
+def setup_volumes(volumes: list[Volume]):
     for volume in volumes:
         logger.debug(f"Mounting /dev/{volume.device} on {volume.mount}")
         os.makedirs(volume.mount, exist_ok=True)
@@ -213,7 +203,7 @@ async def receive():
             "type": f"lifespan.{event}",
         }
 
-    async def send(response: Dict):
+    async def send(response: dict):
         response_type = response.get("type")
         if response_type == f"lifespan.{event}.complete":
             lifespan_completion.set()
@@ -260,7 +250,7 @@ async def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> A
         app = getattr(module, app_name)
     elif encoding == Encoding.plain:
         # Execute the code and extract the entrypoint
-        locals: Dict[str, Any] = {}
+        locals: dict[str, Any] = {}
         exec(code, globals(), locals)
         app = locals[entrypoint]
     else:
@@ -313,7 +303,7 @@ async def setup_code(
         raise ValueError("Invalid interface. This should never happen.")
 
 
-async def run_python_code_http(application: ASGIApplication, scope: dict) -> Tuple[Dict, Dict, str, Optional[bytes]]:
+async def run_python_code_http(application: ASGIApplication, scope: dict) -> tuple[dict, dict, str, Optional[bytes]]:
     logger.debug("Running code")
     with StringIO() as buf, redirect_stdout(buf):
         # Execute in the same process, saves ~20ms than a subprocess
@@ -335,14 +325,14 @@ async def send(dico):
         await application(scope, receive, send)
 
         logger.debug("Waiting for headers")
-        headers: Dict
+        headers: dict
         if scope["type"] == "http":
             headers = await send_queue.get()
         else:
             headers = {}
 
         logger.debug("Waiting for body")
-        response_body: Dict = await send_queue.get()
+        response_body: dict = await send_queue.get()
 
         logger.debug("Waiting for buffer")
         output = buf.getvalue()
@@ -394,7 +384,7 @@ def show_loading():
     return headers, body
 
 
-async def run_executable_http(scope: dict) -> Tuple[Dict, Dict, str, Optional[bytes]]:
+async def run_executable_http(scope: dict) -> tuple[dict, dict, str, Optional[bytes]]:
     logger.debug("Calling localhost")
 
     tries = 0
@@ -453,8 +443,8 @@ async def process_instruction(
 
         output: Optional[str] = None
         try:
-            headers: Dict
-            body: Dict
+            headers: dict
+            body: dict
             output_data: Optional[bytes]
 
             if interface == Interface.asgi:
@@ -532,7 +522,7 @@ def setup_system(config: ConfigurationPayload):
     logger.debug("Setup finished")
 
 
-def umount_volumes(volumes: List[Volume]):
+def umount_volumes(volumes: list[Volume]):
     "Umount user related filesystems"
     system("sync")
     for volume in volumes:
diff --git a/tests/supervisor/test_jwk.py b/tests/supervisor/test_jwk.py
index f2fbd2efe..cc3b0ab09 100644
--- a/tests/supervisor/test_jwk.py
+++ b/tests/supervisor/test_jwk.py
@@ -7,7 +7,7 @@
 # Avoid failures linked to settings when initializing the global VmPool object
 os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
 
-from typing import Any, Dict
+from typing import Any
 
 import pytest
 
@@ -23,7 +23,7 @@ def valid_jwk_headers(mocker):
 
 @pytest.mark.skip(reason="TODO: Fix this test")
 @pytest.mark.asyncio
-async def test_valid_signature(valid_jwk_headers: Dict[str, Any], mocker):
+async def test_valid_signature(valid_jwk_headers: dict[str, Any], mocker):
     request = mocker.AsyncMock()
     request.headers = valid_jwk_headers
     await authenticate_jwk(request)
@@ -31,7 +31,7 @@ async def test_valid_signature(valid_jwk_headers: Dict[str, Any], mocker):
 
 @pytest.mark.skip(reason="TODO: Fix this test")
 @pytest.mark.asyncio
-async def test_invalid_signature(valid_jwk_headers: Dict[str, Any], mocker):
+async def test_invalid_signature(valid_jwk_headers: dict[str, Any], mocker):
     valid_jwk_headers["X-SignedOperation"] = (
         '{"time":"2023-07-14T22:14:14.132Z","signature":"96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0ade"}'
     )
@@ -44,7 +44,7 @@ async def test_invalid_signature(valid_jwk_headers: Dict[str, Any], mocker):
 
 @pytest.mark.skip(reason="TODO: Fix this test")
 @pytest.mark.asyncio
-async def test_expired_token(valid_jwk_headers: Dict[str, Any], mocker):
+async def test_expired_token(valid_jwk_headers: dict[str, Any], mocker):
     mocker.patch("aleph.vm.orchestrator.views.authentication.is_token_still_valid", lambda timestamp: False)
     request = mocker.AsyncMock()
     request.headers = valid_jwk_headers
@@ -55,7 +55,7 @@ async def test_expired_token(valid_jwk_headers: Dict[str, Any], mocker):
 
 @pytest.mark.parametrize("missing_header", ["X-SignedPubKey", "X-SignedOperation"])
 @pytest.mark.asyncio
-async def test_missing_headers(valid_jwk_headers: Dict[str, Any], mocker, missing_header: str):
+async def test_missing_headers(valid_jwk_headers: dict[str, Any], mocker, missing_header: str):
     del valid_jwk_headers[missing_header]
 
     request = mocker.AsyncMock()
diff --git a/vm_connector/main.py b/vm_connector/main.py
index 02662d923..22a566f6b 100644
--- a/vm_connector/main.py
+++ b/vm_connector/main.py
@@ -1,6 +1,6 @@
 import json
 import logging
-from typing import Dict, Optional, Union
+from typing import Optional
 
 import aiohttp
 from aleph_client.asynchronous import create_post
@@ -24,7 +24,7 @@ def read_root():
     return {"Server": "Aleph.im VM Connector"}
 
 
-async def get_latest_message_amend(ref: str, sender: str) -> Optional[Dict]:
+async def get_latest_message_amend(ref: str, sender: str) -> Optional[dict]:
     async with aiohttp.ClientSession() as session:
         url = (
             f"{settings.API_SERVER}/api/v0/messages.json?msgType=STORE&sort_order=-1" f"&refs={ref}&addresses={sender}"
@@ -38,7 +38,7 @@ async def get_latest_message_amend(ref: str, sender: str) -> Optional[Dict]:
             return None
 
 
-async def get_message(hash_: str) -> Optional[Dict]:
+async def get_message(hash_: str) -> Optional[dict]:
     async with aiohttp.ClientSession() as session:
         url = f"{settings.API_SERVER}/api/v0/messages.json?hashes={hash_}"
         resp = await session.get(url)
@@ -63,7 +63,7 @@ async def stream_url_chunks(url):
 
 
 @app.get("/download/message/{ref}")
-async def download_message(ref: str) -> Dict:
+async def download_message(ref: str) -> dict:
     """
     Fetch on Aleph and return a VM function message, after checking its validity.
     Used by the VM Supervisor run the code.

From 07c34f941a6e1d3e14f66b10bb595fbb8af5b002 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:42:36 +0100
Subject: [PATCH 718/990] Fix typing: SQLAlchemy annotations were incorrect

---
 src/aleph/vm/orchestrator/metrics.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index 35563f0d2..3b8cdf9f3 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -15,9 +15,12 @@
     delete,
     select,
 )
-from sqlalchemy.engine import Engine
-from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.ext.asyncio import (
+    AsyncEngine,
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
 
 try:
     from sqlalchemy.orm import declarative_base
@@ -26,7 +29,7 @@
 
 from aleph.vm.conf import make_db_url, settings
 
-AsyncSessionMaker: sessionmaker
+AsyncSessionMaker: async_sessionmaker[AsyncSession]
 
 logger = logging.getLogger(__name__)
 
@@ -36,11 +39,11 @@
 def setup_engine():
     global AsyncSessionMaker
     engine = create_async_engine(make_db_url(), echo=True)
-    AsyncSessionMaker = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
+    AsyncSessionMaker = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
     return engine
 
 
-async def create_tables(engine: Engine):
+async def create_tables(engine: AsyncEngine):
     async with engine.begin() as conn:
         await conn.run_sync(Base.metadata.create_all)
 

From c2b540c4ac2fe6f5aaa14444bfb9fde8c065003b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:43:04 +0100
Subject: [PATCH 719/990] Fix typing: Missing return types on methods

---
 src/aleph/vm/hypervisors/firecracker/microvm.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 589e73282..888aeae1b 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -104,18 +104,18 @@ def namespace_path(self) -> str:
         return str(self.jailer_base_directory / firecracker_bin_name / str(self.vm_id))
 
     @property
-    def jailer_path(self):
+    def jailer_path(self) -> str:
         return os.path.join(self.namespace_path, "root")
 
     @property
-    def socket_path(self):
+    def socket_path(self) -> str:
         if self.use_jailer:
             return f"{self.jailer_path}/run/firecracker.socket"
         else:
             return f"/tmp/firecracker-{self.vm_id}.socket"
 
     @property
-    def vsock_path(self):
+    def vsock_path(self) -> str:
         if self.use_jailer:
             return f"{self.jailer_path}{VSOCK_PATH}"
         else:
@@ -140,7 +140,7 @@ def __init__(
         self.runtime_config = None
         self.log_queues: list[asyncio.Queue] = []
 
-    def to_dict(self):
+    def to_dict(self) -> dict:
         return {
             "jailer_path": self.jailer_path,
             "socket_path": self.socket_path,
@@ -148,9 +148,9 @@ def to_dict(self):
             **self.__dict__,
         }
 
-    def prepare_jailer(self):
+    def prepare_jailer(self) -> None:
         if not self.use_jailer:
-            return False
+            return
         system(f"rm -fr {self.jailer_path}")
 
         # system(f"rm -fr {self.jailer_path}/run/")

From 7e11125132a7bb9a2887bfe04a9e176b4b1211f8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:43:39 +0100
Subject: [PATCH 720/990] Fix typing: Strict type checks raised issues

---
 src/aleph/vm/models.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 1a39655da..9d38eea97 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -86,7 +86,7 @@ class VmExecution:
     @property
     def is_running(self) -> bool:
         return (
-            self.times.starting_at and not self.times.stopping_at
+            bool(self.times.starting_at and not self.times.stopping_at)
             if not self.persistent
             else self.systemd_manager.is_service_active(self.controller_service)
         )
@@ -130,7 +130,11 @@ def uses_payment_stream(self) -> bool:
     @property
     def has_resources(self) -> bool:
         assert self.vm, "The VM attribute has to be set before calling has_resources()"
-        return self.vm.resources_path.exists() if self.hypervisor == HypervisorType.firecracker else True
+        if isinstance(self.vm, AlephFirecrackerExecutable):
+            assert self.hypervisor == HypervisorType.firecracker
+            return self.vm.resources_path.exists()
+        else:
+            return True
 
     def __init__(
         self,
@@ -173,7 +177,7 @@ async def prepare(self) -> None:
                 return
 
             self.times.preparing_at = datetime.now(tz=timezone.utc)
-            resources = None
+            resources: Union[AlephProgramResources, AlephInstanceResources, AlephQemuResources]
             if self.is_program:
                 resources = AlephProgramResources(self.message, namespace=self.vm_hash)
             elif self.is_instance:
@@ -181,6 +185,10 @@ async def prepare(self) -> None:
                     resources = AlephInstanceResources(self.message, namespace=self.vm_hash)
                 elif self.hypervisor == HypervisorType.qemu:
                     resources = AlephQemuResources(self.message, namespace=self.vm_hash)
+                else:
+                    raise ValueError(f"Unknown hypervisor type {self.hypervisor}")
+            else:
+                raise ValueError("Unknown executable message type")
 
             if not resources:
                 msg = "Unknown executable message type"

From ceecc106dd8996e12cd634c5a7c4213bb136c1e2 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:44:11 +0100
Subject: [PATCH 721/990] Fix: Properties were initialized with a global object

---
 src/aleph/vm/hypervisors/qemu/qemuvm.py | 4 ++--
 src/aleph/vm/pool.py                    | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index b4f6bb1f5..87ba9724e 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -21,6 +21,7 @@ class QemuVM:
     mem_size_mb: int
     interface_name: str
     qemu_process = None
+    log_queues: list[asyncio.Queue]
 
     def __repr__(self) -> str:
         if self.qemu_process:
@@ -37,6 +38,7 @@ def __init__(self, config: QemuVMConfiguration):
         self.vcpu_count = config.vcpu_count
         self.mem_size_mb = config.mem_size_mb
         self.interface_name = config.interface_name
+        self.log_queues = []
 
     def prepare_start(self):
         pass
@@ -94,8 +96,6 @@ async def start(
         logger.debug(f"started qemu vm {self}, {proc}")
         return proc
 
-    log_queues: list[asyncio.Queue] = []
-
     # TODO : convert when merging with log fixing branch
     async def _process_stderr(self):
         while not self.qemu_process:
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 02889f68e..3e5c5f3ec 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -39,7 +39,7 @@ class VmPool:
 
     counter: int  # Used to provide distinct ids to network interfaces
     executions: dict[ItemHash, VmExecution]
-    message_cache: dict[str, ExecutableMessage] = {}
+    message_cache: dict[str, ExecutableMessage]
     network: Optional[Network]
     snapshot_manager: Optional[SnapshotManager] = None
     systemd_manager: SystemDManager
@@ -48,6 +48,7 @@ class VmPool:
     def __init__(self, loop: asyncio.AbstractEventLoop):
         self.counter = settings.START_ID_INDEX
         self.executions = {}
+        self.message_cache = {}
 
         asyncio.set_event_loop(loop)
         self.creation_lock = asyncio.Lock()

From f31819cdc53c2cb673d80c70875f07b48900775e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:47:37 +0100
Subject: [PATCH 722/990] Fix typing: `...` in class is not allowed

---
 src/aleph/vm/controllers/firecracker/program.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/controllers/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
index 2a7c0082f..2bc060200 100644
--- a/src/aleph/vm/controllers/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -84,7 +84,8 @@ class ProgramVmConfiguration(MsgpackSerializable):
 
 
 @dataclass
-class ConfigurationPayload(MsgpackSerializable): ...
+class ConfigurationPayload(MsgpackSerializable):
+    pass
 
 
 @dataclass

From 811d7913e03a39c56c62f8862bbc0ee57172a245 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:48:07 +0100
Subject: [PATCH 723/990] Fix typing: `continue` inside try/finally block is
 unimplemented in `mypyc`

---
 src/aleph/vm/storage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index cf72440a2..eb652b02b 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -121,7 +121,7 @@ async def download_file(url: str, local_path: Path) -> None:
         ) as error:
             if attempt < (download_attempts - 1):
                 logger.warning(f"Download failed, retrying attempt {attempt + 1}/{download_attempts}...")
-                continue
+                # continue  #  continue inside try/finally block is unimplemented in `mypyc`
             else:
                 raise error
         finally:

From 4381075aa29ee47a3dcaf89d474463398383b4c5 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:48:31 +0100
Subject: [PATCH 724/990] Fix typing: Use of lambda caused typing errors

---
 src/aleph/vm/orchestrator/supervisor.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 20452df90..7c1b8467c 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -82,6 +82,11 @@ async def allow_cors_on_endpoint(request: web.Request):
     )
 
 
+async def http_not_found(request: web.Request):
+    """Return a 404 error for unknown URLs."""
+    return web.HTTPNotFound()
+
+
 app = web.Application(middlewares=[server_version_middleware])
 cors = aiohttp_cors.setup(app)
 
@@ -110,9 +115,9 @@ async def allow_cors_on_endpoint(request: web.Request):
         web.get("/status/check/ipv6", status_check_ipv6),
         web.get("/status/config", status_public_config),
         # Raise an HTTP Error 404 if attempting to access an unknown URL within these paths.
-        web.get("/about/{suffix:.*}", lambda _: web.HTTPNotFound()),
-        web.get("/control/{suffix:.*}", lambda _: web.HTTPNotFound()),
-        web.get("/status/{suffix:.*}", lambda _: web.HTTPNotFound()),
+        web.get("/about/{suffix:.*}", http_not_found),
+        web.get("/control/{suffix:.*}", http_not_found),
+        web.get("/status/{suffix:.*}", http_not_found),
         # /static is used to serve static files
         web.static("/static", Path(__file__).parent / "views/static"),
         # /vm is used to launch VMs on-demand

From 3bfe28def9a2bb3bcbfc702e5b7174509e25313a Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 09:49:17 +0100
Subject: [PATCH 725/990] Fix: Automated code cleanup

Use f-strings, black, ...
---
 examples/example_django/example_django/asgi.py |  2 +-
 packaging/version_from_git.py                  | 12 ++++++------
 vm_connector/main.py                           |  4 +---
 3 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/examples/example_django/example_django/asgi.py b/examples/example_django/example_django/asgi.py
index 1a2020727..feac32c9f 100644
--- a/examples/example_django/example_django/asgi.py
+++ b/examples/example_django/example_django/asgi.py
@@ -17,4 +17,4 @@
 
 os.system("/usr/bin/python3 /opt/code/manage.py migrate")
 
-os.system("/usr/bin/python3 /opt/code/manage.py " "loaddata /opt/code/blog/fixtures/default_articles.json")
+os.system("/usr/bin/python3 /opt/code/manage.py loaddata /opt/code/blog/fixtures/default_articles.json")
diff --git a/packaging/version_from_git.py b/packaging/version_from_git.py
index 327b2e2f4..10252dc2a 100755
--- a/packaging/version_from_git.py
+++ b/packaging/version_from_git.py
@@ -30,7 +30,7 @@
         sys.exit(1)
 
 if not os.path.isfile(target_file_path):
-    print("No such file: '{}'".format(target_file_path))
+    print(f"No such file: '{target_file_path}'")
     sys.exit(2)
 
 
@@ -41,17 +41,17 @@ def get_git_version():
 
 version = get_git_version()
 
-with open(target_file_path, "r") as target_file:
+with open(target_file_path) as target_file:
     target_content = target_file.read()
 
 if format_ == "deb":
-    updated_content = re.sub(r"(Version:)\w*(.*)", "\\1 {}".format(version), target_content)
+    updated_content = re.sub(r"(Version:)\w*(.*)", f"\\1 {version}", target_content)
 elif format_ == "setup.py":
-    updated_content = re.sub(r"(version)\w*=(.*)'", "\\1='{}'".format(version), target_content)
+    updated_content = re.sub(r"(version)\w*=(.*)'", f"\\1='{version}'", target_content)
 elif format_ == "__version__":
-    updated_content = re.sub(r"(__version__)\w*(.*)", "\\1 = '{}'".format(version), target_content)
+    updated_content = re.sub(r"(__version__)\w*(.*)", f"\\1 = '{version}'", target_content)
 else:
-    print("Format must be 'deb', 'setup.py' or '__version__', not '{}'".format(format_))
+    print(f"Format must be 'deb', 'setup.py' or '__version__', not '{format_}'")
 
 if "--inplace" in args:
     with open(target_file_path, "w") as target_file:
diff --git a/vm_connector/main.py b/vm_connector/main.py
index 22a566f6b..86494dd53 100644
--- a/vm_connector/main.py
+++ b/vm_connector/main.py
@@ -26,9 +26,7 @@ def read_root():
 
 async def get_latest_message_amend(ref: str, sender: str) -> Optional[dict]:
     async with aiohttp.ClientSession() as session:
-        url = (
-            f"{settings.API_SERVER}/api/v0/messages.json?msgType=STORE&sort_order=-1" f"&refs={ref}&addresses={sender}"
-        )
+        url = f"{settings.API_SERVER}/api/v0/messages.json?msgType=STORE&sort_order=-1&refs={ref}&addresses={sender}"
         resp = await session.get(url)
         resp.raise_for_status()
         resp_data = await resp.json()

From d38637460c098b82f505faa3138e3591628dfe6c Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 10:05:59 +0100
Subject: [PATCH 726/990] Fix typing: Missing annotation on methods

---
 src/aleph/vm/controllers/firecracker/program.py   | 6 +++---
 src/aleph/vm/controllers/qemu/instance.py         | 2 +-
 src/aleph/vm/orchestrator/views/authentication.py | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/program.py b/src/aleph/vm/controllers/firecracker/program.py
index 2bc060200..6b7436899 100644
--- a/src/aleph/vm/controllers/firecracker/program.py
+++ b/src/aleph/vm/controllers/firecracker/program.py
@@ -284,7 +284,7 @@ def __init__(
             prepare_jailer,
         )
 
-    async def setup(self):
+    async def setup(self) -> None:
         logger.debug(f"Setup started for VM={self.vm_id}")
         await setfacl()
 
@@ -326,7 +326,7 @@ async def wait_for_init(self) -> None:
         """Wait for the custom init inside the virtual machine to signal it is ready."""
         await self.fvm.wait_for_init()
 
-    async def load_configuration(self):
+    async def load_configuration(self) -> None:
         code: bytes | None
         volumes: list[Volume]
 
@@ -342,7 +342,7 @@ async def _setup_configuration(
         input_data: bytes | None,
         interface: Interface,
         volumes: list[Volume],
-    ):
+    ) -> None:
         """Set up the VM configuration. The program mode uses a VSOCK connection to the custom init of the virtual
         machine to send this configuration. Other modes may use Cloud-init, ..."""
         logger.debug("Sending configuration")
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index a8b9cde4e..d0d341c91 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -111,7 +111,7 @@ def make_logs_queue(stdout_identifier, stderr_identifier, skip_past=True) -> tup
     r.add_match(SYSLOG_IDENTIFIER=stderr_identifier)
     queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
 
-    def _ready_for_read():
+    def _ready_for_read() -> None:
         change_type = r.process()  # reset fd status
         if change_type != journal.APPEND:
             return
diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index f1acb2f01..84dd96982 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -68,7 +68,7 @@ def payload_must_be_hex(cls, v: bytes) -> bytes:
         return bytes.fromhex(v.decode())
 
     @root_validator(pre=False, skip_on_failure=True)
-    def check_expiry(cls, values):
+    def check_expiry(cls, values) -> dict[str, bytes]:
         """Check that the token has not expired"""
         payload: bytes = values["payload"]
         content = SignedPubKeyPayload.parse_raw(payload)
@@ -78,7 +78,7 @@ def check_expiry(cls, values):
         return values
 
     @root_validator(pre=False, skip_on_failure=True)
-    def check_signature(cls, values):
+    def check_signature(cls, values) -> dict[str, bytes]:
         """Check that the signature is valid"""
         signature: bytes = values["signature"]
         payload: bytes = values["payload"]

From f1fd3069f2a1445c8d6ceab40d675b028d429ef2 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 10:06:40 +0100
Subject: [PATCH 727/990] Fix typing: Missing path was not considered

---
 src/aleph/vm/controllers/qemu/instance.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index d0d341c91..7d6128e20 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -44,7 +44,10 @@ async def download_all(self) -> None:
 
     async def make_writable_volume(self, parent_image_path, volume: Union[PersistentVolume, RootfsVolume]):
         """Create a new qcow2 image file based on the passed one, that we give to the VM to write onto"""
-        qemu_img_path = shutil.which("qemu-img")
+        qemu_img_path: Optional[str] = shutil.which("qemu-img")
+        if not qemu_img_path:
+            raise VmSetupError("qemu-img not found in PATH")
+
         volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
 
         # detect the image format

From 553d1aae4d48dd2bdd72190d54690be737aea2e7 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 14 Mar 2024 10:07:06 +0100
Subject: [PATCH 728/990] Fix: Domain name must always be specified

Not Optional
---
 src/aleph/vm/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 4c44bd6cc..b568cf6ae 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -118,7 +118,7 @@ class Settings(BaseSettings):
     SUPERVISOR_PORT: int = 4020
 
     # Public domain name
-    DOMAIN_NAME: Optional[str] = Field(
+    DOMAIN_NAME: str = Field(
         default="localhost",
         description="Default public domain name",
     )

From 49898927301892cb841611cd623206fd78f5c5b0 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 26 Mar 2024 13:33:05 +0100
Subject: [PATCH 729/990] Fix: Imports could be cleaned up

---
 src/aleph/vm/controllers/__main__.py        |  8 +-------
 src/aleph/vm/guest_api/__main__.py          |  6 +-----
 src/aleph/vm/orchestrator/cli.py            | 14 ++++----------
 src/aleph/vm/orchestrator/views/__init__.py |  1 -
 4 files changed, 6 insertions(+), 23 deletions(-)

diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index 8f2640012..533b0a7a5 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -6,16 +6,10 @@
 import sys
 from pathlib import Path
 
+from aleph.vm.hypervisors.firecracker.microvm import MicroVM
 from aleph.vm.hypervisors.qemu.qemuvm import QemuVM
 from aleph.vm.network.hostnetwork import Network, make_ipv6_allocator
 
-try:
-    import sentry_sdk
-except ImportError:
-    sentry_sdk = None
-
-from aleph.vm.hypervisors.firecracker.microvm import MicroVM
-
 from .configuration import (
     Configuration,
     HypervisorType,
diff --git a/src/aleph/vm/guest_api/__main__.py b/src/aleph/vm/guest_api/__main__.py
index 72e2cd26e..b1df069fc 100644
--- a/src/aleph/vm/guest_api/__main__.py
+++ b/src/aleph/vm/guest_api/__main__.py
@@ -5,17 +5,13 @@
 
 import aiohttp
 import aioredis
+import sentry_sdk
 from aiohttp import web
 from setproctitle import setproctitle
 
 from aleph.vm.conf import settings
 from aleph.vm.version import get_version_from_apt, get_version_from_git
 
-try:
-    import sentry_sdk
-except ImportError:
-    sentry_sdk = None
-
 logger = logging.getLogger(__name__)
 
 ALEPH_API_SERVER = "https://official.aleph.cloud"
diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index 29b527def..22bd44147 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -9,22 +9,16 @@
 from statistics import mean
 from typing import Callable, Optional, cast
 
-from aiohttp.web import Request, Response
-from sqlalchemy.ext.asyncio import create_async_engine
-
-from aleph.vm.version import get_version_from_apt, get_version_from_git
-
-try:
-    import sentry_sdk
-except ImportError:
-    sentry_sdk = None
-
 import alembic.command
 import alembic.config
+import sentry_sdk
+from aiohttp.web import Request, Response
 from aleph_message.models import ItemHash
+from sqlalchemy.ext.asyncio import create_async_engine
 
 from aleph.vm.conf import ALLOW_DEVELOPER_SSH_KEYS, make_db_url, settings
 from aleph.vm.pool import VmPool
+from aleph.vm.version import get_version_from_apt, get_version_from_git
 
 from . import metrics, supervisor
 from .pubsub import PubSub
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 78cf4d61d..61f4e87a9 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -1,6 +1,5 @@
 import binascii
 import logging
-from collections.abc import Awaitable
 from decimal import Decimal
 from hashlib import sha256
 from json import JSONDecodeError

From 2f5aa795a3b78a6480ca986e7774b0d8131a90f5 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 19 Mar 2024 15:59:37 +0100
Subject: [PATCH 730/990] Fix: Error data was JSON encoded twice

The result sent was always a string containing encoded JSON.

Solution: Use the argument `text` instead of `data` to pass the data as a UTF-8 string.
---
 src/aleph/vm/orchestrator/views/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 61f4e87a9..7c1fd370e 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -350,7 +350,7 @@ async def update_allocations(request: web.Request):
         data = await request.json()
         allocation = Allocation.parse_obj(data)
     except ValidationError as error:
-        return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code)
+        return web.json_response(text=error.json(), status=web.HTTPBadRequest.status_code)
 
     pubsub: PubSub = request.app["pubsub"]
     pool: VmPool = request.app["vm_pool"]

From 0a4f75fbd4f5d06e58a5388608521009e2f0786d Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 19 Mar 2024 16:00:10 +0100
Subject: [PATCH 731/990] Fix: Invalid ItemHash did not raise a ValidationError

Problem: Posting invalid item hashes to the `update_allocations` API failed far after the initial validation, with:

 ```   File "/opt/aleph-vm/aleph_message/models/item_hash.py", line 25, in from_hash
     raise UnknownHashError(f"Could not determine hash type: '{item_hash}'")
aleph_message.exceptions.UnknownHashError: Could not determine hash type: '${{ matrix.check_vm.item_hash }}'
```

Solution: Require ItemHash as part of the `Allocation` schema, in order to fail validation earlier, in the section dedicated to the validation of the uploaded data.
---
 pyproject.toml                         |  1 +
 src/aleph/vm/orchestrator/resources.py |  8 ++++----
 tests/supervisor/test_views.py         | 26 ++++++++++++++++++++++++++
 3 files changed, 31 insertions(+), 4 deletions(-)
 create mode 100644 tests/supervisor/test_views.py

diff --git a/pyproject.toml b/pyproject.toml
index 4619dbd82..f0d862b4b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -88,6 +88,7 @@ dependencies = [
   "pytest-cov==4.1.0",
   "pytest-mock==3.12.0",
   "pytest-asyncio==0.23.5",
+  "pytest-aiohttp==1.0.5",
 ]
 [tool.hatch.envs.testing.scripts]
 test = "pytest {args:tests}"
diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 5be767dac..6c042f056 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -124,10 +124,10 @@ class Allocation(BaseModel):
     It contains the item_hashes of all persistent VMs, instances, on-demand VMs and jobs.
     """
 
-    persistent_vms: set[str] = Field(default_factory=set)
-    instances: set[str] = Field(default_factory=set)
-    on_demand_vms: Optional[set[str]] = None
-    jobs: Optional[set[str]] = None
+    persistent_vms: set[ItemHash] = Field(default_factory=set)
+    instances: set[ItemHash] = Field(default_factory=set)
+    on_demand_vms: Optional[set[ItemHash]] = None
+    jobs: Optional[set[ItemHash]] = None
 
 
 class VMNotification(BaseModel):
diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
new file mode 100644
index 000000000..49a6fa91e
--- /dev/null
+++ b/tests/supervisor/test_views.py
@@ -0,0 +1,26 @@
+import pytest
+from aiohttp import web
+
+from aleph.vm.conf import settings
+from aleph.vm.orchestrator.supervisor import app
+
+
+@pytest.mark.asyncio
+async def test_allocation_fails_on_invalid_item_hash(aiohttp_client):
+    """Test that the allocation endpoint fails when an invalid item_hash is provided."""
+    client = await aiohttp_client(app)
+    settings.ALLOCATION_TOKEN_HASH = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"  # = "test"
+    response: web.Response = await client.post(
+        "/control/allocations", json={"persistent_vms": ["not-an-ItemHash"]}, headers={"X-Auth-Signature": "test"}
+    )
+    assert response.status == 400
+    assert await response.json() == [
+        {
+            "loc": [
+                "persistent_vms",
+                0,
+            ],
+            "msg": "Could not determine hash type: 'not-an-ItemHash'",
+            "type": "value_error.unknownhash",
+        },
+    ]

From da112e685bc2d3663b216b4682fbb3d8f26862f9 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 10 Apr 2024 17:23:09 +0200
Subject: [PATCH 732/990] Doc: Update outdated orchestrator README (#592)

---
 src/aleph/vm/orchestrator/README.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/README.md b/src/aleph/vm/orchestrator/README.md
index 215b5e853..a9d9a3136 100644
--- a/src/aleph/vm/orchestrator/README.md
+++ b/src/aleph/vm/orchestrator/README.md
@@ -28,8 +28,7 @@ Intel Skylake, Intel Cascade Lake, AMD Zen2 and ARM64 Neoverse N1.
 
 ### Operating System
 
-These instructions have been tested on Debian 11 Bullseye, and should work on recent versions
-of Ubuntu as well (22.04+).
+These instructions have been tested on Debian 11 Bullseye, Debian 12 Bookworm and Ubuntu 22.04.
 
 ### Hosting providers
 

From 950d3e8f9b5f94ff447fa522871d014c5b1c7b56 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 6 Mar 2024 09:49:06 +0100
Subject: [PATCH 733/990] Fix: Execution creation was not tested

Solution: Add a test that creates a new VM execution and checks that it starts properly.
---
 pyproject.toml                     |  7 ++++
 tests/supervisor/test_execution.py | 55 ++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 tests/supervisor/test_execution.py

diff --git a/pyproject.toml b/pyproject.toml
index f0d862b4b..99f094a25 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -135,6 +135,13 @@ all = [
 pythonpath = [
   "src"
 ]
+testpaths = [
+  "tests"
+]
+ignore = [
+  "runtimes/aleph-debian-11-python/rootfs/",
+  "runtimes/aleph-debian-12-python/rootfs/",
+]
 
 [tool.black]
 target-version = ["py39"]
diff --git a/tests/supervisor/test_execution.py b/tests/supervisor/test_execution.py
new file mode 100644
index 000000000..902daa195
--- /dev/null
+++ b/tests/supervisor/test_execution.py
@@ -0,0 +1,55 @@
+import asyncio
+
+import pytest
+from aleph_message.models import ItemHash
+
+from aleph.vm.conf import settings
+from aleph.vm.controllers.firecracker import AlephFirecrackerProgram
+from aleph.vm.models import VmExecution
+from aleph.vm.orchestrator import metrics
+from aleph.vm.storage import get_message
+
+
+@pytest.mark.asyncio
+async def test_create_execution():
+    """
+    Create a new VM execution and check that it starts properly.
+    """
+
+    settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM
+    settings.ALLOW_VM_NETWORKING = False
+    settings.USE_JAILER = False
+
+    import logging
+    logging.basicConfig(level=logging.DEBUG)
+
+    # Ensure that the settings are correct and required files present.
+    settings.setup()
+    settings.check()
+
+    # The database is required for the metrics and is currently not optional.
+    engine = metrics.setup_engine()
+    await metrics.create_tables(engine)
+
+    vm_hash = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")
+    message = await get_message(ref=vm_hash)
+
+    execution = VmExecution(
+        vm_hash=vm_hash,
+        message=message.content,
+        original=message.content,
+        snapshot_manager=None,
+        systemd_manager=None,
+        persistent=False,
+    )
+
+    # Downloading the resources required may take some time, limit it to 10 seconds
+    await asyncio.wait_for(execution.prepare(), timeout=30)
+
+    vm = execution.create(vm_id=3, tap_interface=None)
+
+    # Test that the VM is created correctly. It is not started yet.
+    assert isinstance(vm, AlephFirecrackerProgram)
+    assert vm.vm_id == 3
+
+    await asyncio.wait_for(execution.start(), timeout=30)

From cacb83c23072affbaaa4cd3e6cedd48acf1dbaee Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 13 Mar 2024 12:08:28 +0100
Subject: [PATCH 734/990] Fix: Circular imports prevented pytest to run

---
 src/aleph/vm/orchestrator/__init__.py | 29 +--------------------------
 1 file changed, 1 insertion(+), 28 deletions(-)

diff --git a/src/aleph/vm/orchestrator/__init__.py b/src/aleph/vm/orchestrator/__init__.py
index b4c1907a4..d3c1f4225 100644
--- a/src/aleph/vm/orchestrator/__init__.py
+++ b/src/aleph/vm/orchestrator/__init__.py
@@ -1,30 +1,3 @@
 from aleph.vm.version import __version__
 
-from . import (
-    messages,
-    metrics,
-    pubsub,
-    reactor,
-    resources,
-    run,
-    status,
-    supervisor,
-    tasks,
-    views,
-    vm,
-)
-
-__all__ = (
-    "__version__",
-    "messages",
-    "metrics",
-    "pubsub",
-    "reactor",
-    "resources",
-    "run",
-    "status",
-    "supervisor",
-    "tasks",
-    "views",
-    "vm",
-)
+__all__ = ("__version__",)

From f09613451bcfe05ac625b8d0bf6906d85cc8bca9 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 13 Mar 2024 12:19:03 +0100
Subject: [PATCH 735/990] Fix: Pytest failed due to missing files

---
 .github/workflows/test-on-droplets-matrix.yml | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index 61853bf7e..8f372a273 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -37,6 +37,32 @@ jobs:
         run: |
           sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
 
+      - name: Download and build required files for running tests
+        run: |
+          sudo mkdir --parents /opt/firecracker/
+          sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy"
+          
+          rm -fr /tmp/firecracker-release
+          mkdir --parents /tmp/firecracker-release /opt/firecracker
+          curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory /tmp/firecracker-release
+          # Copy binaries:
+          cp /tmp/firecracker-release/release-v*/firecracker-v*[!.debug] /opt/firecracker/firecracker
+          cp /tmp/firecracker-release/release-v*/jailer-v*[!.debug] /opt/firecracker/jailer
+          chmod +x /opt/firecracker/firecracker
+          chmod +x /opt/firecracker/jailer
+          
+          find /opt
+
+      - name: "Build custom runtime"
+        run: |
+          sudo apt update
+          sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86
+          cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
+
+      - name: "Build example volume"
+        run: |
+          cd examples/volumes && bash build_squashfs.sh
+
         # Unit tests create and delete network interfaces, and therefore require to run as root
       - name: Run unit tests
         run: |

From 6d1482d0d3c756f32119aa508d9fc43aff89bd18 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 13 Mar 2024 15:18:58 +0100
Subject: [PATCH 736/990] Doc: Add docstring to settings.setup(), .check()

---
 src/aleph/vm/conf.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index b568cf6ae..18bd23b4c 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -317,6 +317,7 @@ def update(self, **kwargs):
                 raise ValueError(msg)
 
     def check(self):
+        """Check that the settings are valid. Call this method after self.setup()."""
         assert Path("/dev/kvm").exists(), "KVM not found on `/dev/kvm`."
         assert isfile(self.FIRECRACKER_PATH), f"File not found {self.FIRECRACKER_PATH}"
         assert isfile(self.JAILER_PATH), f"File not found {self.JAILER_PATH}"
@@ -363,6 +364,7 @@ def check(self):
             ), "Command `qemu-system-x86_64` not found, run `apt install qemu-system-x86`"
 
     def setup(self):
+        """Setup the environment defined by the settings. Call this method after loading the settings."""
         os.makedirs(self.MESSAGE_CACHE, exist_ok=True)
         os.makedirs(self.CODE_CACHE, exist_ok=True)
         os.makedirs(self.RUNTIME_CACHE, exist_ok=True)

From d66094840f30e099f80f5e29a9eff9060dab31e0 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 13 Mar 2024 15:30:20 +0100
Subject: [PATCH 737/990] Fix: Assertion errors did not display missing path

---
 src/aleph/vm/conf.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 18bd23b4c..29a5317f3 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -341,11 +341,17 @@ def check(self):
             assert self.FAKE_DATA_RUNTIME, "Local runtime .squashfs build not specified"
             assert self.FAKE_DATA_VOLUME, "Local data volume .squashfs not specified"
 
-            assert isdir(self.FAKE_DATA_PROGRAM), "Local fake program directory is missing"
-            assert isfile(self.FAKE_DATA_MESSAGE), "Local fake message is missing"
-            assert isdir(self.FAKE_DATA_DATA), "Local fake data directory is missing"
-            assert isfile(self.FAKE_DATA_RUNTIME), "Local runtime .squashfs build is missing"
-            assert isfile(self.FAKE_DATA_VOLUME), "Local data volume .squashfs is missing"
+            assert isdir(
+                self.FAKE_DATA_PROGRAM
+            ), f"Local fake program directory is missing, no directory '{self.FAKE_DATA_PROGRAM}'"
+            assert isfile(self.FAKE_DATA_MESSAGE), f"Local fake message '{self.FAKE_DATA_MESSAGE}' not found"
+            assert isdir(self.FAKE_DATA_DATA), f"Local fake data directory '{self.FAKE_DATA_DATA}' is missing"
+            assert isfile(
+                self.FAKE_DATA_RUNTIME
+            ), f"Local runtime '{self.FAKE_DATA_RUNTIME}' is missing, did you build it ?"
+            assert isfile(
+                self.FAKE_DATA_VOLUME
+            ), f"Local data volume '{self.FAKE_DATA_VOLUME}' is missing, did you build it ?"
 
         assert is_command_available("setfacl"), "Command `setfacl` not found, run `apt install acl`"
         if self.USE_NDP_PROXY:

From e28fcaa678d6e31cc9f19f65376afd736ed81338 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 26 Mar 2024 17:31:33 +0100
Subject: [PATCH 738/990] Fix: run_guest_api mixed str and Path

Solution: Accept a `pathlib.Path` as argument and convert as string.
---
 src/aleph/vm/guest_api/__main__.py | 7 ++++---
 tests/supervisor/test_execution.py | 6 ++++--
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/guest_api/__main__.py b/src/aleph/vm/guest_api/__main__.py
index b1df069fc..1d35997dd 100644
--- a/src/aleph/vm/guest_api/__main__.py
+++ b/src/aleph/vm/guest_api/__main__.py
@@ -1,6 +1,7 @@
 import json
 import logging
 import re
+from pathlib import Path
 from typing import Optional
 
 import aiohttp
@@ -152,7 +153,7 @@ async def list_keys_from_cache(request: web.Request):
 
 
 def run_guest_api(
-    unix_socket_path,
+    unix_socket_path: Path,
     vm_hash: Optional[str] = None,
     sentry_dsn: Optional[str] = None,
     server_name: Optional[str] = None,
@@ -195,8 +196,8 @@ def run_guest_api(
     app.router.add_route(method="POST", path="/api/v0/p2p/pubsub/pub", handler=repost)
 
     # web.run_app(app=app, port=9000)
-    web.run_app(app=app, path=unix_socket_path)
+    web.run_app(app=app, path=str(unix_socket_path))
 
 
 if __name__ == "__main__":
-    run_guest_api("/tmp/guest-api", vm_hash="vm")
+    run_guest_api(Path("/tmp/guest-api"), vm_hash="vm")
diff --git a/tests/supervisor/test_execution.py b/tests/supervisor/test_execution.py
index 902daa195..dba816bed 100644
--- a/tests/supervisor/test_execution.py
+++ b/tests/supervisor/test_execution.py
@@ -1,4 +1,5 @@
 import asyncio
+import logging
 
 import pytest
 from aleph_message.models import ItemHash
@@ -20,8 +21,8 @@ async def test_create_execution():
     settings.ALLOW_VM_NETWORKING = False
     settings.USE_JAILER = False
 
-    import logging
     logging.basicConfig(level=logging.DEBUG)
+    settings.PRINT_SYSTEM_LOGS = True
 
     # Ensure that the settings are correct and required files present.
     settings.setup()
@@ -52,4 +53,5 @@ async def test_create_execution():
     assert isinstance(vm, AlephFirecrackerProgram)
     assert vm.vm_id == 3
 
-    await asyncio.wait_for(execution.start(), timeout=30)
+    await execution.start()
+    await execution.stop()

From 1b6d4295b6fa2c369b9d7a1c2b0e0b9fb11d4926 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 28 Mar 2024 17:52:31 +0100
Subject: [PATCH 739/990] Fix: Missed documentation, no TESTING.md

Co-authored-by: Olivier Le Thanh Duong 
---
 TESTING.md | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100644 TESTING.md

diff --git a/TESTING.md b/TESTING.md
new file mode 100644
index 000000000..ff3b9d534
--- /dev/null
+++ b/TESTING.md
@@ -0,0 +1,63 @@
+# Testing aleph-vm
+
+This procedure describes how to run tests on a local system.
+
+Tests also run on GitHub Actions via [the following workflow](./.github/workflows/test-on-droplets-matrix.yml).
+
+Since these tests create block devices and manipulate network interfaces, they need to run as root.
+If you are not comfortable with this, run them in a virtual machine.
+
+## 1. Clone this repository
+
+```shell
+git clone https://github.com/aleph-im/aleph-vm.git
+```
+
+## 2. Install [hatch](https://hatch.pypa.io/), the project manager
+
+Since installing tools globally is not recommended, we will install `hatch`
+ in a dedicated virtual environment. Alternatives include using [pipx](https://pipx.pypa.io)
+or your distribution.
+
+```shell
+python3 -m venv /opt/venv
+source /opt/venv/bin/activate
+
+# Inside the venv
+pip install hatch
+```
+
+## 3. Initialize hatch for running the tests
+
+It is required that the testing virtual environment relies on system packages
+for `nftables` instead of the package obtained from `salsa.debian.org` as defined in 
+[pyproject.toml](./pyproject.toml).
+
+Create the testing virtual environment:
+```shell
+hatch env create testing
+```
+
+Obtain the path to the testing virtual environment. 
+```
+hatch run testing:which python
+```
+
+Locate the file named `pyvenv.cfg` in your virtual environment.
+Edit it to use system site packages:
+```
+vim /root/.local/share/hatch/env/virtual/aleph-vm/i5XWCcQ_/testing/pyvenv.cfg
+```
+
+Set `include-system-site-packages` to `true`.
+
+Remove the Python library `nftables` from the `hatch` virtual environment:
+```shell
+hatch run testing:pip uninstall nftables
+```
+
+## 4. Run tests
+
+```shell
+hatch run testing:test
+```

From af5b5fd2ad624ad8cce68ec10046948b108752cb Mon Sep 17 00:00:00 2001
From: Antony JIN <91880456+Antonyjin@users.noreply.github.com>
Date: Mon, 8 Apr 2024 14:58:37 +0200
Subject: [PATCH 740/990] Fix: Could not launch a VM without building it
 locally (#588)

This tests that a VM from the aleph.im network can be downloaded and launched.

Co-authored-by: ajin 
---
 tests/supervisor/test_execution.py | 38 ++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/tests/supervisor/test_execution.py b/tests/supervisor/test_execution.py
index dba816bed..3cecb19ac 100644
--- a/tests/supervisor/test_execution.py
+++ b/tests/supervisor/test_execution.py
@@ -55,3 +55,41 @@ async def test_create_execution():
 
     await execution.start()
     await execution.stop()
+
+
+@pytest.mark.asyncio
+async def test_create_execution_online():
+    """
+    Create a new VM execution without building it locally and check that it starts properly.
+    """
+
+    # Ensure that the settings are correct and required files present.
+    settings.setup()
+    settings.check()
+
+    # The database is required for the metrics and is currently not optional.
+    engine = metrics.setup_engine()
+    await metrics.create_tables(engine)
+
+    vm_hash = ItemHash("3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af")
+    message = await get_message(ref=vm_hash)
+
+    execution = VmExecution(
+        vm_hash=vm_hash,
+        message=message.content,
+        original=message.content,
+        snapshot_manager=None,
+        systemd_manager=None,
+        persistent=False,
+    )
+
+    # Downloading the resources required may take some time, limit it to 10 seconds
+    await asyncio.wait_for(execution.prepare(), timeout=30)
+
+    vm = execution.create(vm_id=3, tap_interface=None)
+    # Test that the VM is created correctly. It is not started yet.
+    assert isinstance(vm, AlephFirecrackerProgram)
+    assert vm.vm_id == 3
+
+    await execution.start()
+    await execution.stop()
\ No newline at end of file

From 53b52b63ee74ccd83e9cfa8cece68999c278775c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 8 Apr 2024 12:54:36 +0200
Subject: [PATCH 741/990] Problem: hatch envs needed manual manipulation for
 testing

Solution: Modify hatch configuration to have the environmnent properly set up using the virtual environment builtin module

https://hatch.pypa.io/1.3/plugins/environment/virtual/
---
 TESTING.md                         | 17 -----------------
 pyproject.toml                     |  2 ++
 tests/supervisor/test_execution.py |  2 +-
 3 files changed, 3 insertions(+), 18 deletions(-)

diff --git a/TESTING.md b/TESTING.md
index ff3b9d534..67f9143b5 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -38,23 +38,6 @@ Create the testing virtual environment:
 hatch env create testing
 ```
 
-Obtain the path to the testing virtual environment. 
-```
-hatch run testing:which python
-```
-
-Locate the file named `pyvenv.cfg` in your virtual environment.
-Edit it to use system site packages:
-```
-vim /root/.local/share/hatch/env/virtual/aleph-vm/i5XWCcQ_/testing/pyvenv.cfg
-```
-
-Set `include-system-site-packages` to `true`.
-
-Remove the Python library `nftables` from the `hatch` virtual environment:
-```shell
-hatch run testing:pip uninstall nftables
-```
 
 ## 4. Run tests
 
diff --git a/pyproject.toml b/pyproject.toml
index 99f094a25..bf6a39e90 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -83,6 +83,8 @@ config = "aleph-vm orchestrator config {args:--help}"
 check = "aleph-vm controller run {args:--help}"
 
 [tool.hatch.envs.testing]
+type = "virtual"
+system-packages = true
 dependencies = [
   "pytest==8.0.1",
   "pytest-cov==4.1.0",
diff --git a/tests/supervisor/test_execution.py b/tests/supervisor/test_execution.py
index 3cecb19ac..551e866e9 100644
--- a/tests/supervisor/test_execution.py
+++ b/tests/supervisor/test_execution.py
@@ -92,4 +92,4 @@ async def test_create_execution_online():
     assert vm.vm_id == 3
 
     await execution.start()
-    await execution.stop()
\ No newline at end of file
+    await execution.stop()

From 64af5a16008e70f3baea98d2434c602bfa01aeec Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 9 Apr 2024 13:55:30 +0200
Subject: [PATCH 742/990] Fix: Missing comments in workflow

---
 .github/workflows/test-on-droplets-matrix.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index 8f372a273..c9563ab82 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -37,7 +37,7 @@ jobs:
         run: |
           sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
 
-      - name: Download and build required files for running tests
+      - name: Download and build required files for running tests. Copied from packaging/Makefile.
         run: |
           sudo mkdir --parents /opt/firecracker/
           sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy"

From f0922f299b8f65ed74fc28c478e0b2961f46acef Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 10 Apr 2024 15:07:00 +0200
Subject: [PATCH 743/990] Problem: Execution test hanging. Python runtime slow

It's a problem surfaced by another

The visible problem was that the new exection test were hanging, inside the runtime, during the import at the line
 from aleph.sdk.chains.remote import RemoteAccount

after some more investigative work, it was pin pointed to an inner import of eth_utils module (specifically eth_utils.network )

Second problem that made the first visible: in the runtime the pre-compiled bytecode, created during runtime creation in create_disk_image.sh was not used, which made the import of module slower. This surfaced the first problem. The cause of that second problem was that the init1.py code which run the user caude was not launched with the same optimization level as the pre-compiled bytecode and thus recompiled everything. (this is specified in the init1.py #! sheebang on the first line)

Solution: Compile the bytecode with the same optimisation level (-o 2 )
as during run

We haven't found out yet why the eth_utils.network import hang when it
is not precompiler. But this fix the test hanging issue
---
 runtimes/aleph-debian-11-python/create_disk_image.sh | 7 +++++--
 runtimes/aleph-debian-12-python/create_disk_image.sh | 6 ++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
index 705b1fe84..bf05fbf48 100755
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-11-python/create_disk_image.sh
@@ -38,8 +38,11 @@ pip3 install 'fastapi~=0.103.1'
 echo "Pip installing aleph-client"
 pip3 install 'aleph-sdk-python==0.7.0'
 
-# Compile all Python bytecode
-python3 -m compileall -f /usr/local/lib/python3.9
+# Compile Python code to bytecode for faster execution
+# -o2 is needed to compile with optimization level 2 which is what we launch init1.py (`python -OO`)
+# otherwise they are not used
+python3 -m compileall -o 2 -f /usr/local/lib/python3.9
+
 
 echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
 echo "PasswordAuthentication no" >> /etc/ssh/sshd_config
diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index 18f2605a3..6a0c2265a 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -39,8 +39,10 @@ mkdir -p /opt/aleph/libs
 pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.9.0' 'fastapi~=0.109.2'
 
 # Compile Python code to bytecode for faster execution
-python3 -m compileall -f /usr/local/lib/python3.11
-python3 -m compileall -f /opt/aleph/libs
+# -o2 is needed to compile with optimization level 2 which is what we launch init1.py (`python -OO`)
+# otherwise they are not used
+python3 -m compileall -o 2 -f /usr/local/lib/python3.11
+python3 -m compileall -o 2 -f /opt/aleph/libs
 
 echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
 echo "PasswordAuthentication no" >> /etc/ssh/sshd_config

From fe2e74f8469b12bea3b7b8438825fdbbe8d8db80 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 10 Apr 2024 15:58:42 +0200
Subject: [PATCH 744/990] Problem cannot import name 'async_sessionmaker' from
 'sqlalchemy.ext.asyncio'

Fix: async_sessionmaker was introduced in sqlachemy 2.0, ensure we have at least this version
otherwhise it was using a older system package
---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index bf6a39e90..cd803673e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,7 +46,7 @@ dependencies = [
   "systemd-python==235",
   "systemd-python==235",
   "superfluid~=0.2.1",
-  "sqlalchemy[asyncio]",
+  "sqlalchemy[asyncio]>=2.0",
   "aiosqlite==0.19.0",
   "alembic==1.13.1",
   "aiohttp_cors~=0.7.0",

From 0f77070b141034b49a6202e94c67af0321900b88 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 9 Apr 2024 14:48:54 +0200
Subject: [PATCH 745/990] Cleanup: Frozen requirements were not maintained

A frozen copy of the requirements.txt extracted from different systems
was present in the repository but not used nor maintained.
---
 packaging/requirements-debian-11.txt    | 32 ----------------------
 packaging/requirements-ubuntu-20.04.txt | 32 ----------------------
 packaging/requirements-ubuntu-22.04.txt | 35 -------------------------
 3 files changed, 99 deletions(-)
 delete mode 100644 packaging/requirements-debian-11.txt
 delete mode 100644 packaging/requirements-ubuntu-20.04.txt
 delete mode 100644 packaging/requirements-ubuntu-22.04.txt

diff --git a/packaging/requirements-debian-11.txt b/packaging/requirements-debian-11.txt
deleted file mode 100644
index d708640db..000000000
--- a/packaging/requirements-debian-11.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-aiodns==2.0.0
-aiohttp==3.7.4
-aioredis==1.3.1
-alembic==1.4.3
-async-timeout==3.0.1
-attrs==20.3.0
-chardet==4.0.0
-hiredis==1.0.1
-idna==2.10
-importlib-metadata==1.6.0
-jsonschema==3.2.0
-Mako==1.1.3
-MarkupSafe==1.1.1
-more-itertools==4.2.0
-msgpack==1.0.0
-multidict==5.1.0
-git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
-packaging==20.9
-psutil==5.8.0
-py-cpuinfo==5.0.0
-pycares==3.1.1
-pyparsing==2.4.7
-pyrsistent==0.15.5
-python-dateutil==2.8.1
-python-editor==1.0.3
-redis==3.5.3
-setproctitle==1.2.1
-six==1.16.0
-SQLAlchemy==1.3.22
-typing-extensions==3.7.4.3
-yarl==1.6.3
-zipp==1.0.0
diff --git a/packaging/requirements-ubuntu-20.04.txt b/packaging/requirements-ubuntu-20.04.txt
deleted file mode 100644
index 1175ab784..000000000
--- a/packaging/requirements-ubuntu-20.04.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-aiodns==2.0.0
-aiohttp==3.6.2
-aioredis==1.3.1
-alembic==1.1.0
-async-timeout==3.0.1
-attrs==19.3.0
-chardet==3.0.4
-dbus-python==1.2.16
-hiredis==1.0.0
-idna==2.8
-importlib-metadata==1.5.0
-jsonschema==3.2.0
-Mako==1.1.0
-MarkupSafe==1.1.0
-more-itertools==4.2.0
-msgpack==0.6.2
-multidict==4.7.3
-git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
-packaging==20.3
-psutil==5.5.1
-py-cpuinfo==5.0.0
-pycares==3.1.1
-PyGObject==3.36.0
-pyparsing==2.4.6
-pyrsistent==0.15.5
-python-dateutil==2.7.3
-redis==3.3.11
-setproctitle==1.1.10
-six==1.14.0
-SQLAlchemy==1.3.12
-yarl==1.4.2
-zipp==1.0.0
diff --git a/packaging/requirements-ubuntu-22.04.txt b/packaging/requirements-ubuntu-22.04.txt
deleted file mode 100644
index 580dc68ef..000000000
--- a/packaging/requirements-ubuntu-22.04.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-aiodns==3.0.0
-aiohttp==3.8.1
-aioredis==1.3.1
-aiosignal==1.2.0
-alembic==1.7.6
-async-timeout==4.0.1
-attrs==21.2.0
-charset-normalizer==2.0.6
-dbus-python==1.2.18
-frozenlist==1.2.0
-greenlet==1.1.2
-hiredis==1.0.1
-idna==3.3
-importlib-metadata==4.6.4
-jsonschema==3.2.0
-Mako==1.1.3
-MarkupSafe==2.0.1
-more-itertools==8.10.0
-msgpack==1.0.3
-multidict==5.1.0
-git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
-packaging==21.3
-psutil==5.9.0
-py-cpuinfo==5.0.0
-pycares==4.1.2
-PyGObject==3.42.1
-pyparsing==2.4.7
-pyrsistent==0.18.1
-redis==3.5.3
-setproctitle==1.2.2
-six==1.16.0
-SQLAlchemy==1.4.31
-typing-extensions==3.10.0.2
-yarl==1.7.2
-zipp==1.0.0

From 57695a1ff26b8f35e88ca4ef935df7071dba659b Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Mon, 15 Apr 2024 15:44:20 +0200
Subject: [PATCH 746/990] Fix: Prevent diagnostic VM to fail if the ipv6 or
 ipv4 raises a Timeout.

---
 examples/example_fastapi/main.py | 39 ++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index 516d667a8..b41e4a4d9 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -137,10 +137,15 @@ async def ip_address():
 @app.get("/ip/4")
 async def connect_ipv4():
     """Connect to the Quad9 VPN provider using their IPv4 address."""
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.settimeout(5)
-    sock.connect(("9.9.9.9", 53))
-    return {"result": True}
+    ipv4_host = "9.9.9.9"
+    try:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.settimeout(5)
+        sock.connect((ipv4_host, 53))
+        return {"result": True}
+    except socket.timeout:
+        logger.warning(f"Socket connection for host {ipv4_host} failed")
+        return {"result": False}
 
 
 @app.get("/ip/6")
@@ -148,23 +153,33 @@ async def connect_ipv6():
     """Connect to the Quad9 VPN provider using their IPv6 address.
     The webserver on that address returns a 404 error, so we accept that response code.
     """
+    ipv6_host = "https://[2620:fe::fe]"
     timeout = aiohttp.ClientTimeout(total=5)
     async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
-        async with session.get("https://[2620:fe::fe]") as resp:
-            # We expect this endpoint to return a 404 error
-            if resp.status != 404:
-                resp.raise_for_status()
-            return {"result": True, "headers": resp.headers}
+        try:
+            async with session.get(ipv6_host) as resp:
+                # We expect this endpoint to return a 404 error
+                if resp.status != 404:
+                    resp.raise_for_status()
+                return {"result": True, "headers": resp.headers}
+        except aiohttp.ClientTimeout:
+            logger.warning(f"Session connection for host {ipv6_host} failed")
+            return {"result": False, "headers": resp.headers}
 
 
 @app.get("/internet")
 async def read_internet():
     """Connect the aleph.im official website to check Internet connectivity."""
+    internet_host = "https://aleph.im/"
     timeout = aiohttp.ClientTimeout(total=5)
     async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
-        async with session.get("https://aleph.im/") as resp:
-            resp.raise_for_status()
-            return {"result": resp.status, "headers": resp.headers}
+        try:
+            async with session.get(internet_host) as resp:
+                resp.raise_for_status()
+                return {"result": resp.status, "headers": resp.headers}
+        except aiohttp.ClientTimeout:
+            logger.warning(f"Session connection for host {internet_host} failed")
+            return {"result": False, "headers": resp.headers}
 
 
 @app.get("/post_a_message")

From 8bbd65b9865aed67a233686913d55789b5b5980b Mon Sep 17 00:00:00 2001
From: "Andres D. Molins" 
Date: Mon, 15 Apr 2024 15:01:49 +0200
Subject: [PATCH 747/990] Fix: Solved CORS issues on PAYG creation.

---
 src/aleph/vm/orchestrator/supervisor.py | 92 +++++++++++++++----------
 1 file changed, 55 insertions(+), 37 deletions(-)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 7c1b8467c..9b2c3c1c1 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -13,8 +13,8 @@
 from secrets import token_urlsafe
 from typing import Callable
 
-import aiohttp_cors
 from aiohttp import web
+from aiohttp_cors import ResourceOptions, setup
 
 from aleph.vm.conf import settings
 from aleph.vm.pool import VmPool
@@ -88,44 +88,62 @@ async def http_not_found(request: web.Request):
 
 
 app = web.Application(middlewares=[server_version_middleware])
-cors = aiohttp_cors.setup(app)
-
-app.add_routes(
-    [
-        # /about APIs return information about the VM Orchestrator
-        web.get("/about/login", about_login),
-        web.get("/about/executions/list", list_executions),
-        web.get("/about/executions/details", about_executions),
-        web.get("/about/executions/records", about_execution_records),
-        web.get("/about/usage/system", about_system_usage),
-        web.get("/about/config", about_config),
-        # /control APIs are used to control the VMs and access their logs
-        web.post("/control/allocations", update_allocations),
-        web.post("/control/allocation/notify", notify_allocation),
-        web.get("/control/machine/{ref}/logs", stream_logs),
-        web.post("/control/machine/{ref}/expire", operate_expire),
-        web.post("/control/machine/{ref}/stop", operate_stop),
-        web.post("/control/machine/{ref}/erase", operate_erase),
-        web.post("/control/machine/{ref}/reboot", operate_reboot),
-        # /status APIs are used to check that the VM Orchestrator is running properly
-        web.get("/status/check/fastapi", status_check_fastapi),
-        web.get("/status/check/fastapi/legacy", status_check_fastapi_legacy),
-        web.get("/status/check/host", status_check_host),
-        web.get("/status/check/version", status_check_version),
-        web.get("/status/check/ipv6", status_check_ipv6),
-        web.get("/status/config", status_public_config),
-        # Raise an HTTP Error 404 if attempting to access an unknown URL within these paths.
-        web.get("/about/{suffix:.*}", http_not_found),
-        web.get("/control/{suffix:.*}", http_not_found),
-        web.get("/status/{suffix:.*}", http_not_found),
-        # /static is used to serve static files
-        web.static("/static", Path(__file__).parent / "views/static"),
-        # /vm is used to launch VMs on-demand
-        web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
-        web.route("*", "/{suffix:.*}", run_code_from_hostname),
-    ]
+cors = setup(
+    app,
+    defaults={
+        "*": ResourceOptions(
+            allow_credentials=True,
+            expose_headers="*",
+            allow_headers="*",
+        )
+    },
 )
 
+# Routes that need CORS enabled
+cors_routes = [
+    # /about APIs return information about the VM Orchestrator
+    web.get("/about/login", about_login),
+    web.get("/about/executions/list", list_executions),
+    web.get("/about/executions/details", about_executions),
+    web.get("/about/executions/records", about_execution_records),
+    web.get("/about/usage/system", about_system_usage),
+    web.get("/about/config", about_config),
+    # /control APIs are used to control the VMs and access their logs
+    web.post("/control/allocation/notify", notify_allocation),
+    web.get("/control/machine/{ref}/logs", stream_logs),
+    web.post("/control/machine/{ref}/expire", operate_expire),
+    web.post("/control/machine/{ref}/stop", operate_stop),
+    web.post("/control/machine/{ref}/erase", operate_erase),
+    web.post("/control/machine/{ref}/reboot", operate_reboot),
+    # /status APIs are used to check that the VM Orchestrator is running properly
+    web.get("/status/check/fastapi", status_check_fastapi),
+    web.get("/status/check/fastapi/legacy", status_check_fastapi_legacy),
+    web.get("/status/check/host", status_check_host),
+    web.get("/status/check/version", status_check_version),
+    web.get("/status/check/ipv6", status_check_ipv6),
+    web.get("/status/config", status_public_config),
+]
+routes = app.add_routes(cors_routes)
+for route in routes:
+    cors.add(route)
+
+
+# Routes that don't need CORS enabled
+other_routes = [
+    # /control APIs are used to control the VMs and access their logs
+    web.post("/control/allocations", update_allocations),
+    # Raise an HTTP Error 404 if attempting to access an unknown URL within these paths.
+    web.get("/about/{suffix:.*}", http_not_found),
+    web.get("/control/{suffix:.*}", http_not_found),
+    web.get("/status/{suffix:.*}", http_not_found),
+    # /static is used to serve static files
+    web.static("/static", Path(__file__).parent / "views/static"),
+    # /vm is used to launch VMs on-demand
+    web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
+    web.route("*", "/{suffix:.*}", run_code_from_hostname),
+]
+app.add_routes(other_routes)
+
 
 async def stop_all_vms(app: web.Application):
     pool: VmPool = app["vm_pool"]

From c74ed5aca8eb7a53d56d2072ed27e68aa083dac5 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 22 Apr 2024 11:49:44 +0200
Subject: [PATCH 748/990] Fix: Internet diagnostic due to single endpoint

Internet connectivity checks by the diagnostic VM relied on a single URL. If that endpoint was down, the internet connectivity of the system was assumed to be down.

Solution: Check connectivity to multiple endpoints in parallel.
---
 examples/example_fastapi/main.py | 49 +++++++++++++++++++++++++-------
 1 file changed, 39 insertions(+), 10 deletions(-)

diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index b41e4a4d9..ebe1a8bd0 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -1,3 +1,4 @@
+import asyncio
 import json
 import logging
 import os
@@ -7,14 +8,14 @@
 from datetime import datetime
 from os import listdir
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 import aiohttp
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import PlainTextResponse
 from pip._internal.operations.freeze import freeze
-from pydantic import BaseModel
+from pydantic import BaseModel, HttpUrl
 from starlette.responses import JSONResponse
 
 from aleph.sdk.chains.remote import RemoteAccount
@@ -167,19 +168,47 @@ async def connect_ipv6():
             return {"result": False, "headers": resp.headers}
 
 
-@app.get("/internet")
-async def read_internet():
-    """Connect the aleph.im official website to check Internet connectivity."""
-    internet_host = "https://aleph.im/"
-    timeout = aiohttp.ClientTimeout(total=5)
+async def check_url(internet_host: HttpUrl, timeout_seconds: int = 5):
+    """Check the connectivity of a single URL."""
+    timeout = aiohttp.ClientTimeout(total=timeout_seconds)
     async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(), timeout=timeout) as session:
         try:
             async with session.get(internet_host) as resp:
                 resp.raise_for_status()
-                return {"result": resp.status, "headers": resp.headers}
-        except aiohttp.ClientTimeout:
+                return {"result": resp.status, "headers": resp.headers, "url": internet_host}
+        except (aiohttp.ClientConnectionError, TimeoutError):
             logger.warning(f"Session connection for host {internet_host} failed")
-            return {"result": False, "headers": resp.headers}
+            return {"result": False, "url": internet_host}
+
+
+@app.get("/internet")
+async def read_internet():
+    """Check Internet connectivity of the system, requiring IP connectivity, domain resolution and HTTPS/TLS."""
+    internet_hosts: List[HttpUrl] = [
+        HttpUrl(url="https://aleph.im/", scheme="https"),
+        HttpUrl(url="https://ethereum.org", scheme="https"),
+        HttpUrl(url="https://ipfs.io/", scheme="https"),
+    ]
+    timeout_seconds = 5
+
+    # Create a list of tasks to check the URLs in parallel
+    tasks: set[asyncio.Task] = set(asyncio.create_task(check_url(host, timeout_seconds)) for host in internet_hosts)
+
+    # While no tasks have completed, keep waiting for the next one to finish
+    while tasks:
+        done, tasks = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+        result = done.pop().result()
+
+        if result["result"]:
+            # The task was successful, cancel the remaining tasks and return the result
+            for task in tasks:
+                task.cancel()
+            return result
+        else:
+            continue
+
+    # No URL was reachable
+    return {"result": False}
 
 
 @app.get("/post_a_message")

From b7d92027311c816ba98477581191fe08a17630c1 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 16 Apr 2024 12:05:47 +0200
Subject: [PATCH 749/990] Problem: Makefile for publishing example were not
 working

'aleph program' now need an 'update' argument.
Solution: Update makefile and documentation
---
 examples/example_http_js/Makefile   | 2 +-
 examples/example_http_rust/Makefile | 2 +-
 tutorials/REQUIREMENTS.md           | 2 +-
 tutorials/SERVER.md                 | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/example_http_js/Makefile b/examples/example_http_js/Makefile
index 3b2ac89e8..6c43a3f06 100644
--- a/examples/example_http_js/Makefile
+++ b/examples/example_http_js/Makefile
@@ -16,4 +16,4 @@ docker-publish:
 
 publish:
 	chmod +x ./src/run.sh
-	aleph program ./src "run.sh"
+	aleph program upload ./src "run.sh"
diff --git a/examples/example_http_rust/Makefile b/examples/example_http_rust/Makefile
index 0f82bdd02..dbf618cd9 100644
--- a/examples/example_http_rust/Makefile
+++ b/examples/example_http_rust/Makefile
@@ -15,4 +15,4 @@ publish:
 	cargo build --release
 	mkdir -p ./dist
 	cp target/release/example_http_rust ./dist/
-	aleph program ./dist example_http_rust
+	aleph program upload ./dist example_http_rust
diff --git a/tutorials/REQUIREMENTS.md b/tutorials/REQUIREMENTS.md
index 905ddfbbd..6a87fe359 100644
--- a/tutorials/REQUIREMENTS.md
+++ b/tutorials/REQUIREMENTS.md
@@ -89,7 +89,7 @@ aleph pin QmWWX6BaaRkRSr2iNdwH5e29ACPg2nCHHXTRTfuBmVm3Ga
 ## 3. Create your program
 
 ```shell
-aleph program ./my-program main:app
+aleph program upload ./my-program main:app
 ```
 
 Press Enter at the following prompt to use the default runtime:
diff --git a/tutorials/SERVER.md b/tutorials/SERVER.md
index c34548364..2dcff54e5 100644
--- a/tutorials/SERVER.md
+++ b/tutorials/SERVER.md
@@ -88,9 +88,9 @@ cargo build --release
 Publish it on Aleph using the same procedure as with the Python example, except the entrypoint refers to the name of the binary to execute. 
 
 ```shell
-aleph program ./target/release/example_http_rust example_http_rust
+aleph program upload ./target/release/example_http_rust example_http_rust
 ```
 
 If your program takes some arguments, pass them in the entrypoint by using quotes: `"example_http_rust --help`.
 
-ℹ️ If you get the error `Invalid zip archive`, you are probably missing the Squashfs user tool `mksquashfs`. In that case, first create the squashfs archive and then upload it using `aleph program ./target/release/example_http_rust.squashfs example_http_rust`
+ℹ️ If you get the error `Invalid zip archive`, you are probably missing the Squashfs user tool `mksquashfs`. In that case, first create the squashfs archive and then upload it using `aleph program upload ./target/release/example_http_rust.squashfs example_http_rust`

From 54680ba7244c931cc601d0e2a962ca948c6e04d5 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 25 Apr 2024 15:26:37 +0200
Subject: [PATCH 750/990] Problem: could not start Instances from command line
 (#597)

Problem: could not start Instances from command line

Problem  happened when launching with --run-fake-instance
Solution: Adapt to new VMPool API that take a loop
Also fix benchmarks function
---
 src/aleph/vm/orchestrator/cli.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index 22bd44147..65b290ba2 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -17,6 +17,7 @@
 from sqlalchemy.ext.asyncio import create_async_engine
 
 from aleph.vm.conf import ALLOW_DEVELOPER_SSH_KEYS, make_db_url, settings
+from aleph.vm.models import VmExecution
 from aleph.vm.pool import VmPool
 from aleph.vm.version import get_version_from_apt, get_version_from_git
 
@@ -187,7 +188,8 @@ async def fake_read() -> bytes:
 
     bench: list[float] = []
 
-    pool = VmPool()
+    loop = asyncio.get_event_loop()
+    pool = VmPool(loop)
     pool.setup()
 
     # Does not make sense in benchmarks
@@ -236,25 +238,24 @@ async def fake_read() -> bytes:
     print("Event result", result)
 
 
-async def start_instance(item_hash: ItemHash) -> None:
+async def start_instance(item_hash: ItemHash, pubsub: Optional[PubSub], pool) -> VmExecution:
     """Run an instance from an InstanceMessage."""
-    pool = VmPool()
+    return await start_persistent_vm(item_hash, pubsub, pool)
 
+
+async def run_instances(instances: list[ItemHash]) -> None:
+    """Run instances from a list of message identifiers."""
+    logger.info(f"Instances to run: {instances}")
+    loop = asyncio.get_event_loop()
+    pool = VmPool(loop)
     # The main program uses a singleton pubsub instance in order to watch for updates.
     # We create another instance here since that singleton is not initialized yet.
     # Watching for updates on this instance will therefore not work.
     pubsub: Optional[PubSub] = None
 
-    await start_persistent_vm(item_hash, pubsub, pool)
-
-
-async def run_instances(instances: list[ItemHash]) -> None:
-    """Run instances from a list of message identifiers."""
-    logger.info(f"Instances to run: {instances}")
+    await asyncio.gather(*[start_instance(instance_id, pubsub, pool) for instance_id in instances])
 
-    await asyncio.gather(*[start_instance(item_hash=instance_id) for instance_id in instances])
     await asyncio.Event().wait()  # wait forever
-    # TODO : should we really wait forever?
 
 
 @contextlib.contextmanager

From ab79b779c22e9652611f9aef6f74a28574055f35 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Fri, 26 Apr 2024 10:14:18 +0200
Subject: [PATCH 751/990] Solve last CORS issues about duplicated headers
 (#604)

Fix: Solve last CORS errors raised cause by duplication of headers returned.
---
 src/aleph/vm/orchestrator/resources.py         |  4 +++-
 src/aleph/vm/orchestrator/supervisor.py        | 13 -------------
 src/aleph/vm/orchestrator/views/__init__.py    | 18 +++++-------------
 .../vm/orchestrator/views/authentication.py    |  2 --
 4 files changed, 8 insertions(+), 29 deletions(-)

diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 6c042f056..a40c6ff13 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -11,6 +11,7 @@
 from pydantic import BaseModel, Field
 
 from aleph.vm.conf import settings
+from aleph.vm.utils import cors_allow_all
 
 
 class Period(BaseModel):
@@ -92,6 +93,7 @@ def get_machine_properties() -> MachineProperties:
     )
 
 
+@cors_allow_all
 async def about_system_usage(_: web.Request):
     """Public endpoint to expose information about the system usage."""
     period_start = datetime.now(timezone.utc).replace(second=0, microsecond=0)
@@ -116,7 +118,7 @@ async def about_system_usage(_: web.Request):
         ),
         properties=get_machine_properties(),
     )
-    return web.json_response(text=usage.json(exclude_none=True), headers={"Access-Control-Allow-Origin:": "*"})
+    return web.json_response(text=usage.json(exclude_none=True))
 
 
 class Allocation(BaseModel):
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 9b2c3c1c1..4846104ae 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -69,19 +69,6 @@ async def server_version_middleware(
     return resp
 
 
-async def allow_cors_on_endpoint(request: web.Request):
-    """Allow CORS on endpoints that VM owners use to control their machine."""
-    return web.Response(
-        status=200,
-        headers={
-            "Access-Control-Allow-Headers": "*",
-            "Access-Control-Allow-Methods": "*",
-            "Access-Control-Allow-Origin": "*",
-            "Allow": "POST",
-        },
-    )
-
-
 async def http_not_found(request: web.Request):
     """Return a 404 error for unknown URLs."""
     return web.HTTPNotFound()
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 7c1fd370e..994476cba 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -214,13 +214,9 @@ async def status_check_fastapi(request: web.Request, vm_id: Optional[ItemHash] =
                     # "ipv6": await status.check_ipv6(session),
                 }
 
-            return web.json_response(
-                result, status=200 if all(result.values()) else 503, headers={"Access-Control-Allow-Origin": "*"}
-            )
+            return web.json_response(result, status=200 if all(result.values()) else 503)
     except aiohttp.ServerDisconnectedError as error:
-        return web.json_response(
-            {"error": f"Server disconnected: {error}"}, status=503, headers={"Access-Control-Allow-Origin": "*"}
-        )
+        return web.json_response({"error": f"Server disconnected: {error}"}, status=503)
 
 
 @cors_allow_all
@@ -246,7 +242,7 @@ async def status_check_host(request: web.Request):
         },
     }
     result_status = 200 if all(result["ipv4"].values()) and all(result["ipv6"].values()) else 503
-    return web.json_response(result, status=result_status, headers={"Access-Control-Allow-Origin": "*"})
+    return web.json_response(result, status=result_status)
 
 
 @cors_allow_all
@@ -260,7 +256,7 @@ async def status_check_ipv6(request: web.Request):
             vm_ipv6 = False
 
     result = {"host": await check_host_egress_ipv6(), "vm": vm_ipv6}
-    return web.json_response(result, headers={"Access-Control-Allow-Origin": "*"})
+    return web.json_response(result)
 
 
 @cors_allow_all
@@ -283,7 +279,6 @@ async def status_check_version(request: web.Request):
         return web.Response(
             status=200,
             text=f"Up-to-date: version {current} >= {reference}",
-            headers={"Access-Control-Allow-Origin": "*"},
         )
     else:
         return web.HTTPForbidden(text=f"Outdated: version {current} < {reference}")
@@ -327,7 +322,6 @@ async def status_public_config(request: web.Request):
             },
         },
         dumps=dumps_for_json,
-        headers={"Access-Control-Allow-Origin": "*"},
     )
 
 
@@ -436,9 +430,7 @@ async def notify_allocation(request: web.Request):
     except JSONDecodeError:
         return web.HTTPBadRequest(reason="Body is not valid JSON")
     except ValidationError as error:
-        return web.json_response(
-            data=error.json(), status=web.HTTPBadRequest.status_code, headers={"Access-Control-Allow-Origin": "*"}
-        )
+        return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code)
 
     pubsub: PubSub = request.app["pubsub"]
     pool: VmPool = request.app["vm_pool"]
diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index 84dd96982..d38587015 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -227,8 +227,6 @@ async def wrapper(request):
             return web.json_response(data={"error": e.reason}, status=e.status)
 
         response = await handler(request, authenticated_sender)
-        # Allow browser clients to access the body of the response
-        response.headers.update({"Access-Control-Allow-Origin": request.headers.get("Origin", "")})
         return response
 
     return wrapper

From fe9235ac658915eea20d5371ae45cedabe1f7b17 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 26 Apr 2024 13:00:35 +0200
Subject: [PATCH 752/990] Fix: Diagnostic API was not updated

We published multiple changes to the diagnostic VM recently but none of these was released.

This provides a new diagnostic VM, based on a new runtime [1], with fixes:

- Reading messages with the newer SDK
- Better handling of IPv6 detection errors
- Two different tests for signing messages (local and remote)
- aleph-message version was not specified
- fetching a single message was not tested
---
 .github/workflows/test-on-droplets-matrix.yml |   5 +-
 examples/example_fastapi/README.md            |   6 +
 examples/example_fastapi/main.py              | 187 ++++++++++++++----
 .../create_disk_image.sh                      |   2 +-
 src/aleph/vm/conf.py                          |   2 +-
 src/aleph/vm/orchestrator/run.py              |   1 +
 src/aleph/vm/orchestrator/status.py           |  58 +++++-
 src/aleph/vm/orchestrator/views/__init__.py   |   4 +
 8 files changed, 224 insertions(+), 41 deletions(-)
 create mode 100644 examples/example_fastapi/README.md

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index c9563ab82..c67c1688f 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -134,8 +134,11 @@ jobs:
           - alias: "runtime-6770"  # Old runtime, using Debian 11
             item_hash: "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
             query_params: "?retro-compatibility=true"
-          - alias: "runtime-3fc0"  # New runtime, using Debian 12
+          - alias: "runtime-3fc0"  # Newer runtime, using Debian 12 but now old SDK
             item_hash: "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af"
+            query_params: "?retro-compatibility=true"
+          - alias: "runtime-63fa"  # Latest runtime, using Debian 12 and SDK 0.9.0
+            item_hash: "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace"
             query_params: ""
 
     steps:
diff --git a/examples/example_fastapi/README.md b/examples/example_fastapi/README.md
new file mode 100644
index 000000000..231ce255b
--- /dev/null
+++ b/examples/example_fastapi/README.md
@@ -0,0 +1,6 @@
+Publish using:
+
+```shell
+ aleph program upload ../aleph-vm/examples/example_fastapi main:app \
+  --persistent-volume "persistence=host,size_mib=1,mount=/var/lib/example,name=increment-storage,comment=Persistence"
+```
diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index ebe1a8bd0..81055c723 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -5,12 +5,19 @@
 import socket
 import subprocess
 import sys
-from datetime import datetime
+from datetime import datetime, timezone
 from os import listdir
 from pathlib import Path
-from typing import List, Optional
+from typing import Any, Optional
 
 import aiohttp
+from aleph_message.models import (
+    MessagesResponse,
+    PostMessage,
+    ProgramMessage,
+    StoreMessage,
+)
+from aleph_message.status import MessageStatus
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import PlainTextResponse
@@ -18,8 +25,10 @@
 from pydantic import BaseModel, HttpUrl
 from starlette.responses import JSONResponse
 
+from aleph.sdk.chains.ethereum import get_fallback_account
 from aleph.sdk.chains.remote import RemoteAccount
-from aleph.sdk.client import AlephClient, AuthenticatedAlephClient
+from aleph.sdk.client import AlephHttpClient, AuthenticatedAlephHttpClient
+from aleph.sdk.query.filters import MessageFilter
 from aleph.sdk.types import StorageEnum
 from aleph.sdk.vm.app import AlephApp
 from aleph.sdk.vm.cache import VmCache
@@ -42,13 +51,13 @@
 
 
 @app.on_event("startup")
-async def startup_event():
+async def startup_event() -> None:
     global startup_lifespan_executed
     startup_lifespan_executed = True
 
 
 @app.get("/")
-async def index():
+async def index() -> dict[str, Any]:
     if os.path.exists("/opt/venv"):
         opt_venv = list(listdir("/opt/venv"))
     else:
@@ -56,16 +65,33 @@ async def index():
     return {
         "Example": "example_fastapi",
         "endpoints": [
+            # Features
+            "/lifespan",
             "/environ",
-            "/messages",
+            "/state/increment",
+            "/wait-for/{delay}",
+            # Local cache
+            "/cache/get/{key}",
+            "/cache/set/{key}/{value}",
+            "/cache/remove/{key}",
+            "/cache/keys",
+            # Networking
             "/dns",
-            "ip/address",
+            "/ip/address",
             "/ip/4",
             "/ip/6",
             "/internet",
+            # Error handling
+            "/raise",
+            "/crash",
+            # Aleph.im
+            "/messages",
+            "/get_a_message",
             "/post_a_message",
-            "/state/increment",
-            "/wait-for/{delay}",
+            "/post_a_message_local_account",
+            "/post_a_file",
+            "/sign_a_message",
+            # Platform properties
             "/platform/os",
             "/platform/python",
             "/platform/pip-freeze",
@@ -91,10 +117,11 @@ async def environ() -> dict[str, str]:
 
 
 @app.get("/messages")
-async def read_aleph_messages():
+async def read_aleph_messages() -> dict[str, MessagesResponse]:
     """Read data from Aleph using the Aleph Client library."""
-    async with AlephClient() as client:
-        data = await client.get_messages(hashes=["f246f873c3e0f637a15c566e7a465d2ecbb83eaa024d54ccb8fb566b549a929e"])
+    async with AlephHttpClient() as client:
+        message_filter = MessageFilter(hashes=["f246f873c3e0f637a15c566e7a465d2ecbb83eaa024d54ccb8fb566b549a929e"])
+        data = await client.get_messages(message_filter=message_filter)
     return {"Messages": data}
 
 
@@ -163,9 +190,13 @@ async def connect_ipv6():
                 if resp.status != 404:
                     resp.raise_for_status()
                 return {"result": True, "headers": resp.headers}
-        except aiohttp.ClientTimeout:
-            logger.warning(f"Session connection for host {ipv6_host} failed")
-            return {"result": False, "headers": resp.headers}
+        except TimeoutError:
+            logger.warning(f"Session connection to host {ipv6_host} timed out")
+            return {"result": False, "reason": "Timeout"}
+        except aiohttp.ClientConnectionError as error:
+            logger.warning(f"Client connection to host {ipv6_host} failed: {error}")
+            # Get a string that describes the error
+            return {"result": False, "reason": str(error.args[0])}
 
 
 async def check_url(internet_host: HttpUrl, timeout_seconds: int = 5):
@@ -184,7 +215,7 @@ async def check_url(internet_host: HttpUrl, timeout_seconds: int = 5):
 @app.get("/internet")
 async def read_internet():
     """Check Internet connectivity of the system, requiring IP connectivity, domain resolution and HTTPS/TLS."""
-    internet_hosts: List[HttpUrl] = [
+    internet_hosts: list[HttpUrl] = [
         HttpUrl(url="https://aleph.im/", scheme="https"),
         HttpUrl(url="https://ethereum.org", scheme="https"),
         HttpUrl(url="https://ipfs.io/", scheme="https"),
@@ -192,7 +223,7 @@ async def read_internet():
     timeout_seconds = 5
 
     # Create a list of tasks to check the URLs in parallel
-    tasks: set[asyncio.Task] = set(asyncio.create_task(check_url(host, timeout_seconds)) for host in internet_hosts)
+    tasks: set[asyncio.Task] = {asyncio.create_task(check_url(host, timeout_seconds)) for host in internet_hosts}
 
     # While no tasks have completed, keep waiting for the next one to finish
     while tasks:
@@ -211,34 +242,121 @@ async def read_internet():
     return {"result": False}
 
 
-@app.get("/post_a_message")
-async def post_a_message():
-    """Post a message on the Aleph network"""
+@app.get("/get_a_message")
+async def get_a_message():
+    """Get a message from the Aleph.im network"""
+    item_hash = "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af"
+    async with AlephHttpClient() as client:
+        message = await client.get_message(
+            item_hash=item_hash,
+            message_type=ProgramMessage,
+        )
+        return message.dict()
 
-    account = await RemoteAccount.from_crypto_host(host="http://localhost", unix_socket="/tmp/socat-socket")
+
+@app.post("/post_a_message")
+async def post_with_remote_account():
+    """Post a message on the Aleph.im network using the remote account of the host."""
+    try:
+        account = await RemoteAccount.from_crypto_host(host="http://localhost", unix_socket="/tmp/socat-socket")
+
+        content = {
+            "date": datetime.now(tz=timezone.utc).isoformat(),
+            "test": True,
+            "answer": 42,
+            "something": "interesting",
+        }
+        async with AuthenticatedAlephHttpClient(
+            account=account,
+        ) as client:
+            message: PostMessage
+            status: MessageStatus
+            message, status = await client.create_post(
+                post_content=content,
+                post_type="test",
+                ref=None,
+                channel="TEST",
+                inline=True,
+                storage_engine=StorageEnum.storage,
+                sync=True,
+            )
+            if status != MessageStatus.PROCESSED:
+                return JSONResponse(status_code=500, content={"error": status})
+        return {
+            "message": message,
+        }
+    except aiohttp.client_exceptions.UnixClientConnectorError:
+        return JSONResponse(status_code=500, content={"error": "Could not connect to the remote account"})
+
+
+@app.post("/post_a_message_local_account")
+async def post_with_local_account():
+    """Post a message on the Aleph.im network using a local private key."""
+
+    account = get_fallback_account()
 
     content = {
-        "date": datetime.utcnow().isoformat(),
+        "date": datetime.now(tz=timezone.utc).isoformat(),
         "test": True,
         "answer": 42,
         "something": "interesting",
     }
-    async with AuthenticatedAlephClient(
+    async with AuthenticatedAlephHttpClient(
         account=account,
+        api_server="https://api2.aleph.im",
+        allow_unix_sockets=False,
     ) as client:
-        response = await client.create_post(
+        message: PostMessage
+        status: MessageStatus
+        message, status = await client.create_post(
             post_content=content,
             post_type="test",
             ref=None,
             channel="TEST",
             inline=True,
             storage_engine=StorageEnum.storage,
+            sync=True,
+        )
+        if status != MessageStatus.PROCESSED:
+            return JSONResponse(status_code=500, content={"error": status})
+    return {
+        "message": message,
+    }
+
+
+@app.post("/post_a_file")
+async def post_a_file():
+    account = get_fallback_account()
+    file_path = Path(__file__).absolute()
+    async with AuthenticatedAlephHttpClient(
+        account=account,
+    ) as client:
+        message: StoreMessage
+        status: MessageStatus
+        message, status = await client.create_store(
+            file_path=file_path,
+            ref=None,
+            channel="TEST",
+            storage_engine=StorageEnum.storage,
+            sync=True,
         )
+        if status != MessageStatus.PROCESSED:
+            return JSONResponse(status_code=500, content={"error": status})
     return {
-        "response": response,
+        "message": message,
     }
 
 
+@app.get("/sign_a_message")
+async def sign_a_message():
+    """Sign a message using a locally managed account within the virtual machine."""
+    # FIXME: Broken, fixing this depends on https://github.com/aleph-im/aleph-sdk-python/pull/120
+    account = get_fallback_account()
+    message = {"hello": "world", "chain": "ETH"}
+    signed_message = await account.sign_message(message)
+    return {"message": signed_message}
+
+
 @app.get("/cache/get/{key}")
 async def get_from_cache(key: str):
     """Get data in the VM cache"""
@@ -265,7 +383,7 @@ async def keys_from_cache(pattern: str = "*"):
 
 
 @app.get("/state/increment")
-async def increment():
+async def increment() -> dict[str, int]:
     path = "/var/lib/example/storage.json"
     try:
         with open(path) as fd:
@@ -284,7 +402,7 @@ class Data(BaseModel):
 
 
 @app.post("/post")
-async def receive_post(data: Data):
+async def receive_post(data: Data) -> str:
     return str(data)
 
 
@@ -293,13 +411,14 @@ class CustomError(Exception):
 
 
 @app.get("/raise")
-def raise_error():
+def raise_error() -> None:
     """Raises an error to check that the init handles it properly without crashing"""
-    raise CustomError("Whoops")
+    error_message = "Whoops"
+    raise CustomError(error_message)
 
 
 @app.get("/crash")
-def crash():
+def crash() -> None:
     """Crash the entire VM in order to check that the supervisor can handle it"""
     sys.exit(1)
 
@@ -313,22 +432,22 @@ def crash():
 
 
 @app.get("/platform/os")
-def platform_os():
+def platform_os() -> PlainTextResponse:
     return PlainTextResponse(content=Path("/etc/os-release").read_text())
 
 
 @app.get("/platform/python")
-def platform_python():
+def platform_python() -> PlainTextResponse:
     return PlainTextResponse(content=sys.version)
 
 
 @app.get("/platform/pip-freeze")
-def platform_pip_freeze():
+def platform_pip_freeze() -> list[str]:
     return list(freeze())
 
 
 @app.event(filters=filters)
-async def aleph_event(event):
+async def aleph_event(event) -> dict[str, str]:
     print("aleph_event", event)
     async with aiohttp.ClientSession(connector=aiohttp.TCPConnector()) as session:
         async with session.get("https://official.aleph.cloud/api/v0/info/public.json") as resp:
diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index 6a0c2265a..78c96b897 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -36,7 +36,7 @@ locale-gen en_US.UTF-8
 
 echo "Pip installing aleph-sdk-python"
 mkdir -p /opt/aleph/libs
-pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.9.0' 'fastapi~=0.109.2'
+pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.9.0' 'aleph-message==0.4.4' 'fastapi~=0.109.2'
 
 # Compile Python code to bytecode for faster execution
 # -o2 is needed to compile with optimization level 2 which is what we launch init1.py (`python -OO`)
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 29a5317f3..e84c58c31 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -289,7 +289,7 @@ class Settings(BaseSettings):
     )
     FAKE_INSTANCE_MESSAGE = Path(abspath(join(__file__, "../../../../examples/instance_message_from_aleph.json")))
 
-    CHECK_FASTAPI_VM_ID = "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af"
+    CHECK_FASTAPI_VM_ID = "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace"
     LEGACY_CHECK_FASTAPI_VM_ID = "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
 
     # Developer options
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index 6e429ff87..8dec7e963 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -44,6 +44,7 @@ async def build_asgi_scope(path: str, request: web.Request) -> dict[str, Any]:
 
 
 async def build_event_scope(event) -> dict[str, Any]:
+    """Build an ASGI scope for an event."""
     return {
         "type": "aleph.message",
         "body": event,
diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py
index 8c9c8064a..b0d76554d 100644
--- a/src/aleph/vm/orchestrator/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -15,19 +15,31 @@
 logger = logging.getLogger(__name__)
 
 
-def make_check_vm_url(vm_id: ItemHash) -> str:
+def assemble_vm_url(vm_id: ItemHash) -> str:
+    """Assemble the URL for a VM based on the host and port that the orchestrator is running on and the VM ID."""
     return f"http://{settings.SUPERVISOR_HOST}:{settings.SUPERVISOR_PORT}/vm/{vm_id}"
 
 
 async def get_json_from_vm(session: ClientSession, vm_id: ItemHash, suffix: str) -> Any:
-    vm_url = make_check_vm_url(vm_id)
+    """Get JSON from a VM running locally."""
+    vm_url = assemble_vm_url(vm_id)
     url = f"{vm_url}{suffix}"
     async with session.get(url) as resp:
         resp.raise_for_status()
         return await resp.json()
 
 
+async def post_to_vm(session: ClientSession, vm_id: ItemHash, suffix: str, data: Any = None) -> Any:
+    """Post data to a VM running locally."""
+    vm_url = assemble_vm_url(vm_id)
+    url = f"{vm_url}{suffix}"
+    async with session.post(url, json=data) as resp:
+        resp.raise_for_status()
+        return await resp.json()
+
+
 async def check_index(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the index page of the VM is working."""
     try:
         result: dict = await get_json_from_vm(session, vm_id, "/")
         assert result["Example"] == "example_fastapi"
@@ -37,6 +49,7 @@ async def check_index(session: ClientSession, vm_id: ItemHash) -> bool:
 
 
 async def check_lifespan(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the lifespan endpoint of the VM is working."""
     try:
         result: dict = await get_json_from_vm(session, vm_id, "/lifespan")
         return result["Lifespan"] is True
@@ -45,6 +58,7 @@ async def check_lifespan(session: ClientSession, vm_id: ItemHash) -> bool:
 
 
 async def check_environ(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the environ endpoint of the VM returns the expected environment variables."""
     try:
         result: dict = await get_json_from_vm(session, vm_id, "/environ")
         assert "ALEPH_API_HOST" in result
@@ -58,6 +72,7 @@ async def check_environ(session: ClientSession, vm_id: ItemHash) -> bool:
 
 
 async def check_messages(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the messages endpoint of the VM returns a list of messages."""
     try:
         result: dict = await get_json_from_vm(session, vm_id, "/messages")
         assert "Messages" in result
@@ -69,6 +84,7 @@ async def check_messages(session: ClientSession, vm_id: ItemHash) -> bool:
 
 
 async def check_dns(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the DNS endpoint of the VM returns both IPv4 and IPv6 results."""
     try:
         result: dict = await get_json_from_vm(session, vm_id, "/dns")
         assert result["ipv4"]
@@ -79,6 +95,7 @@ async def check_dns(session: ClientSession, vm_id: ItemHash) -> bool:
 
 
 async def check_ipv4(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the VM has IPv4 connectivity."""
     try:
         result: dict = await get_json_from_vm(session, vm_id, "/ip/4")
         assert result["result"] is True
@@ -88,6 +105,7 @@ async def check_ipv4(session: ClientSession, vm_id: ItemHash) -> bool:
 
 
 async def check_ipv6(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the VM has IPv6 connectivity."""
     try:
         result: dict = await get_json_from_vm(session, vm_id, "/ip/6")
         assert result["result"] is True
@@ -98,6 +116,7 @@ async def check_ipv6(session: ClientSession, vm_id: ItemHash) -> bool:
 
 
 async def check_internet(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the VM has internet connectivity. This requires DNS, IP, HTTP and TLS to work."""
     try:
         result: dict = await get_json_from_vm(session, vm_id, "/internet")
         assert result["result"] == HTTPOk.status_code
@@ -108,6 +127,7 @@ async def check_internet(session: ClientSession, vm_id: ItemHash) -> bool:
 
 
 async def check_cache(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the VM can set and get a value in its cache."""
     try:
         result1: bool = await get_json_from_vm(session, vm_id, "/cache/set/a/42")
         assert result1 is True
@@ -121,6 +141,7 @@ async def check_cache(session: ClientSession, vm_id: ItemHash) -> bool:
 
 
 async def check_persistent_storage(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the VM can set and get a value in its persistent storage."""
     try:
         result: dict = await get_json_from_vm(session, vm_id, "/state/increment")
         counter = result["counter"]
@@ -134,7 +155,8 @@ async def check_persistent_storage(session: ClientSession, vm_id: ItemHash) -> b
 
 
 async def check_error_raised(session: ClientSession, vm_id: ItemHash) -> bool:
-    vm_url = make_check_vm_url(vm_id)
+    """Check that the VM can raise an error and return a traceback instead of crashing."""
+    vm_url = assemble_vm_url(vm_id)
     try:
         async with session.get(f"{vm_url}/raise") as resp:
             text = await resp.text()
@@ -144,8 +166,9 @@ async def check_error_raised(session: ClientSession, vm_id: ItemHash) -> bool:
 
 
 async def check_crash_and_restart(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that a crash in the VM would cause it to restart and work as expected."""
     # Crash the VM init.
-    vm_url = make_check_vm_url(vm_id)
+    vm_url = assemble_vm_url(vm_id)
     async with session.get(f"{vm_url}/crash") as resp:
         if resp.status != HTTPBadGateway.status_code:
             return False
@@ -158,3 +181,30 @@ async def check_crash_and_restart(session: ClientSession, vm_id: ItemHash) -> bo
 
     except ClientResponseError:
         return False
+
+
+async def check_get_a_message(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the VM can get a message from the aleph.im network."""
+    try:
+        result: dict = await get_json_from_vm(session, vm_id, "/get_a_message")
+        return "item_hash" in result
+    except ClientResponseError:
+        return False
+
+
+async def check_post_a_message(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the VM can post a message to the aleph.im network using a remote key present on the host."""
+    try:
+        result: dict = await post_to_vm(session, vm_id, "/post_a_message")
+        return "item_hash" in result
+    except ClientResponseError:
+        return False
+
+
+async def check_sign_a_message(session: ClientSession, vm_id: ItemHash) -> bool:
+    """Check that the VM can sign a message using a key local to the VM."""
+    try:
+        result: dict = await post_to_vm(session, vm_id, "/sign_a_message")
+        return "item_hash" in result
+    except ClientResponseError:
+        return False
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 994476cba..177e6a348 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -199,6 +199,9 @@ async def status_check_fastapi(request: web.Request, vm_id: Optional[ItemHash] =
                 "index": await status.check_index(session, fastapi_vm_id),
                 "environ": await status.check_environ(session, fastapi_vm_id),
                 "messages": await status.check_messages(session, fastapi_vm_id),
+                # Using the remote account currently causes issues
+                # "post_a_message": await status.check_post_a_message(session, fastapi_vm_id),
+                # "sign_a_message": await status.check_sign_a_message(session, fastapi_vm_id),
                 "dns": await status.check_dns(session, fastapi_vm_id),
                 "ipv4": await status.check_ipv4(session, fastapi_vm_id),
                 "internet": await status.check_internet(session, fastapi_vm_id),
@@ -209,6 +212,7 @@ async def status_check_fastapi(request: web.Request, vm_id: Optional[ItemHash] =
             if not retro_compatibility:
                 # These fields were added in the runtime running Debian 12.
                 result = result | {
+                    "get_a_message": await status.check_get_a_message(session, fastapi_vm_id),
                     "lifespan": await status.check_lifespan(session, fastapi_vm_id),
                     # IPv6 requires extra work from node operators and is not required yet.
                     # "ipv6": await status.check_ipv6(session),

From c7db4efb963f58c485e599e67bf9b9d14ff81947 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 23 Apr 2024 16:41:49 +0200
Subject: [PATCH 753/990] Fix: Pytest did not test legacy diagnostic

---
 tests/supervisor/test_execution.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tests/supervisor/test_execution.py b/tests/supervisor/test_execution.py
index 551e866e9..afaa82ce7 100644
--- a/tests/supervisor/test_execution.py
+++ b/tests/supervisor/test_execution.py
@@ -58,11 +58,13 @@ async def test_create_execution():
 
 
 @pytest.mark.asyncio
-async def test_create_execution_online():
+async def test_create_execution_online(vm_hash: ItemHash = None):
     """
     Create a new VM execution without building it locally and check that it starts properly.
     """
 
+    vm_hash = vm_hash or settings.CHECK_FASTAPI_VM_ID
+
     # Ensure that the settings are correct and required files present.
     settings.setup()
     settings.check()
@@ -71,7 +73,6 @@ async def test_create_execution_online():
     engine = metrics.setup_engine()
     await metrics.create_tables(engine)
 
-    vm_hash = ItemHash("3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af")
     message = await get_message(ref=vm_hash)
 
     execution = VmExecution(
@@ -93,3 +94,11 @@ async def test_create_execution_online():
 
     await execution.start()
     await execution.stop()
+
+
+@pytest.mark.asyncio
+async def test_create_execution_legacy():
+    """
+    Create a new VM execution based on the legacy FastAPI check and ensure that it starts properly.
+    """
+    await test_create_execution_online(vm_hash=settings.LEGACY_CHECK_FASTAPI_VM_ID)

From 8318c435966ab4443bbec844f767bdb080200816 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 3 May 2024 07:58:11 +0200
Subject: [PATCH 754/990] Installation documentation was moved to aleph doc

Point to it in the documentation
and removed duplicated information here
---
 README.md                   |   4 +-
 doc/INSTALL-Debian-11.md    | 169 +----------------------------------
 doc/INSTALL-Debian-12.md    | 170 +----------------------------------
 doc/INSTALL-Ubuntu-20.04.md |  11 +--
 doc/INSTALL-Ubuntu-22.04.md | 171 +-----------------------------------
 doc/INSTALL.md              |   4 +-
 6 files changed, 6 insertions(+), 523 deletions(-)

diff --git a/README.md b/README.md
index 9b3733300..783f02a09 100644
--- a/README.md
+++ b/README.md
@@ -17,9 +17,7 @@ Writing programs in Python using ASGI compatible frameworks (
 
 Install Aleph-VM to run an Aleph.im Compute Resource Node easily from official pre-built packages.
 
-- [On Debian 11](./doc/INSTALL-Debian-11.md)
-- [On Debian 12](./doc/INSTALL-Debian-12.md)
-- [On Ubuntu 22.04](./doc/INSTALL-Ubuntu-22.04.md)
+See the official user doc https://docs.aleph.im/nodes/compute/
 
 ## 2. Install Aleph-VM from source
 
diff --git a/doc/INSTALL-Debian-11.md b/doc/INSTALL-Debian-11.md
index 577fe4ac0..242296c04 100644
--- a/doc/INSTALL-Debian-11.md
+++ b/doc/INSTALL-Debian-11.md
@@ -1,168 +1 @@
-# Installing Aleph-VM on a server / Debian 11 Bullseye
-
-## 0. Introduction
-
-For production using official Debian packages.
-
-## 1. Requirements
-
-- A [supported Linux server](../src/aleph/vm/orchestrator/README.md#1-supported-platforms)
-- A public domain name from a registrar and top level domain you trust. 
-
-In order to run an official Aleph.im Compute Resource Node (CRN), you will also need the following resources:
-
-- CPU (2 options):
-  - Min. 8 cores / 16 threads, 3.0 ghz+ CPU (gaming CPU for fast boot-up of microVMs)
-  - Min. 12 core / 24 threads, 2.4ghz+ CPU (datacenter CPU for multiple concurrent loads)
-- RAM: 64GB
-- STORAGE: 1TB (NVMe SSD preferred, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
-- BANDWIDTH: Minimum of 500 MB/s
-
-You will need a public domain name with access to add TXT and wildcard records.
-
-> 💡 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
-
-## 2. Installation
-
-Run the following commands as `root`:
-
-First install the [VM-Connector](../vm_connector/README.md) using Docker:
-```shell
-apt update
-apt upgrade
-apt install -y docker.io apparmor-profiles
-docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
-```
-
-Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
-The procedure is similar for updates.
-```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.1/aleph-vm.debian-11.deb
-apt install /opt/aleph-vm.debian-11.deb
-```
-
-Reboot if required (new kernel, ...).
-
-### Configuration
-
-Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
-
-#### Hostname
-
-You will want to insert your domain name in the form of:
-```
-ALEPH_VM_DOMAIN_NAME=vm.example.org
-```
-
-#### Network configuration
-
-On some systems, the default network interface is not `eth0` and you will want to configure the default interface
-by adding:
-```
-ALEPH_VM_NETWORK_INTERFACE=enp0s1
-```
-(don't forget to replace `enp0s1` with the name of your default network interface).
-
-Debian 11 by default uses `/etc/resolv.conf` for DNS resolution. The VM Supervisor uses this by default.
-If your system uses [systemd-resolved](https://manpages.debian.org/bullseye/systemd/systemd-resolved.8.en.html)
-instead, uncomment and add the following setting:
-```
-#ALEPH_VM_DNS_RESOLUTION=resolvctl
-```
-
-> 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
-
-#### Volumes and partitions
-
-Two directories are used to store data from the network:
-- `/var/lib/aleph/vm` contains all the execution and persistent data.
-- `/var/cache/aleph/vm` contains data downloaded from the network.
-
-These two directories must be stored on the same partition.
-That partition must meet the minimum requirements specified for a CRN.
-
-> 💡 This is required due to the software using hard links to optimize performance and disk usage.
-
-#### Applying changes
-
-Finally, restart the service:
-```shell
-systemctl restart aleph-vm-supervisor
-```
-
-## 3. Reverse Proxy
-
-We document how to use Caddy as a reverse proxy since it manages and renews HTTPS certificates automatically.
-
-Any other reverse-proxy (Nginx, HAProxy, Apache2, ...) should do the job as well, just make sure to renew the 
-HTTPS/TLS certificates on time.
-
-First, create a domain name that points to the server on IPv4 (A) and IPv6 (AAAA).
-
-This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](/CONFIGURE_CADDY.md).
-
-Again, run these commands as `root`:
-```shell
- apt install -y debian-keyring debian-archive-keyring apt-transport-https
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list
-apt update
-apt install caddy
-```
-
-Then, after replacing the domain `vm.example.org` with your own, use configure Caddy:
-```shell
-cat >/etc/caddy/Caddyfile < ![image](https://user-images.githubusercontent.com/404665/150202090-91a02536-4e04-4af2-967f-fe105d116e1f.png)
-
-If you face an issue, check the logs of the different services for errors:
-
-VM-Supervisor:
-```shell
-journalctl -f -u aleph-vm-supervisor.service 
-```
-
-Caddy:
-```shell
-journalctl -f -u caddy.service 
-```
-
-VM-Connector:
-```shell
-docker logs -f vm-connector
-```
-
-### Common errors
-
-#### "Network interface eth0 does not exist"
-
-Did you update the configuration file `/etc/aleph-vm/supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
-the default network interface of your server ?
-
-#### "Aleph Connector unavailable"
-
-Investigate the installation of the VM-Connector using Docker in step 2.
+[[https://docs.aleph.im/nodes/compute/installation/debian-11/]]
\ No newline at end of file
diff --git a/doc/INSTALL-Debian-12.md b/doc/INSTALL-Debian-12.md
index e3826b327..bc2b74cb7 100644
--- a/doc/INSTALL-Debian-12.md
+++ b/doc/INSTALL-Debian-12.md
@@ -1,169 +1 @@
-# Installing Aleph-VM on a server / Debian 12 Bookworm
-
-## 0. Introduction
-
-For production using official Debian packages.
-
-## 1. Requirements
-
-- A [supported Linux server](../src/aleph/vm/orchestrator/README.md#1-supported-platforms)
-- A public domain name from a registrar and top level domain you trust. 
-
-In order to run an official Aleph.im Compute Resource Node (CRN), you will also need the following resources:
-
-- CPU (2 options):
-  - Min. 8 cores / 16 threads, 3.0 ghz+ CPU (gaming CPU for fast boot-up of microVMs)
-  - Min. 12 core / 24 threads, 2.4ghz+ CPU (datacenter CPU for multiple concurrent loads)
-- RAM: 64GB
-- STORAGE: 1TB (NVMe SSD preferred, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
-- BANDWIDTH: Minimum of 500 MB/s
-
-You will need a public domain name with access to add TXT and wildcard records.
-
-> 💡 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
-
-## 2. Installation
-
-Run the following commands as `root`:
-
-First install the [VM-Connector](../vm_connector/README.md) using Docker:
-```shell
-apt update
-apt upgrade
-apt install -y docker.io apparmor-profiles
-docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
-```
-
-Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
-The procedure is similar for updates.
-```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.1/aleph-vm.debian-12.deb
-apt install /opt/aleph-vm.debian-12.deb
-```
-
-Reboot if required (new kernel, ...).
-
-### Configuration
-
-Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
-
-#### Hostname
-
-You will want to insert your domain name in the form of:
-```
-ALEPH_VM_DOMAIN_NAME=vm.example.org
-```
-
-#### Network configuration
-
-The network configuration is detected automatically.
-
-The default network interface is detected automatically from the IP routes. 
-You can configure the default interface manually instead by adding:
-```
-ALEPH_VM_NETWORK_INTERFACE=enp0s1
-```
-(don't forget to replace `enp0s1` with the name of your default network interface).
-
-You can configure the DNS resolver manually by using one of the following options:
-```
-ALEPH_VM_DNS_RESOLUTION=resolvectl
-ALEPH_VM_DNS_RESOLUTION=resolv.conf
-```
-
-> 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
-
-#### Volumes and partitions
-
-Two directories are used to store data from the network:
-- `/var/lib/aleph/vm` contains all the execution and persistent data.
-- `/var/cache/aleph/vm` contains data downloaded from the network.
-
-These two directories must be stored on the same partition.
-That partition must meet the minimum requirements specified for a CRN.
-
-> 💡 This is required due to the software using hard links to optimize performance and disk usage.
-
-#### Applying changes
-
-Finally, restart the service:
-```shell
-systemctl restart aleph-vm-supervisor
-```
-
-## 3. Reverse Proxy
-
-We document how to use Caddy as a reverse proxy since it manages and renews HTTPS certificates automatically.
-
-Any other reverse-proxy (Nginx, HAProxy, Apache2, ...) should do the job as well, just make sure to renew the 
-HTTPS/TLS certificates on time.
-
-First, create a domain name that points to the server on IPv4 (A) and IPv6 (AAAA).
-
-This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](/CONFIGURE_CADDY.md).
-
-Again, run these commands as `root`:
-```shell
- apt install -y debian-keyring debian-archive-keyring apt-transport-https
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list
-apt update
-apt install caddy
-```
-
-Then, after replacing the domain `vm.example.org` with your own, use configure Caddy:
-```shell
-cat >/etc/caddy/Caddyfile < ![image](https://user-images.githubusercontent.com/404665/150202090-91a02536-4e04-4af2-967f-fe105d116e1f.png)
-
-If you face an issue, check the logs of the different services for errors:
-
-VM-Supervisor:
-```shell
-journalctl -f -u aleph-vm-supervisor.service 
-```
-
-Caddy:
-```shell
-journalctl -f -u caddy.service 
-```
-
-VM-Connector:
-```shell
-docker logs -f vm-connector
-```
-
-### Common errors
-
-#### "Network interface eth0 does not exist"
-
-Did you update the configuration file `/etc/aleph-vm/supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
-the default network interface of your server ?
-
-#### "Aleph Connector unavailable"
-
-Investigate the installation of the VM-Connector using Docker in step 2.
+[[https://docs.aleph.im/nodes/compute/installation/debian-12/]]
\ No newline at end of file
diff --git a/doc/INSTALL-Ubuntu-20.04.md b/doc/INSTALL-Ubuntu-20.04.md
index 0cd067b36..6705c5bb7 100644
--- a/doc/INSTALL-Ubuntu-20.04.md
+++ b/doc/INSTALL-Ubuntu-20.04.md
@@ -1,10 +1 @@
-# Installing Aleph-VM on a server / Ubuntu 20.04 Focal Fossa (Deprecated) 
-
-Support for Ubuntu 20.04 was due to compatibility issues with
-the NFTables firewall introduced in version 
-[0.2.6](https://github.com/aleph-im/aleph-vm/releases/tag/0.2.6).
-
-We recommend upgrading to the newest Ubuntu LTS version 
-and then use the 
-[following instructions on Ubuntu 22.04](./INSTALL-Ubuntu-22.04.md)
-).
+Moved to [[https://docs.aleph.im/nodes/compute/installation/ubuntu-20.04/]]
\ No newline at end of file
diff --git a/doc/INSTALL-Ubuntu-22.04.md b/doc/INSTALL-Ubuntu-22.04.md
index b4fdfeb0d..010b81536 100644
--- a/doc/INSTALL-Ubuntu-22.04.md
+++ b/doc/INSTALL-Ubuntu-22.04.md
@@ -1,170 +1 @@
-# Installing Aleph-VM on a server / Ubuntu 22.04 Jammy Jellyfish
-
-## 0. Introduction
-
-For production using official Debian packages.
-
-## 1. Requirements
-
-- A [supported Linux server](../src/aleph/vm/orchestrator/README.md#1-supported-platforms)
-- A public domain name from a registrar and top level domain you trust. 
-
-In order to run an official Aleph.im Compute Resource Node (CRN), you will also need the following resources:
-
-- CPU (2 options):
-  - Min. 8 cores / 16 threads, 3.0 ghz+ CPU (gaming CPU for fast boot-up of microVMs)
-  - Min. 12 core / 24 threads, 2.4ghz+ CPU (datacenter CPU for multiple concurrent loads)
-- RAM: 64GB
-- STORAGE: 1TB (NVMe SSD preferred, datacenter fast HDD possible under conditions, you’ll want a big and fast cache)
-- BANDWIDTH: Minimum of 500 MB/s
-
-You will need a public domain name with access to add TXT and wildcard records.
-
-> 💡 This documentation will use the invalid `vm.example.org` domain name. Replace it when needed.
-
-## 2. Installation
-
-Run the following commands:
-
-First install the [VM-Connector](../vm_connector/README.md) using Docker:
-```shell
-sudo apt update
-sudo apt upgrade
-sudo apt install -y docker.io
-docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
-```
-
-Then install the [VM-Supervisor](../src/aleph/vm/orchestrator/README.md) using the official Debian package.
-The procedure is similar for updates.
-```shell
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/0.3.1/aleph-vm.ubuntu-22.04.deb
-sudo apt install /opt/aleph-vm.ubuntu-22.04.deb
-```
-
-Reboot if required (new kernel, ...).
-
-### Configuration
-
-#### Hostname
-
-Update the configuration in `/etc/aleph-vm/supervisor.env` using your favourite editor.
-
-You will want to insert your domain name in the form of:
-```
-ALEPH_VM_DOMAIN_NAME=vm.example.org
-```
-
-#### Network configuration
-
-The network configuration is detected automatically.
-
-The default network interface is detected automatically from the IP routes. 
-You can configure the default interface manually instead by adding:
-```
-ALEPH_VM_NETWORK_INTERFACE=enp0s1
-```
-(don't forget to replace `enp0s1` with the name of your default network interface).
-
-You can configure the DNS resolver manually by using one of the following options:
-```
-ALEPH_VM_DNS_RESOLUTION=resolvectl
-ALEPH_VM_DNS_RESOLUTION=resolv.conf
-```
-
-> 💡 You can instead specify the DNS resolvers used by the VMs using `ALEPH_VM_DNS_NAMESERVERS=["1.2.3.4", "5.6.7.8"]`.
-
-
-#### Volumes and partitions
-
-Two directories are used to store data from the network:
-- `/var/lib/aleph/vm` contains all the execution and persistent data.
-- `/var/cache/aleph/vm` contains data downloaded from the network.
-
-These two directories must be stored on the same partition.
-That partition must meet the minimum requirements specified for a CRN.
-
-> 💡 This is required due to the software using hard links to optimize performance and disk usage.
-
-#### Applying changes
-
-Finally, restart the service:
-```shell
-sudo systemctl restart aleph-vm-supervisor
-```
-
-## 3. Reverse Proxy
-
-We document how to use Caddy as a reverse proxy since it manages and renews HTTPS certificates automatically.
-
-Any other reverse-proxy (Nginx, HAProxy, Apache2, ...) should do the job as well, just make sure to renew the 
-HTTPS/TLS certificates on time.
-
-First, create a domain name that points to the server on IPv4 (A) and IPv6 (AAAA).
-
-This is a simple configuration. For more options, check [CONFIGURE_CADDY.md](/CONFIGURE_CADDY.md).
-
-Again, run these commands as `root`:
-```shell
-sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
-curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list
-sudo apt update
-sudo apt install caddy
-```
-
-Then, after replacing the domain `vm.example.org` with your own, use configure Caddy:
-```shell
-sudo cat >/etc/caddy/Caddyfile < ![image](https://user-images.githubusercontent.com/404665/150202090-91a02536-4e04-4af2-967f-fe105d116e1f.png)
-
-If you face an issue, check the logs of the different services for errors:
-
-VM-Supervisor:
-```shell
-sudo journalctl -f -u aleph-vm-supervisor.service 
-```
-
-Caddy:
-```shell
-sudo journalctl -f -u caddy.service 
-```
-
-VM-Connector:
-```shell
-sudo docker logs -f vm-connector
-```
-
-### Common errors
-
-#### "Network interface eth0 does not exist"
-
-Did you update the configuration file `/etc/aleph-vm/supervisor.env` with `ALEPH_VM_NETWORK_INTERFACE` equal to 
-the default network interface of your server ?
-
-#### "Aleph Connector unavailable"
-
-Investigate the installation of the VM-Connector using Docker in step 2.
+[[https://docs.aleph.im/nodes/compute/installation/ubuntu-22.04/]]
\ No newline at end of file
diff --git a/doc/INSTALL.md b/doc/INSTALL.md
index 79e8d18cf..d42a9f50e 100644
--- a/doc/INSTALL.md
+++ b/doc/INSTALL.md
@@ -1,4 +1,2 @@
 # Installing Aleph-VM
-
-- [On Debian 11](./INSTALL-Debian-11.md)
-- [On Ubuntu 22.04](./INSTALL-Ubuntu-22.04.md)
+see [[ https://docs.aleph.im/nodes/compute/]]
\ No newline at end of file

From ab1b9cd90a18d5cb4635c0e1c6712bdc06a940c9 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 29 Apr 2024 16:45:01 +0200
Subject: [PATCH 755/990] CI check system usage endpoint

---
 .github/workflows/test-on-droplets-matrix.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index c67c1688f..f99f30a0b 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -238,6 +238,14 @@ jobs:
             -d '{"persistent_vms": [], "instances": ["${{ matrix.check_vm.item_hash }}"]}' \
             "http://${DROPLET_IPV4}:4020/control/allocations"
 
+      - name: Get system usage
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          curl -X GET -H "Content-Type: application/json" \
+            -H "X-Auth-Signature: test" \
+            "http://${DROPLET_IPV4}:4020/about/usage/system"
+
+
       - name: Export aleph logs
         if: always()
         run: |

From ca8ae7bfffd1c085ddb73bab6a1ea4c2b1599f64 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 29 Apr 2024 17:42:46 +0200
Subject: [PATCH 756/990] add unit test for system usage

---
 tests/supervisor/test_views.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
index 49a6fa91e..73bcfec45 100644
--- a/tests/supervisor/test_views.py
+++ b/tests/supervisor/test_views.py
@@ -24,3 +24,16 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client):
             "type": "value_error.unknownhash",
         },
     ]
+
+
+@pytest.mark.asyncio
+async def test_system_usage(aiohttp_client):
+    """Test that the allocation endpoint fails when an invalid item_hash is provided."""
+    client = await aiohttp_client(app)
+    settings.ALLOCATION_TOKEN_HASH = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"  # = "test"
+    response: web.Response = await client.get("/about/usage/system")
+    assert response.status == 200
+    # check if it is valid json
+    resp = await response.json()
+    assert "cpu" in resp
+    assert resp["cpu"]["count"] > 0

From d100df1a01eaf35aff2a4cb2dd5f17e713fa56b9 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 30 Apr 2024 09:53:13 +0200
Subject: [PATCH 757/990] Set up a fresh web_app for each test as required by
 aiohttp

---
 src/aleph/vm/orchestrator/resources.py  |   5 +-
 src/aleph/vm/orchestrator/supervisor.py | 112 ++++++++++++------------
 tests/supervisor/test_views.py          |   4 +-
 3 files changed, 63 insertions(+), 58 deletions(-)

diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index a40c6ff13..29e819079 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -87,8 +87,8 @@ def get_machine_properties() -> MachineProperties:
     cpu_info = cpuinfo.get_cpu_info()  # Slow
     return MachineProperties(
         cpu=CpuProperties(
-            architecture=cpu_info["raw_arch_string"],
-            vendor=cpu_info["vendor_id"],
+            architecture=cpu_info.get("raw_arch_string"),
+            vendor=cpu_info.get("vendor_id"),
         ),
     )
 
@@ -118,6 +118,7 @@ async def about_system_usage(_: web.Request):
         ),
         properties=get_machine_properties(),
     )
+
     return web.json_response(text=usage.json(exclude_none=True))
 
 
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 4846104ae..892106ba0 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -74,62 +74,63 @@ async def http_not_found(request: web.Request):
     return web.HTTPNotFound()
 
 
-app = web.Application(middlewares=[server_version_middleware])
-cors = setup(
-    app,
-    defaults={
-        "*": ResourceOptions(
-            allow_credentials=True,
-            expose_headers="*",
-            allow_headers="*",
-        )
-    },
-)
+def setup_webapp():
+    app = web.Application(middlewares=[server_version_middleware])
+    cors = setup(
+        app,
+        defaults={
+            "*": ResourceOptions(
+                allow_credentials=True,
+                expose_headers="*",
+                allow_headers="*",
+            )
+        },
+    )
 
-# Routes that need CORS enabled
-cors_routes = [
-    # /about APIs return information about the VM Orchestrator
-    web.get("/about/login", about_login),
-    web.get("/about/executions/list", list_executions),
-    web.get("/about/executions/details", about_executions),
-    web.get("/about/executions/records", about_execution_records),
-    web.get("/about/usage/system", about_system_usage),
-    web.get("/about/config", about_config),
-    # /control APIs are used to control the VMs and access their logs
-    web.post("/control/allocation/notify", notify_allocation),
-    web.get("/control/machine/{ref}/logs", stream_logs),
-    web.post("/control/machine/{ref}/expire", operate_expire),
-    web.post("/control/machine/{ref}/stop", operate_stop),
-    web.post("/control/machine/{ref}/erase", operate_erase),
-    web.post("/control/machine/{ref}/reboot", operate_reboot),
-    # /status APIs are used to check that the VM Orchestrator is running properly
-    web.get("/status/check/fastapi", status_check_fastapi),
-    web.get("/status/check/fastapi/legacy", status_check_fastapi_legacy),
-    web.get("/status/check/host", status_check_host),
-    web.get("/status/check/version", status_check_version),
-    web.get("/status/check/ipv6", status_check_ipv6),
-    web.get("/status/config", status_public_config),
-]
-routes = app.add_routes(cors_routes)
-for route in routes:
-    cors.add(route)
-
-
-# Routes that don't need CORS enabled
-other_routes = [
-    # /control APIs are used to control the VMs and access their logs
-    web.post("/control/allocations", update_allocations),
-    # Raise an HTTP Error 404 if attempting to access an unknown URL within these paths.
-    web.get("/about/{suffix:.*}", http_not_found),
-    web.get("/control/{suffix:.*}", http_not_found),
-    web.get("/status/{suffix:.*}", http_not_found),
-    # /static is used to serve static files
-    web.static("/static", Path(__file__).parent / "views/static"),
-    # /vm is used to launch VMs on-demand
-    web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
-    web.route("*", "/{suffix:.*}", run_code_from_hostname),
-]
-app.add_routes(other_routes)
+    # Routes that need CORS enabled
+    cors_routes = [
+        # /about APIs return information about the VM Orchestrator
+        web.get("/about/login", about_login),
+        web.get("/about/executions/list", list_executions),
+        web.get("/about/executions/details", about_executions),
+        web.get("/about/executions/records", about_execution_records),
+        web.get("/about/usage/system", about_system_usage),
+        web.get("/about/config", about_config),
+        # /control APIs are used to control the VMs and access their logs
+        web.post("/control/allocation/notify", notify_allocation),
+        web.get("/control/machine/{ref}/logs", stream_logs),
+        web.post("/control/machine/{ref}/expire", operate_expire),
+        web.post("/control/machine/{ref}/stop", operate_stop),
+        web.post("/control/machine/{ref}/erase", operate_erase),
+        web.post("/control/machine/{ref}/reboot", operate_reboot),
+        # /status APIs are used to check that the VM Orchestrator is running properly
+        web.get("/status/check/fastapi", status_check_fastapi),
+        web.get("/status/check/fastapi/legacy", status_check_fastapi_legacy),
+        web.get("/status/check/host", status_check_host),
+        web.get("/status/check/version", status_check_version),
+        web.get("/status/check/ipv6", status_check_ipv6),
+        web.get("/status/config", status_public_config),
+    ]
+    routes = app.add_routes(cors_routes)
+    for route in routes:
+        cors.add(route)
+
+    # Routes that don't need CORS enabled
+    other_routes = [
+        # /control APIs are used to control the VMs and access their logs
+        web.post("/control/allocations", update_allocations),
+        # Raise an HTTP Error 404 if attempting to access an unknown URL within these paths.
+        web.get("/about/{suffix:.*}", http_not_found),
+        web.get("/control/{suffix:.*}", http_not_found),
+        web.get("/status/{suffix:.*}", http_not_found),
+        # /static is used to serve static files
+        web.static("/static", Path(__file__).parent / "views/static"),
+        # /vm is used to launch VMs on-demand
+        web.route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
+        web.route("*", "/{suffix:.*}", run_code_from_hostname),
+    ]
+    app.add_routes(other_routes)
+    return app
 
 
 async def stop_all_vms(app: web.Application):
@@ -153,6 +154,7 @@ def run():
 
     # Require a random token to access /about APIs
     secret_token = token_urlsafe(nbytes=32)
+    app = setup_webapp()
     # Store app singletons. Note that app["pubsub"] will also be created.
     app["secret_token"] = secret_token
     app["vm_pool"] = pool
diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
index 73bcfec45..58cad0d69 100644
--- a/tests/supervisor/test_views.py
+++ b/tests/supervisor/test_views.py
@@ -2,12 +2,13 @@
 from aiohttp import web
 
 from aleph.vm.conf import settings
-from aleph.vm.orchestrator.supervisor import app
+from aleph.vm.orchestrator.supervisor import setup_webapp
 
 
 @pytest.mark.asyncio
 async def test_allocation_fails_on_invalid_item_hash(aiohttp_client):
     """Test that the allocation endpoint fails when an invalid item_hash is provided."""
+    app = setup_webapp()
     client = await aiohttp_client(app)
     settings.ALLOCATION_TOKEN_HASH = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"  # = "test"
     response: web.Response = await client.post(
@@ -29,6 +30,7 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client):
 @pytest.mark.asyncio
 async def test_system_usage(aiohttp_client):
     """Test that the allocation endpoint fails when an invalid item_hash is provided."""
+    app = setup_webapp()
     client = await aiohttp_client(app)
     settings.ALLOCATION_TOKEN_HASH = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"  # = "test"
     response: web.Response = await client.get("/about/usage/system")

From 4303223cbc7b5da8c8db6ac03a658d17222d4166 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 30 Apr 2024 11:42:19 +0200
Subject: [PATCH 758/990] revert local compat change

---
 src/aleph/vm/orchestrator/resources.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 29e819079..1679c0525 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -87,8 +87,8 @@ def get_machine_properties() -> MachineProperties:
     cpu_info = cpuinfo.get_cpu_info()  # Slow
     return MachineProperties(
         cpu=CpuProperties(
-            architecture=cpu_info.get("raw_arch_string"),
-            vendor=cpu_info.get("vendor_id"),
+            architecture=cpu_info["raw_arch_string"],
+            vendor=cpu_info["vendor_id"],
         ),
     )
 

From 143112d09685fbff5c6ee7b4a2b6cb8f92089fb0 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 30 Apr 2024 14:11:52 +0200
Subject: [PATCH 759/990] Apparently CI also don't have matching arch

---
 src/aleph/vm/orchestrator/resources.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 1679c0525..448a822c5 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -87,8 +87,8 @@ def get_machine_properties() -> MachineProperties:
     cpu_info = cpuinfo.get_cpu_info()  # Slow
     return MachineProperties(
         cpu=CpuProperties(
-            architecture=cpu_info["raw_arch_string"],
-            vendor=cpu_info["vendor_id"],
+            architecture=cpu_info.get("raw_arch_string", cpu_info.get("arch_string_raw")),
+            vendor=cpu_info.get("vendor_id", cpu_info.get("vendor_id_raw")),
         ),
     )
 

From 448d97dc0b3ba949fa8428d33aed36d251aefec9 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 30 Apr 2024 15:00:09 +0200
Subject: [PATCH 760/990] Fix test description

---
 tests/supervisor/test_views.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
index 58cad0d69..abd375be1 100644
--- a/tests/supervisor/test_views.py
+++ b/tests/supervisor/test_views.py
@@ -29,10 +29,9 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client):
 
 @pytest.mark.asyncio
 async def test_system_usage(aiohttp_client):
-    """Test that the allocation endpoint fails when an invalid item_hash is provided."""
+    """Test that the usage system endpoints responds. No auth needed"""
     app = setup_webapp()
     client = await aiohttp_client(app)
-    settings.ALLOCATION_TOKEN_HASH = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"  # = "test"
     response: web.Response = await client.get("/about/usage/system")
     assert response.status == 200
     # check if it is valid json

From 4725269902ac7995667bdc18983f58ef176b1a20 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 30 Apr 2024 15:41:43 +0200
Subject: [PATCH 761/990] Better model real usage in Droplet test

---
 .github/workflows/test-on-droplets-matrix.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index f99f30a0b..cb47fbc31 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -238,11 +238,10 @@ jobs:
             -d '{"persistent_vms": [], "instances": ["${{ matrix.check_vm.item_hash }}"]}' \
             "http://${DROPLET_IPV4}:4020/control/allocations"
 
-      - name: Get system usage
+      - name: Fetch system usage endpoint
         run: |
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           curl -X GET -H "Content-Type: application/json" \
-            -H "X-Auth-Signature: test" \
             "http://${DROPLET_IPV4}:4020/about/usage/system"
 
 

From 8ac2c1bfd0f897243d3529d779185a15dc1fbeff Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 2 May 2024 11:29:03 +0200
Subject: [PATCH 762/990] Add a test with mock

---
 tests/supervisor/test_views.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
index abd375be1..15b0c995d 100644
--- a/tests/supervisor/test_views.py
+++ b/tests/supervisor/test_views.py
@@ -1,3 +1,4 @@
+from unittest import mock
 import pytest
 from aiohttp import web
 
@@ -38,3 +39,33 @@ async def test_system_usage(aiohttp_client):
     resp = await response.json()
     assert "cpu" in resp
     assert resp["cpu"]["count"] > 0
+
+
+@pytest.mark.asyncio
+async def test_system_usage_mock(aiohttp_client, mocker):
+    """Test that the usage system endpoints response value. No auth needed"""
+    mocker.patch(
+        "cpuinfo.cpuinfo.get_cpu_info",
+        {
+            "arch_string_raw": "x86_64",
+            "vendor_id_raw": "AuthenticAMD",
+        },
+    )
+    mocker.patch(
+        "psutil.getloadavg",
+        lambda: [1, 2, 3],
+    )
+    mocker.patch(
+        "psutil.cpu_count",
+        lambda: 200,
+    )
+    app = setup_webapp()
+    client = await aiohttp_client(app)
+    response: web.Response = await client.get("/about/usage/system")
+    assert response.status == 200
+    # check if it is valid json
+    resp = await response.json()
+    assert resp["properties"]["cpu"]["architecture"] == "x86_64"
+    assert resp["properties"]["cpu"]["vendor"] == "AuthenticAMD"
+    assert resp["cpu"]["load_average"] == {"load1": 1.0, "load15": 3.0, "load5": 2.0}
+    assert resp["cpu"]["count"] == 200

From 47f1ab57e8cb76130135ccee139352e3b1937f66 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 2 May 2024 11:37:03 +0200
Subject: [PATCH 763/990] isort

---
 tests/supervisor/test_views.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
index 15b0c995d..60fa9578d 100644
--- a/tests/supervisor/test_views.py
+++ b/tests/supervisor/test_views.py
@@ -1,4 +1,5 @@
 from unittest import mock
+
 import pytest
 from aiohttp import web
 

From 24f93391b97767e05c7ee7b2687406b4d9048231 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 30 Apr 2024 15:03:11 +0200
Subject: [PATCH 764/990] Problem: allocation endpoints was not tested

Solution: Start by adding some simple tests
We don't test the full allocation and deallocation here. just auth
---
 src/aleph/vm/orchestrator/views/__init__.py |  6 +++
 tests/supervisor/test_views.py              | 55 ++++++++++++++++++++-
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 177e6a348..1c614d428 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -341,6 +341,12 @@ def authenticate_api_request(request: web.Request) -> bool:
 
 
 async def update_allocations(request: web.Request):
+    """Main entry for the start of persistence VM and instance, called by the CCN,
+
+
+    auth via the SETTINGS.ALLOCATION_TOKEN_HASH  sent in header X-Auth-Signature.
+    Receive a list of vm and instance that should be present and then match that state by stopping and launching VMs
+    """
     if not authenticate_api_request(request):
         return web.HTTPUnauthorized(text="Authentication token received is invalid")
 
diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
index 60fa9578d..254e326df 100644
--- a/tests/supervisor/test_views.py
+++ b/tests/supervisor/test_views.py
@@ -1,5 +1,3 @@
-from unittest import mock
-
 import pytest
 from aiohttp import web
 
@@ -70,3 +68,56 @@ async def test_system_usage_mock(aiohttp_client, mocker):
     assert resp["properties"]["cpu"]["vendor"] == "AuthenticAMD"
     assert resp["cpu"]["load_average"] == {"load1": 1.0, "load15": 3.0, "load5": 2.0}
     assert resp["cpu"]["count"] == 200
+
+
+@pytest.mark.asyncio
+async def test_allocation_invalid_auth_token(aiohttp_client):
+    """Test that the allocation endpoint fails when an invalid auth token is provided."""
+    settings.ALLOCATION_TOKEN_HASH = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"  # = "test"
+    app = setup_webapp()
+    client = await aiohttp_client(app)
+    response = await client.post(
+        "/control/allocations",
+        json={"persistent_vms": []},
+        headers={"X-Auth-Signature": "notTest"},
+    )
+    assert response.status == 401
+    assert await response.text() == "Authentication token received is invalid"
+
+
+@pytest.mark.asyncio
+async def test_allocation_missing_auth_token(aiohttp_client):
+    """Test that the allocation endpoint fails when auth token is not provided."""
+    app = setup_webapp()
+    client = await aiohttp_client(app)
+    response: web.Response = await client.post(
+        "/control/allocations",
+        json={"persistent_vms": []},
+    )
+    assert response.status == 401
+    assert await response.text() == "Authentication token is missing"
+
+
+@pytest.mark.asyncio
+async def test_allocation_valid_token(aiohttp_client):
+    """Test that the allocation endpoint fails when an invalid auth is provided.
+
+    This is a very simple test that don't start or stop any VM so the mock is minimal"""
+
+    class FakeVmPool:
+        def get_persistent_executions(self):
+            return []
+
+    settings.ALLOCATION_TOKEN_HASH = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"  # = "test"
+    app = setup_webapp()
+    app["vm_pool"] = FakeVmPool()
+    app["pubsub"] = FakeVmPool()
+    client = await aiohttp_client(app)
+
+    response: web.Response = await client.post(
+        "/control/allocations",
+        json={"persistent_vms": []},
+        headers={"X-Auth-Signature": "test"},
+    )
+    assert response.status == 200
+    assert await response.json() == {"success": True, "successful": [], "failing": [], "errors": {}}

From 9bd3c0388b1564793648d9f6a7623a69ce5266a8 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 30 Apr 2024 17:10:19 +0200
Subject: [PATCH 765/990] Update docstring
 src/aleph/vm/orchestrator/views/__init__.py

Co-authored-by: nesitor 
---
 src/aleph/vm/orchestrator/views/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 1c614d428..4c729d6f2 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -341,7 +341,7 @@ def authenticate_api_request(request: web.Request) -> bool:
 
 
 async def update_allocations(request: web.Request):
-    """Main entry for the start of persistence VM and instance, called by the CCN,
+    """Main entry for the start of persistence VM and instance, called by the Scheduler,
 
 
     auth via the SETTINGS.ALLOCATION_TOKEN_HASH  sent in header X-Auth-Signature.

From ffa2322215c08e9a03af50d03d80b5a27916f2da Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 2 May 2024 16:40:00 +0200
Subject: [PATCH 766/990] Fix: Backquote in shell script executed command

When executed using `bash`, the `create_disk_image` was interrupted by a Python REPL due to the `python -OO` command being surrounded by backquotes.
---
 runtimes/aleph-debian-11-python/create_disk_image.sh | 2 +-
 runtimes/aleph-debian-12-python/create_disk_image.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
index bf05fbf48..2f426b903 100755
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-11-python/create_disk_image.sh
@@ -39,7 +39,7 @@ echo "Pip installing aleph-client"
 pip3 install 'aleph-sdk-python==0.7.0'
 
 # Compile Python code to bytecode for faster execution
-# -o2 is needed to compile with optimization level 2 which is what we launch init1.py (`python -OO`)
+# -o2 is needed to compile with optimization level 2 which is what we launch init1.py ("python -OO")
 # otherwise they are not used
 python3 -m compileall -o 2 -f /usr/local/lib/python3.9
 
diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index 78c96b897..bfaf050f1 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -39,7 +39,7 @@ mkdir -p /opt/aleph/libs
 pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.9.0' 'aleph-message==0.4.4' 'fastapi~=0.109.2'
 
 # Compile Python code to bytecode for faster execution
-# -o2 is needed to compile with optimization level 2 which is what we launch init1.py (`python -OO`)
+# -o2 is needed to compile with optimization level 2 which is what we launch init1.py ("python -OO")
 # otherwise they are not used
 python3 -m compileall -o 2 -f /usr/local/lib/python3.11
 python3 -m compileall -o 2 -f /opt/aleph/libs

From c942e27ef6d4de8dc2e3d2bc7e6b7986765467d1 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 3 May 2024 12:01:01 +0200
Subject: [PATCH 767/990] Fix: System testing on DO took too many resources

Problem: Running system testing on DigitalOcean for every push consumed a lot of resources and failed frequently.

We now start to have integration testing using `pytest`, which provides a better confidence that things actually work.

Solution: Only test on DO open Pull Requests and not every push. In the future, consider only running when merged on `main`.
---
 .github/workflows/test-on-droplets-matrix.yml | 113 +++---------------
 .github/workflows/test-using-pytest.yml       |  95 +++++++++++++++
 2 files changed, 113 insertions(+), 95 deletions(-)
 create mode 100644 .github/workflows/test-using-pytest.yml

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index cb47fbc31..4462e60b6 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -1,107 +1,30 @@
-name: "Test on DigitalOcean Droplets"
-
+# These are end-to-end tests running on ephemeral DigitalOcean "Droplet" virtual machines
+# with the different operating systems that are supported.
+#
+# The main focus of these tests is to ensure that the packaging works on all supported platforms
+# and to ensure the compatibility of dependencies (system and vendored) across these platforms.
+name: "Testing on DigitalOcean Droplets"
+
+# Run automatically on main branches, Pull Request updates and allow manual execution using `workflow_dispatch`.
 on:
-  push
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - "opened"
+      - "reopened"
+      - "synchronize"
+      - "ready_for_review"
+  workflow_dispatch:
 
 jobs:
-  tests-python:
-    name: "Test Python code"
-    runs-on: ubuntu-22.04
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
-        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
-
-      - name: Install required system packages only for Ubuntu Linux
-        run: |
-          sudo apt-get update
-          sudo apt-get -y upgrade
-          sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables
-          pip install --upgrade typing-extensions types-PyYAML
-
-      - name: Install required Python packages
-        run: |
-          python3 -m pip install hatch hatch-vcs coverage
-
-      - name: Test style wth ruff, black and isort
-        run: |
-          hatch run lint:style
-
-      - name: Test typing with Mypy
-        run: |
-          hatch run lint:typing
-
-      - name: Install required system packages for installing and running tests
-        run: |
-          sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
-
-      - name: Download and build required files for running tests. Copied from packaging/Makefile.
-        run: |
-          sudo mkdir --parents /opt/firecracker/
-          sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy"
-          
-          rm -fr /tmp/firecracker-release
-          mkdir --parents /tmp/firecracker-release /opt/firecracker
-          curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory /tmp/firecracker-release
-          # Copy binaries:
-          cp /tmp/firecracker-release/release-v*/firecracker-v*[!.debug] /opt/firecracker/firecracker
-          cp /tmp/firecracker-release/release-v*/jailer-v*[!.debug] /opt/firecracker/jailer
-          chmod +x /opt/firecracker/firecracker
-          chmod +x /opt/firecracker/jailer
-          
-          find /opt
-
-      - name: "Build custom runtime"
-        run: |
-          sudo apt update
-          sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86
-          cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
-
-      - name: "Build example volume"
-        run: |
-          cd examples/volumes && bash build_squashfs.sh
-
-        # Unit tests create and delete network interfaces, and therefore require to run as root
-      - name: Run unit tests
-        run: |
-          sudo python3 -m pip install hatch hatch-vcs coverage
-          sudo hatch run testing:cov
-
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4.0.1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          slug: aleph-im/aleph-vm
-
-  code-quality-shell:
-    runs-on: ubuntu-22.04
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
-        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
-
-      - name: Install required system packages only for Ubuntu Linux
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y shellcheck
-
-      - name: Run Shellcheck on all shell scripts
-        run: |
-          find ./ -type f -name "*.sh" -exec shellcheck {} \;
-  
 
   run_on_droplet:
     name: "Test Droplet with ${{ matrix.os_config.os_name }}-${{ matrix.check_vm.alias }}"
     runs-on: ubuntu-latest
     concurrency: "${{ matrix.os_config.concurrency_group }}-${{ matrix.check_vm.alias }}"
     timeout-minutes: 10
-    needs:
-      - tests-python
-      - code-quality-shell
 
     strategy:
       matrix:
diff --git a/.github/workflows/test-using-pytest.yml b/.github/workflows/test-using-pytest.yml
new file mode 100644
index 000000000..ba77cea22
--- /dev/null
+++ b/.github/workflows/test-using-pytest.yml
@@ -0,0 +1,95 @@
+name: "Test on DigitalOcean Droplets"
+
+on:
+  push
+
+jobs:
+  tests-python:
+    name: "Test Python code"
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
+      - name: Install required system packages only for Ubuntu Linux
+        run: |
+          sudo apt-get update
+          sudo apt-get -y upgrade
+          sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables
+          pip install --upgrade typing-extensions types-PyYAML
+
+      - name: Install required Python packages
+        run: |
+          python3 -m pip install hatch hatch-vcs coverage
+
+      - name: Test style wth ruff, black and isort
+        run: |
+          hatch run lint:style
+
+      - name: Test typing with Mypy
+        run: |
+          hatch run lint:typing
+
+      - name: Install required system packages for installing and running tests
+        run: |
+          sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
+
+      - name: Download and build required files for running tests. Copied from packaging/Makefile.
+        run: |
+          sudo mkdir --parents /opt/firecracker/
+          sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy"
+          
+          rm -fr /tmp/firecracker-release
+          mkdir --parents /tmp/firecracker-release /opt/firecracker
+          curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory /tmp/firecracker-release
+          # Copy binaries:
+          cp /tmp/firecracker-release/release-v*/firecracker-v*[!.debug] /opt/firecracker/firecracker
+          cp /tmp/firecracker-release/release-v*/jailer-v*[!.debug] /opt/firecracker/jailer
+          chmod +x /opt/firecracker/firecracker
+          chmod +x /opt/firecracker/jailer
+          
+          find /opt
+
+      - name: "Build custom runtime"
+        run: |
+          sudo apt update
+          sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86
+          cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
+
+      - name: "Build example volume"
+        run: |
+          cd examples/volumes && bash build_squashfs.sh
+
+        # Unit tests create and delete network interfaces, and therefore require to run as root
+      - name: Run unit tests
+        run: |
+          sudo python3 -m pip install hatch hatch-vcs coverage
+          sudo hatch run testing:cov
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4.0.1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: aleph-im/aleph-vm
+
+  code-quality-shell:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
+      - name: Install required system packages only for Ubuntu Linux
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Run Shellcheck on all shell scripts
+        run: |
+          find ./ -type f -name "*.sh" -exec shellcheck {} \;
+  

From d26888f8a3b99c51b95a954eef1c17c9b1619a82 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Tue, 7 May 2024 17:37:03 +0200
Subject: [PATCH 768/990] Added Qemu automatic tests (#615)

* Feature: Added automatic tests to check if QEmu runs.

* Fix: Added code quality fixes.

* Fix: Changed runtime generation script name.

* Fix: Solve conflicts with main branch.
---
 .github/workflows/test-using-pytest.yml       |   3 +-
 pyproject.toml                                |   6 +-
 .../create-ubuntu-22-04-qemu-disk.sh          |  18 ++
 src/aleph/vm/conf.py                          |   4 +-
 src/aleph/vm/controllers/__main__.py          |  20 +-
 src/aleph/vm/controllers/qemu/__init__.py     |   3 +
 src/aleph/vm/models.py                        |  21 ++-
 tests/supervisor/test_qemu_instance.py        | 176 ++++++++++++++++++
 8 files changed, 231 insertions(+), 20 deletions(-)
 create mode 100755 runtimes/instance-rootfs/create-ubuntu-22-04-qemu-disk.sh
 create mode 100644 tests/supervisor/test_qemu_instance.py

diff --git a/.github/workflows/test-using-pytest.yml b/.github/workflows/test-using-pytest.yml
index ba77cea22..732a646ea 100644
--- a/.github/workflows/test-using-pytest.yml
+++ b/.github/workflows/test-using-pytest.yml
@@ -53,11 +53,12 @@ jobs:
           
           find /opt
 
-      - name: "Build custom runtime"
+      - name: "Build custom runtimes"
         run: |
           sudo apt update
           sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86
           cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
+          cd runtimes/instance-rootfs && sudo ./create-ubuntu-22-04-qemu-disk.sh && cd ../..
 
       - name: "Build example volume"
         run: |
diff --git a/pyproject.toml b/pyproject.toml
index cd803673e..b5806b106 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,14 +24,14 @@ classifiers = [
   "Topic :: System :: Distributed Computing",
 ]
 dependencies = [
-  "pydantic[dotenv]==1.10.13",
+  "pydantic[dotenv]~=1.10.13",
   "aiohttp==3.8.6",
   "aiodns==3.1.0",
   "setproctitle==1.3.3",
   "pyyaml==6.0.1",
   "aleph-message==0.4.4",
   "jwskate==0.8.0",
-  "eth-account==0.9.0",
+  "eth-account~=0.10",
   "sentry-sdk==1.31.0",
   "aioredis==1.3.1",
   "psutil==5.9.5",
@@ -140,7 +140,7 @@ pythonpath = [
 testpaths = [
   "tests"
 ]
-ignore = [
+norecursedirs = [
   "runtimes/aleph-debian-11-python/rootfs/",
   "runtimes/aleph-debian-12-python/rootfs/",
 ]
diff --git a/runtimes/instance-rootfs/create-ubuntu-22-04-qemu-disk.sh b/runtimes/instance-rootfs/create-ubuntu-22-04-qemu-disk.sh
new file mode 100755
index 000000000..71738ba77
--- /dev/null
+++ b/runtimes/instance-rootfs/create-ubuntu-22-04-qemu-disk.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euf
+
+# Variables
+ROOTFS_FILENAME="./rootfs.img"
+IMAGE_URL="https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64-disk-kvm.img"
+IMAGE_NAME="jammy-server-cloudimg-amd64-disk-kvm.img"
+
+# Cleanup previous run
+rm -f "$ROOTFS_FILENAME"
+
+# Download Ubuntu image
+echo "Downloading Ubuntu 22.04 image"
+curl -L "$IMAGE_URL" -o "$IMAGE_NAME"
+
+# Rename final file
+mv "$IMAGE_NAME" "$ROOTFS_FILENAME"
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index e84c58c31..0d22625e8 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -281,13 +281,15 @@ class Settings(BaseSettings):
     )
 
     USE_FAKE_INSTANCE_BASE = False
-    FAKE_INSTANCE_BASE = Path(abspath(join(__file__, "../../runtimes/instance-debian-rootfs/rootfs.ext4")))
+    FAKE_INSTANCE_BASE = Path(abspath(join(__file__, "../../runtimes/instance-rootfs/rootfs.ext4")))
+    FAKE_QEMU_INSTANCE_BASE = Path(abspath(join(__file__, "../../../../runtimes/instance-rootfs/rootfs.img")))
     FAKE_INSTANCE_ID: str = Field(
         default="decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca",
         description="Identifier used for the 'fake instance' message defined in "
         "examples/instance_message_from_aleph.json",
     )
     FAKE_INSTANCE_MESSAGE = Path(abspath(join(__file__, "../../../../examples/instance_message_from_aleph.json")))
+    FAKE_INSTANCE_QEMU_MESSAGE = Path(abspath(join(__file__, "../../../../examples/qemu_message_from_aleph.json")))
 
     CHECK_FASTAPI_VM_ID = "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace"
     LEGACY_CHECK_FASTAPI_VM_ID = "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index 533b0a7a5..39d606784 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -4,7 +4,9 @@
 import logging
 import signal
 import sys
+from asyncio.subprocess import Process
 from pathlib import Path
+from typing import Union
 
 from aleph.vm.hypervisors.firecracker.microvm import MicroVM
 from aleph.vm.hypervisors.qemu.qemuvm import QemuVM
@@ -54,7 +56,7 @@ def parse_args(args):
     return parser.parse_args(args)
 
 
-async def run_persistent_vm(config: Configuration):
+async def execute_persistent_vm(config: Configuration):
     if config.hypervisor == HypervisorType.firecracker:
         assert isinstance(config.vm_configuration, VMConfiguration)
         execution = MicroVM(
@@ -73,9 +75,13 @@ async def run_persistent_vm(config: Configuration):
         execution = QemuVM(config.vm_configuration)
         process = await execution.start()
 
-        # Catch the terminating signal and send a proper message to the vm to stop it so it close files properly
-        loop = asyncio.get_event_loop()
-        loop.add_signal_handler(signal.SIGTERM, execution.send_shutdown_message)
+    return execution, process
+
+
+async def handle_persistent_vm(config: Configuration, execution: Union[MicroVM, QemuVM], process: Process):
+    # Catch the terminating signal and send a proper message to the vm to stop it so it close files properly
+    loop = asyncio.get_event_loop()
+    loop.add_signal_handler(signal.SIGTERM, execution.send_shutdown_message)
 
     if config.settings.PRINT_SYSTEM_LOGS:
         execution.start_printing_logs()
@@ -83,7 +89,11 @@ async def run_persistent_vm(config: Configuration):
     await process.wait()
     logger.info(f"Process terminated with {process.returncode}")
 
-    return execution
+
+async def run_persistent_vm(config: Configuration):
+    execution, process = await execute_persistent_vm(config)
+    await handle_persistent_vm(config=config, execution=execution, process=process)
+    return execution, process
 
 
 def main():
diff --git a/src/aleph/vm/controllers/qemu/__init__.py b/src/aleph/vm/controllers/qemu/__init__.py
index e69de29bb..eb9414917 100644
--- a/src/aleph/vm/controllers/qemu/__init__.py
+++ b/src/aleph/vm/controllers/qemu/__init__.py
@@ -0,0 +1,3 @@
+from .instance import AlephQemuInstance
+
+__all__ = "AlephQemuInstance"
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 9d38eea97..5a44c132a 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -23,6 +23,7 @@
     AlephFirecrackerResources,
     AlephProgramResources,
 )
+from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
 from aleph.vm.controllers.interface import AlephVmControllerInterface
 from aleph.vm.controllers.qemu.instance import AlephQemuInstance, AlephQemuResources
 from aleph.vm.network.interfaces import TapInterface
@@ -34,12 +35,9 @@
 )
 from aleph.vm.orchestrator.pubsub import PubSub
 from aleph.vm.orchestrator.vm import AlephFirecrackerInstance
+from aleph.vm.systemd import SystemDManager
 from aleph.vm.utils import create_task_log_exceptions, dumps_for_json
 
-if TYPE_CHECKING:
-    from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
-    from aleph.vm.systemd import SystemDManager
-
 logger = logging.getLogger(__name__)
 
 
@@ -81,14 +79,17 @@ class VmExecution:
     expire_task: Optional[asyncio.Task] = None
     update_task: Optional[asyncio.Task] = None
 
+    snapshot_manager: Optional[SnapshotManager]
+    systemd_manager: Optional[SystemDManager]
+
     persistent: bool = False
 
     @property
     def is_running(self) -> bool:
         return (
-            bool(self.times.starting_at and not self.times.stopping_at)
-            if not self.persistent
-            else self.systemd_manager.is_service_active(self.controller_service)
+            self.systemd_manager.is_service_active(self.controller_service)
+            if self.persistent and self.systemd_manager
+            else bool(self.times.starting_at and not self.times.stopping_at)
         )
 
     @property
@@ -141,8 +142,8 @@ def __init__(
         vm_hash: ItemHash,
         message: ExecutableContent,
         original: ExecutableContent,
-        snapshot_manager: "SnapshotManager",
-        systemd_manager: "SystemDManager",
+        snapshot_manager: Optional[SnapshotManager],
+        systemd_manager: Optional[SystemDManager],
         persistent: bool,
     ):
         self.uuid = uuid.uuid1()  # uuid1() includes the hardware address and timestamp
@@ -322,7 +323,7 @@ async def stop(self) -> None:
             self.cancel_expiration()
             self.cancel_update()
 
-            if self.vm.support_snapshot:
+            if self.vm.support_snapshot and self.snapshot_manager:
                 await self.snapshot_manager.stop_for(self.vm_hash)
             self.stop_event.set()
 
diff --git a/tests/supervisor/test_qemu_instance.py b/tests/supervisor/test_qemu_instance.py
new file mode 100644
index 000000000..3792aaa87
--- /dev/null
+++ b/tests/supervisor/test_qemu_instance.py
@@ -0,0 +1,176 @@
+import asyncio
+import logging
+from asyncio.subprocess import Process
+from pathlib import Path
+from typing import Optional
+
+import pytest
+from aleph_message.models import ItemHash
+
+from aleph.vm.conf import settings
+from aleph.vm.controllers.__main__ import configuration_from_file, execute_persistent_vm
+from aleph.vm.controllers.qemu import AlephQemuInstance
+from aleph.vm.hypervisors.qemu.qemuvm import QemuVM
+from aleph.vm.models import VmExecution
+from aleph.vm.network.hostnetwork import Network, make_ipv6_allocator
+from aleph.vm.orchestrator import metrics
+from aleph.vm.storage import get_message
+from aleph.vm.systemd import SystemDManager
+from aleph.vm.vm_type import VmType
+
+
+@pytest.mark.asyncio
+class MockSystemDManager(SystemDManager):
+    execution: Optional[QemuVM] = None
+    process: Optional[Process] = None
+
+    async def enable_and_start(self, vm_hash: str):
+        config_path = Path(f"{settings.EXECUTION_ROOT}/{vm_hash}-controller.json")
+        config = configuration_from_file(config_path)
+        self.execution, self.process = await execute_persistent_vm(config)
+        return self.execution, self.process
+
+    def is_service_enabled(self, service: str):
+        return self.process is not None
+
+    def is_service_active(self, service: str):
+        return self.process is not None
+
+    async def stop_and_disable(self, vm_hash: str):
+        if self.process:
+            self.process.kill()
+        self.process = None
+        self.execution = None
+        return self.execution, self.process
+
+
+@pytest.mark.asyncio
+async def test_create_qemu_instance():
+    """
+    Create an instance and check that it start / init / stop properly.
+    """
+
+    settings.USE_FAKE_INSTANCE_BASE = True
+    settings.FAKE_INSTANCE_MESSAGE = settings.FAKE_INSTANCE_QEMU_MESSAGE
+    settings.FAKE_INSTANCE_BASE = settings.FAKE_QEMU_INSTANCE_BASE
+    settings.ALLOW_VM_NETWORKING = False
+    settings.USE_JAILER = False
+
+    logging.basicConfig(level=logging.DEBUG)
+    settings.PRINT_SYSTEM_LOGS = True
+
+    # Ensure that the settings are correct and required files present.
+    settings.setup()
+    settings.check()
+
+    # The database is required for the metrics and is currently not optional.
+    engine = metrics.setup_engine()
+    await metrics.create_tables(engine)
+
+    vm_hash = ItemHash(settings.FAKE_INSTANCE_ID)
+    message = await get_message(ref=vm_hash)
+
+    mock_systemd_manager = MockSystemDManager()
+
+    execution = VmExecution(
+        vm_hash=vm_hash,
+        message=message.content,
+        original=message.content,
+        snapshot_manager=None,
+        systemd_manager=None,
+        persistent=True,
+    )
+
+    await asyncio.wait_for(execution.prepare(), timeout=60)
+    vm_id = 3
+
+    vm = execution.create(vm_id=vm_id, tap_interface=None)
+
+    # Test that the VM is created correctly. It is not started yet.
+    assert isinstance(vm, AlephQemuInstance)
+    assert vm.vm_id == vm_id
+
+    await execution.start()
+    qemu_execution, process = await mock_systemd_manager.enable_and_start(execution.vm_hash)
+    assert isinstance(qemu_execution, QemuVM)
+    assert qemu_execution.qemu_process is not None
+    qemu_execution, process = await mock_systemd_manager.stop_and_disable(execution.vm_hash)
+    await execution.stop()
+    assert qemu_execution is None
+
+
+@pytest.mark.asyncio
+async def test_create_qemu_instance_online():
+    """
+    Create an instance and check that it start / init / stop properly.
+    """
+
+    settings.USE_FAKE_INSTANCE_BASE = True
+    settings.FAKE_INSTANCE_MESSAGE = settings.FAKE_INSTANCE_QEMU_MESSAGE
+    settings.FAKE_INSTANCE_BASE = settings.FAKE_QEMU_INSTANCE_BASE
+    settings.ALLOW_VM_NETWORKING = True
+    settings.USE_JAILER = False
+
+    logging.basicConfig(level=logging.DEBUG)
+    settings.PRINT_SYSTEM_LOGS = True
+
+    # Ensure that the settings are correct and required files present.
+    settings.setup()
+    settings.check()
+
+    # The database is required for the metrics and is currently not optional.
+    engine = metrics.setup_engine()
+    await metrics.create_tables(engine)
+
+    vm_hash = ItemHash(settings.FAKE_INSTANCE_ID)
+    message = await get_message(ref=vm_hash)
+
+    mock_systemd_manager = MockSystemDManager()
+
+    network = (
+        Network(
+            vm_ipv4_address_pool_range=settings.IPV4_ADDRESS_POOL,
+            vm_network_size=settings.IPV4_NETWORK_PREFIX_LENGTH,
+            external_interface=settings.NETWORK_INTERFACE,
+            ipv6_allocator=make_ipv6_allocator(
+                allocation_policy=settings.IPV6_ALLOCATION_POLICY,
+                address_pool=settings.IPV6_ADDRESS_POOL,
+                subnet_prefix=settings.IPV6_SUBNET_PREFIX,
+            ),
+            use_ndp_proxy=False,
+            ipv6_forwarding_enabled=False,
+        )
+        if settings.ALLOW_VM_NETWORKING
+        else None
+    )
+
+    execution = VmExecution(
+        vm_hash=vm_hash,
+        message=message.content,
+        original=message.content,
+        snapshot_manager=None,
+        systemd_manager=None,
+        persistent=True,
+    )
+
+    await asyncio.wait_for(execution.prepare(), timeout=60)
+    vm_id = 3
+
+    vm_type = VmType.from_message_content(message.content)
+    tap_interface = await network.prepare_tap(vm_id, vm_hash, vm_type)
+    await network.create_tap(vm_id, tap_interface)
+
+    vm = execution.create(vm_id=vm_id, tap_interface=tap_interface)
+
+    # Test that the VM is created correctly. It is not started yet.
+    assert isinstance(vm, AlephQemuInstance)
+    assert vm.vm_id == vm_id
+
+    await execution.start()
+    qemu_execution, process = await mock_systemd_manager.enable_and_start(execution.vm_hash)
+    assert isinstance(qemu_execution, QemuVM)
+    assert qemu_execution.qemu_process is not None
+    await execution.wait_for_init()
+    qemu_execution, process = await mock_systemd_manager.stop_and_disable(execution.vm_hash)
+    await execution.stop()
+    assert qemu_execution is None

From f31140899e215c712a12d02d3c935a7843b7700f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 2 May 2024 17:07:20 +0200
Subject: [PATCH 769/990] Fix: Branch `main` could not be tested easily

This deploys the main branch automatically on the staging servers for system testing.
---
 .github/workflows/deploy-main-on-staging.yml | 62 ++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 .github/workflows/deploy-main-on-staging.yml

diff --git a/.github/workflows/deploy-main-on-staging.yml b/.github/workflows/deploy-main-on-staging.yml
new file mode 100644
index 000000000..51a9f43c3
--- /dev/null
+++ b/.github/workflows/deploy-main-on-staging.yml
@@ -0,0 +1,62 @@
+# This workflow automatically deploys main on staging
+name: "Deploy `main` automatically on staging"
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy_staging_servers:
+    name: "Deploying on ${{ matrix.staging_servers.hostname }}"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        staging_servers:
+          - hostname: "ovh.staging.aleph.sh"
+            # Use `ssh-keyscan -H host | base64 --wrap=0` to obtain the host keys
+            host_keys: "fDF8b3JHVkxyOU83Qnh0QmkvWjd4N0lVTkRDSHFRPXwxZEdZSnNjNlFyejA5QkR6cGROR1BLYjNES009IHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQmdRRHZwSmNpV2dscTNCbEsxY2xOUmNETnVQeFVCeGF3bE5qVElHZFV2MmoyTVo4KzliVVpDSkI1aXFIKzNvbkc5Vklla1RQdW1ybFlXbFMvZkkvSzM3dTh5UXJuQ3JkNi9XRm9XWWJOaTJ4NWxSOUhzaTViRXQ4MFAwRkFyVVpaSkdzbnRQdVhGeFJGR3dHeFNENTN2emVMbmU4VjRlaUxrQ3BjMDU5YzZVVHBublcvSjdRRnlzZURDUXIwVzFsMzBNcjlnTm1LbmpBd2VLWXdCS0hYaG42VGdSd1RYT1E3VXJCc3c2Q1d0OHI2N2g4QkJ2UHQ5OWt5OHl4dUw2Z25TRlhqeWhyKzVhd1lTY3VhVU5JS3B0Y2JFOWpISHhEY1FLSHN0akZZRHRsM0JWN29rUEkvUWJablpSMDVTdDgvZldoK2p5K3ZtR3BTWmtFckJ2NkUwdFhHMDhmdkdheVRXNWFVcWxRQmlLMzJpNmJlUWordjI3b0pUWndvcndBOVJCY1QramlCWVRNVUFpTTJrblhXMGlqT2ViWDNackpITm5QbXJkbjBTd1JldGlLRzg2SGdRK3d3a0dwd3UxVk01eTFlbTVwZ0VUdnU5SHg1RTFKeEJLcXJ3ZkdhTVVRWFZEWG8yNDg5bW1XZzA1aUFHejZIaXNTVWRESFlRKzhnWnA4PQp8MXxvUzkyc1NEb3RxU1hSb0F6MUpFS1V2RDhVTGM9fDVtSHZBSVdqbk1CN2IwcllRQlo0SXBpaFlqQT0gZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkZNanZFOEFsQmYxbkp1Y0ZlaEJjSUY2RE8wdGJOdU96OEx5QlFUdC82RlEwaWYyWVAxQUJ1TjBmYXVIT3R4WEx6b25vSGVhTDZaV0JoakhmRGV4NlY4PQp8MXxMc2lPc3RhVGk5bEhYSlFsWDJYQ3c3Q0lTU1k9fGk1RzlFTHJydHpaYkUrR2JjbWF1SDIxOG1ZND0gc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUp1QVNEMWY1d2dXM3pnd3FGalBXYzhPRi9BZ1pmSFFVa3lRMDE2c1MrRmoK"
+            os: "debian-12"
+            make_target: "all-podman-debian-12"
+            artifact_name: "aleph-vm.debian-12.deb"
+
+          - hostname: "hetzner.staging.aleph.sh"
+            # Use `ssh-keyscan -H host | base64 --wrap=0` to obtain the host keys
+            host_keys: "fDF8WUlKd0FYWnYxZ24vNkRCU0tkYjg0TC9sUngwPXwrRk96RzdoSTJ5Y3JzUW1uSEwrdEFBQkR4YUU9IHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQmdRRHBKcHF5ajUxWUluRkNyZjZUWjE5eUF3cHlXNTNHaFAxNXQ0Wm56cHBwOUVnNTNnZmVWdmk5WUV1bVV6cnVUN01LdFpobjNsb0U5YVFtRUYzSElpb3c5ZmlCWVA3aUMzUUFGdUJCandPUmQwV1RVWDZQQUN2c2p0b1JLWjJpTWZ2YXdITHdrWHErWnkrc2hHNU44L2pwQlJ4MC9paXJta2xPS0F5QWw0QTYzZ2MxMndsVGQzcS9IcDVxd1dSYVV3M1JVUTFTVVJSN2RGRW81VWxqeUZVYS9zdWV1STBYOXdLd0tPZ09iOEo3ZFZDMEdDT3VibkJXL3Jmb3N0YVV5eStaSzdQdzBrM251M2szTFZuUVlPTGlNOG1NMnJub2ZWZ2RSWXpiM3RTUVVrbk9wektBVzBXK3llWmpSOXp1UG4yNXF4bWxsRmRaNmt3QTFDcWY2MmQyQ0dtQ2NDU3dUSUl4ZHJ3M29oOEZOclpROTI4OGQvcmF4djZXZi9oZDI0Y1JqeDdFSEJxOUFWMW02UTZWeGxnSWl0WjIzODlsYjRmODNGclRrNUtib3J3Zm5oM3NaZFRSSkJqRjRhdHZ5NktsWFYxenROc05BeDhFN1RnTDMzVFlnRGc4RWlldGN1TVlzUlcwSnREdldBNGxsZDFQS3JrbDJ1LzZvLzNUb0xVPQp8MXxmQ3FnTjB2WHpMTnAzdklnZXdkSFRQRTA0ZUk9fDhnSituTC9hUGpEQlRMcUNJak1sZFpVbFRpST0gZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQktWbnE5aWsvcHZFaDdXbHFydUtWZmdZeTlwOVpNQnVKV2IrZkVvS0hZY0ZSYld5c0lYRjJlalBnaUMyOFEvZExqeUhXd2RVZlMySFBMbGNxRVFEZlpvPQp8MXxtVzA4T3ZqUnh0bmRjYVNyc0poWXBQcXp2akk9fFlDcktMeUg4ZnJJR0lRV05RS3hiUnArNlIvTT0gc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUl5ZGNhTXF1dkZFTEpNUDBlRmhNUGJWZVBSVjlSUEhVRzhIZGZIQmRvaTEK"
+            os: "debian-12"
+            make_target: "all-podman-debian-12"
+            artifact_name: "aleph-vm.debian-12.deb"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # Fetch the whole history for all tags and branches (required for aleph.__version__)
+          fetch-depth: 0
+
+      - run: |
+          cd packaging && make ${{ matrix.staging_servers.make_target }} && cd ..
+          ls packaging/target
+
+      - name: Setup SSH private key
+        run: |
+          mkdir ~/.ssh
+          echo $STAGING_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519
+          chmod 0700 ~/.ssh
+          chmod 0600 ~/.ssh/id_ed25519
+        env:
+          # Create using:
+          # ssh-keygen -t ed25519 -f ./id_ed25519
+          # cat ./id_ed25519 | base64 --wrap=0
+          STAGING_SSH_PRIVATE_KEY: ${{ secrets.STAGING_SSH_PRIVATE_KEY }}
+
+      - name: Install Aleph-VM on the Staging servers
+        run: |
+          echo ${{ matrix.staging_servers.host_keys }} | base64 --decode > ~/.ssh/known_hosts
+
+          # Wait for /var/lib/apt/lists/lock to be unlocked on the remote host via SSH.
+          while ssh root@${{ matrix.staging_servers.hostname }} lsof /var/lib/apt/lists/lock; do sleep 1; done
+
+          scp packaging/target/${{ matrix.staging_servers.artifact_name }} root@${{ matrix.staging_servers.hostname }}:/opt
+          ssh root@${{ matrix.staging_servers.hostname }} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y --allow-downgrades /opt/${{ matrix.staging_servers.artifact_name }}"
+

From cb0a9f9633e302e66e41477134d4c5403847c162 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 26 Apr 2024 12:52:34 +0200
Subject: [PATCH 770/990] Fix: Unkwnown hashes raised exception

Problem: Many crawlers called URLs that do not exist on CRNs.

The current implementation raises an error when the hash of the VM cannot be found, which fills the logs on Sentry.

Solution: Return an HTTP Not Found status instead.
---
 src/aleph/vm/orchestrator/views/__init__.py | 16 +++++--
 tests/supervisor/views/test_run_code.py     | 46 +++++++++++++++++++++
 2 files changed, 58 insertions(+), 4 deletions(-)
 create mode 100644 tests/supervisor/views/test_run_code.py

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 4c729d6f2..8b7702ba4 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -11,7 +11,7 @@
 import aiodns
 import aiohttp
 from aiohttp import web
-from aiohttp.web_exceptions import HTTPNotFound
+from aiohttp.web_exceptions import HTTPBadRequest, HTTPNotFound
 from aleph_message.exceptions import UnknownHashError
 from aleph_message.models import ItemHash, MessageType
 from pydantic import ValidationError
@@ -65,7 +65,13 @@ async def run_code_from_path(request: web.Request) -> web.Response:
     path = request.match_info["suffix"]
     path = path if path.startswith("/") else f"/{path}"
 
-    message_ref = ItemHash(request.match_info["ref"])
+    try:
+        message_ref = ItemHash(request.match_info["ref"])
+    except UnknownHashError as e:
+        raise HTTPBadRequest(
+            reason="Invalid message reference", text=f"Invalid message reference: {request.match_info['ref']}"
+        ) from e
+
     pool: VmPool = request.app["vm_pool"]
     return await run_code_on_request(message_ref, path, pool, request)
 
@@ -98,8 +104,10 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
             try:
                 message_ref = ItemHash(await get_ref_from_dns(domain=f"_aleph-id.{request.host}"))
                 logger.debug(f"Using DNS TXT record to obtain '{message_ref}'")
-            except aiodns.error.DNSError as error:
-                raise HTTPNotFound(reason="Invalid message reference") from error
+            except aiodns.error.DNSError:
+                return HTTPNotFound(reason="Invalid message reference")
+            except UnknownHashError:
+                return HTTPNotFound(reason="Invalid message reference")
 
     pool = request.app["vm_pool"]
     return await run_code_on_request(message_ref, path, pool, request)
diff --git a/tests/supervisor/views/test_run_code.py b/tests/supervisor/views/test_run_code.py
new file mode 100644
index 000000000..639a8f7bf
--- /dev/null
+++ b/tests/supervisor/views/test_run_code.py
@@ -0,0 +1,46 @@
+import pytest
+from aiohttp import ClientResponseError, web
+from aiohttp.test_utils import make_mocked_request
+from aiohttp.web_exceptions import HTTPBadRequest
+from aleph_message.exceptions import UnknownHashError
+from aleph_message.models import ItemHash
+
+from aleph.vm.conf import settings
+from aleph.vm.orchestrator.views import run_code_from_path
+
+
+@pytest.mark.asyncio
+async def test_run_code_from_invalid_path(aiohttp_client):
+    """
+    Test that the run_code_from_path endpoint raises the right
+    error on invalid paths.
+    """
+    item_hash = "invalid-item-hash"
+    with pytest.raises(UnknownHashError):
+        assert ItemHash(item_hash).is_storage(item_hash)
+
+    app = web.Application()
+
+    app.router.add_route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
+    client = await aiohttp_client(app)
+
+    invalid_hash_request: web.Request = make_mocked_request(
+        "GET",
+        "/vm/" + item_hash,
+        match_info={
+            "ref": item_hash,
+            "suffix": "/some/suffix",
+        },
+        headers={"Host": settings.DOMAIN_NAME},
+        app=app,
+    )
+    with pytest.raises(HTTPBadRequest):
+        await run_code_from_path(invalid_hash_request)
+
+    # Calling the view from an HTTP client should result in a Bad Request error.
+    resp = await client.get("/vm/" + item_hash + "/some/suffix")
+    assert resp.status == HTTPBadRequest.status_code
+    text = await resp.text()
+    assert text == f"Invalid message reference: {item_hash}"
+    with pytest.raises(ClientResponseError):
+        resp.raise_for_status()

From 888f523a559dbb031e3abbdd2f66788c07689e68 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 16 May 2024 10:47:19 +0200
Subject: [PATCH 771/990] Replace dep jwskate by jwcryto

Use the standard jwcrypto package (apt install python3-jwcrypto) instead of the jwskate wrapper for ECDSA verification.
To use in the auth code for checking the owner.

Co-authored-by: Bonjour Internet 
Co-authored-by: Hugo Herter 
---
 packaging/aleph-vm/DEBIAN/control             |  2 +-
 pyproject.toml                                |  2 +-
 .../vm/orchestrator/views/authentication.py   | 57 +++++++++-------
 tests/supervisor/test_jwk.py                  | 65 -------------------
 4 files changed, 36 insertions(+), 90 deletions(-)
 delete mode 100644 tests/supervisor/test_jwk.py

diff --git a/packaging/aleph-vm/DEBIAN/control b/packaging/aleph-vm/DEBIAN/control
index 45aa6bd65..6b42eea41 100644
--- a/packaging/aleph-vm/DEBIAN/control
+++ b/packaging/aleph-vm/DEBIAN/control
@@ -3,6 +3,6 @@ Version: 0.1.8
 Architecture: all
 Maintainer: Aleph.im
 Description: Aleph.im VM execution engine
-Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils,python3-systemd,python3-dbus,btrfs-progs,nftables
+Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils,python3-systemd,python3-dbus,btrfs-progs,nftables,python3-jwcrypto
 Section: aleph-im
 Priority: Extra
diff --git a/pyproject.toml b/pyproject.toml
index b5806b106..95ed874a5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,7 +30,6 @@ dependencies = [
   "setproctitle==1.3.3",
   "pyyaml==6.0.1",
   "aleph-message==0.4.4",
-  "jwskate==0.8.0",
   "eth-account~=0.10",
   "sentry-sdk==1.31.0",
   "aioredis==1.3.1",
@@ -51,6 +50,7 @@ dependencies = [
   "alembic==1.13.1",
   "aiohttp_cors~=0.7.0",
   "pyroute2==0.7.12",
+  "jwcrypto==1.5.6",
 ]
 
 [project.urls]
diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index d38587015..a50dde45e 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -1,15 +1,18 @@
+# Keep datetime import as is as it allow patching in test
+import datetime
 import functools
 import json
 import logging
 from collections.abc import Awaitable, Coroutine
-from datetime import datetime, timedelta, timezone
 from typing import Any, Callable, Literal, Union
 
+import cryptography.exceptions
 import pydantic
 from aiohttp import web
 from eth_account import Account
 from eth_account.messages import encode_defunct
-from jwskate import Jwk
+from jwcrypto import jwk, jws
+from jwcrypto.jwa import JWA
 from pydantic import BaseModel, ValidationError, root_validator, validator
 
 from aleph.vm.conf import settings
@@ -17,12 +20,12 @@
 logger = logging.getLogger(__name__)
 
 
-def is_token_still_valid(timestamp):
+def is_token_still_valid(datestr: str):
     """
     Checks if a token has expired based on its expiry timestamp
     """
-    current_datetime = datetime.now(tz=timezone.utc)
-    expiry_datetime = datetime.fromisoformat(timestamp)
+    current_datetime = datetime.datetime.now(tz=datetime.timezone.utc)
+    expiry_datetime = datetime.datetime.fromisoformat(datestr.replace("Z", "+00:00"))
 
     return expiry_datetime > current_datetime
 
@@ -48,9 +51,9 @@ class SignedPubKeyPayload(BaseModel):
     expires: str
 
     @property
-    def json_web_key(self) -> Jwk:
+    def json_web_key(self) -> jwk.JWK:
         """Return the ephemeral public key as Json Web Key"""
-        return Jwk(self.pubkey)
+        return jwk.JWK(**self.pubkey)
 
 
 class SignedPubKeyHeader(BaseModel):
@@ -95,16 +98,16 @@ def content(self) -> SignedPubKeyPayload:
 
 
 class SignedOperationPayload(BaseModel):
-    time: datetime
+    time: datetime.datetime
     method: Union[Literal["POST"], Literal["GET"]]
     path: str
     # body_sha256: str  # disabled since there is no body
 
     @validator("time")
-    def time_is_current(cls, v: datetime) -> datetime:
+    def time_is_current(cls, v: datetime.datetime) -> datetime.datetime:
         """Check that the time is current and the payload is not a replay attack."""
-        max_past = datetime.now(tz=timezone.utc) - timedelta(minutes=2)
-        max_future = datetime.now(tz=timezone.utc) + timedelta(minutes=2)
+        max_past = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(minutes=2)
+        max_future = datetime.datetime.now(tz=datetime.timezone.utc) + datetime.timedelta(minutes=2)
         if v < max_past:
             raise ValueError("Time is too far in the past")
         if v > max_future:
@@ -154,13 +157,17 @@ def get_signed_pubkey(request: web.Request) -> SignedPubKeyHeader:
         raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey fields") from error
     except json.JSONDecodeError as error:
         raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey format") from error
-    except ValueError as error:
-        if error.args == ("Token expired",):
-            raise web.HTTPUnauthorized(reason="Token expired") from error
-        elif error.args == ("Invalid signature",):
-            raise web.HTTPUnauthorized(reason="Invalid signature") from error
+    except ValueError as errors:
+        logging.debug(errors)
+        for err in errors.args[0]:
+            if isinstance(err.exc, json.JSONDecodeError):
+                raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey format") from errors
+            if str(err.exc) == "Token expired":
+                raise web.HTTPUnauthorized(reason="Token expired") from errors
+            if str(err.exc) == "Invalid signature":
+                raise web.HTTPUnauthorized(reason="Invalid signature") from errors
         else:
-            raise error
+            raise errors
 
 
 def get_signed_operation(request: web.Request) -> SignedOperation:
@@ -179,14 +186,14 @@ def get_signed_operation(request: web.Request) -> SignedOperation:
 
 def verify_signed_operation(signed_operation: SignedOperation, signed_pubkey: SignedPubKeyHeader) -> str:
     """Verify that the operation is signed by the ephemeral key authorized by the wallet."""
-    if signed_pubkey.content.json_web_key.verify(
-        data=signed_operation.payload,
-        signature=signed_operation.signature,
-        alg="ES256",
-    ):
+    pubkey = signed_pubkey.content.json_web_key
+
+    try:
+        JWA.signing_alg("ES256").verify(pubkey, signed_operation.payload, signed_operation.signature)
         logger.debug("Signature verified")
         return signed_pubkey.content.address
-    else:
+    except cryptography.exceptions.InvalidSignature as e:
+        logger.debug("Failing to validate signature for operation", e)
         raise web.HTTPUnauthorized(reason="Signature could not verified")
 
 
@@ -225,6 +232,10 @@ async def wrapper(request):
             authenticated_sender: str = await authenticate_jwk(request)
         except web.HTTPException as e:
             return web.json_response(data={"error": e.reason}, status=e.status)
+        except Exception as e:
+            # Unexpected make sure to log it
+            logging.exception(e)
+            raise
 
         response = await handler(request, authenticated_sender)
         return response
diff --git a/tests/supervisor/test_jwk.py b/tests/supervisor/test_jwk.py
deleted file mode 100644
index cc3b0ab09..000000000
--- a/tests/supervisor/test_jwk.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import os
-
-from aiohttp import web
-
-from aleph.vm.orchestrator.views.authentication import authenticate_jwk
-
-# Avoid failures linked to settings when initializing the global VmPool object
-os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
-
-from typing import Any
-
-import pytest
-
-
-@pytest.fixture
-def valid_jwk_headers(mocker):
-    mocker.patch("aleph.vm.orchestrator.views.authentication.is_token_still_valid", lambda timestamp: True)
-    return {
-        "X-SignedPubKey": '{"payload":"7b227075626b6579223a7b22616c67223a224553323536222c22637276223a22502d323536222c22657874223a747275652c226b65795f6f7073223a5b22766572696679225d2c226b7479223a224543222c2278223a224b65763844614d7356454673365a6b4679525a4272796344564138566a334f656e49756f34743561374634222c2279223a2279597343556d715978654767673643743736794f47525873545867446444795234644f5639514c6f6b6477227d2c22616c67223a224543445341222c22646f6d61696e223a226c6f63616c686f7374222c2261646472657373223a22307833343932346566393435623933316431653932393337353535366636396365326537666535646363222c2265787069726573223a313638393337353132342e3532317d","signature":"0x58e1498a6c4f88ac1982e7147ff49405ffe1b9633e048bb74cf741abb05ce0b63bb406f3079f641ae89f597654ecd2a704d37ffbf86a28e462140033cc0eedcb1c"}',
-        "X-SignedOperation": '{"time":"2023-07-14T22:14:14.132Z","signature":"96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0add"}',
-    }
-
-
-@pytest.mark.skip(reason="TODO: Fix this test")
-@pytest.mark.asyncio
-async def test_valid_signature(valid_jwk_headers: dict[str, Any], mocker):
-    request = mocker.AsyncMock()
-    request.headers = valid_jwk_headers
-    await authenticate_jwk(request)
-
-
-@pytest.mark.skip(reason="TODO: Fix this test")
-@pytest.mark.asyncio
-async def test_invalid_signature(valid_jwk_headers: dict[str, Any], mocker):
-    valid_jwk_headers["X-SignedOperation"] = (
-        '{"time":"2023-07-14T22:14:14.132Z","signature":"96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0ade"}'
-    )
-    request = mocker.AsyncMock()
-    request.headers = valid_jwk_headers
-
-    with pytest.raises(web.HTTPUnauthorized):
-        await authenticate_jwk(request)
-
-
-@pytest.mark.skip(reason="TODO: Fix this test")
-@pytest.mark.asyncio
-async def test_expired_token(valid_jwk_headers: dict[str, Any], mocker):
-    mocker.patch("aleph.vm.orchestrator.views.authentication.is_token_still_valid", lambda timestamp: False)
-    request = mocker.AsyncMock()
-    request.headers = valid_jwk_headers
-
-    with pytest.raises(web.HTTPUnauthorized):
-        await authenticate_jwk(request)
-
-
-@pytest.mark.parametrize("missing_header", ["X-SignedPubKey", "X-SignedOperation"])
-@pytest.mark.asyncio
-async def test_missing_headers(valid_jwk_headers: dict[str, Any], mocker, missing_header: str):
-    del valid_jwk_headers[missing_header]
-
-    request = mocker.AsyncMock()
-    request.headers = valid_jwk_headers
-
-    with pytest.raises(web.HTTPBadRequest):
-        await authenticate_jwk(request)

From 9fd2b150ae3442f19611441f73e16a09909798f3 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 16 May 2024 10:51:41 +0200
Subject: [PATCH 772/990] fix typo in doc

---
 src/aleph/vm/controllers/qemu/QEMU.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/controllers/qemu/QEMU.md b/src/aleph/vm/controllers/qemu/QEMU.md
index e79a6c8b6..66b63e75b 100644
--- a/src/aleph/vm/controllers/qemu/QEMU.md
+++ b/src/aleph/vm/controllers/qemu/QEMU.md
@@ -8,7 +8,7 @@ These are installable via
 
 This branch depends on the version 0.4.1 of `aleph-message` that add the `hypervisor` field. The easiest way is to install tha version using `pip install -e .`
 
-To create a local venv use the `--system-site-packages` option so it can acess nftables 
+To create a local venv use the `--system-site-packages` option so it can access nftables 
 
 ## To test launching a VM instance
 

From 143716390ccb48e890a297f58b1259279f9021eb Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 16 May 2024 10:51:54 +0200
Subject: [PATCH 773/990] Complete test suite for @require_jwk_authentication

Merged with previous test_jwk. Retook one of the old tests
---
 tests/supervisor/test_authentication.py | 322 ++++++++++++++++++++++++
 1 file changed, 322 insertions(+)
 create mode 100644 tests/supervisor/test_authentication.py

diff --git a/tests/supervisor/test_authentication.py b/tests/supervisor/test_authentication.py
new file mode 100644
index 000000000..1a1b76ed7
--- /dev/null
+++ b/tests/supervisor/test_authentication.py
@@ -0,0 +1,322 @@
+import datetime
+import json
+from typing import Any
+
+import eth_account.messages
+import pytest
+from aiohttp import web
+from eth_account.datastructures import SignedMessage
+from jwcrypto import jwk, jws
+from jwcrypto.common import base64url_decode
+from jwcrypto.jwa import JWA
+
+from aleph.vm.orchestrator.views.authentication import (
+    authenticate_jwk,
+    require_jwk_authentication,
+)
+
+
+@pytest.mark.asyncio
+async def test_require_jwk_authentication_missing_header(aiohttp_client):
+    """An HTTP request to a view decorated by `@require_jwk_authentication` must return an error
+    with a status code 400 and an error message in JSON when no authentication is provided.
+    """
+    app = web.Application()
+
+    @require_jwk_authentication
+    async def view(request, authenticated_sender):
+        return web.Response(text="ok")
+
+    app.router.add_get("", view)
+    client = await aiohttp_client(app)
+    resp = await client.get("/")
+    assert resp.status == 400
+
+    r = await resp.json()
+    assert {"error": "Missing X-SignedPubKey header"} == r
+
+
+@pytest.mark.asyncio
+async def test_require_jwk_authentication_invalid_json_bugkey(aiohttp_client):
+    """An HTTP request to a view decorated by `@require_jwk_authentication`  must return an error
+    with a status code 400 and an error message in JSON when the authentication key format is invalid.
+    """
+
+    app = web.Application()
+
+    @require_jwk_authentication
+    async def view(request, authenticated_sender):
+        return web.Response(text="ok")
+
+    app.router.add_get("", view)
+    client = await aiohttp_client(app)
+    resp = await client.get("/", headers={"X-SignedPubKey": "invalid_json"})
+    assert resp.status == 400
+
+    r = await resp.json()
+    assert {"error": "Invalid X-SignedPubKey format"} == r
+
+
+@pytest.fixture
+def patch_datetime_now(monkeypatch):
+    """Fixture for patching the datetime.now() and datetime.utcnow() methods
+    to return a fixed datetime object.
+    This fixture creates a subclass of `datetime.datetime` called `mydatetime`,
+    which overrides the `now()` and `utcnow()` class methods to return a fixed
+    datetime object specified by `FAKE_TIME`.
+    """
+
+    class MockDateTime(datetime.datetime):
+        FAKE_TIME = datetime.datetime(2010, 12, 25, 17, 5, 55)
+
+        @classmethod
+        def now(cls, tz=None, *args, **kwargs):
+            return cls.FAKE_TIME.replace(tzinfo=tz)
+
+        @classmethod
+        def utcnow(cls, *args, **kwargs):
+            return cls.FAKE_TIME
+
+    monkeypatch.setattr(datetime, "datetime", MockDateTime)
+    return MockDateTime
+
+
+@pytest.mark.asyncio
+async def test_require_jwk_authentication_expired(aiohttp_client):
+    app = web.Application()
+    account = eth_account.Account()
+    signer_account = account.create()
+    key = jwk.JWK.generate(
+        kty="EC",
+        crv="P-256",
+        # key_ops=["verify"],
+    )
+
+    pubkey = {
+        "pubkey": json.loads(key.export_public()),
+        "alg": "ECDSA",
+        "domain": "localhost",
+        "address": signer_account.address,
+        "expires": "2023-05-02T10:44:42.754994Z",
+    }
+    pubkey_payload = json.dumps(pubkey).encode("utf-8").hex()
+    signable_message = eth_account.messages.encode_defunct(hexstr=pubkey_payload)
+    signed_message: SignedMessage = signer_account.sign_message(signable_message)
+    pubkey_signature = signed_message.signature.to_0x_hex()
+
+    pubkey_signature_header = json.dumps(
+        {
+            "payload": pubkey_payload,
+            "signature": pubkey_signature,
+        }
+    )
+
+    @require_jwk_authentication
+    async def view(request, authenticated_sender):
+        return web.Response(text="ok")
+
+    app.router.add_get("", view)
+    client = await aiohttp_client(app)
+
+    resp = await client.get("/", headers={"X-SignedPubKey": pubkey_signature_header})
+    assert resp.status == 401
+
+    r = await resp.json()
+    assert {"error": "Token expired"} == r
+
+
+@pytest.mark.asyncio
+async def test_require_jwk_authentication_wrong_key(aiohttp_client, patch_datetime_now):
+    app = web.Application()
+
+    @require_jwk_authentication
+    async def view(request, authenticated_sender):
+        return web.Response(text="ok")
+
+    app.router.add_get("", view)
+    client = await aiohttp_client(app)
+    headers = {
+        "X-SignedPubKey": (
+            json.dumps(
+                {
+                    "payload": "7b227075626b6579223a207b22637276223a2022502d323536222c20226b7479223a20224543222c202278223a202273765759314e5652614a683231527834576a765f67657057772d714d436f774d76304a52353057327a7545222c202279223a2022794950424d6135474e7a49555878656c513762415a5f437776303875763448774d4c49456c656c43534473227d2c2022616c67223a20224543445341222c2022646f6d61696e223a20226c6f63616c686f7374222c202261646472657373223a2022307842323564623537643234304438353132366262364234384661633635343837323161343537343538222c202265787069726573223a2022323032332d30352d30325431303a34343a34322e3735343939345a227d",
+                    "signature": "0x58e1498a6c4f88ac1982e7147ff49405ffe1b9633e048bb74cf741abb05ce0b63bb406f3079f641ae89f597654ecd2a704d37ffbf86a28e462140033cc0eedcb1c",
+                }
+            )
+        )
+    }
+    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/"}
+    headers["X-SignedOperation"] = json.dumps(
+        {
+            "payload": bytes.hex(json.dumps(payload).encode("utf-8")),
+            "signature": "96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0ade",
+        }
+    )
+
+    resp = await client.get("/", headers=headers)
+    assert resp.status == 401, resp.text()
+
+    r = await resp.json()
+    assert {"error": "Invalid signature"} == r
+
+
+@pytest.mark.asyncio
+async def test_require_jwk_eth_signature_dont_match(aiohttp_client, patch_datetime_now):
+    app = web.Application()
+
+    @require_jwk_authentication
+    async def view(request, authenticated_sender):
+        return web.Response(text="ok")
+
+    account = eth_account.Account()
+    signer_account = account.create()
+    key = jwk.JWK.generate(
+        kty="EC",
+        crv="P-256",
+        # key_ops=["verify"],
+    )
+
+    pubkey = {
+        "pubkey": json.loads(key.export_public()),
+        "alg": "ECDSA",
+        "domain": "localhost",
+        "address": signer_account.address,
+        "expires": "2023-05-02T10:44:42.754994Z",
+    }
+    pubkey_payload = json.dumps(pubkey).encode("utf-8").hex()
+    signable_message = eth_account.messages.encode_defunct(hexstr=pubkey_payload)
+    signed_message: SignedMessage = signer_account.sign_message(signable_message)
+    pubkey_signature = signed_message.signature.to_0x_hex()
+
+    # Modify the payload to render the signature invalid
+    pubkey["domain"] = "baddomain"
+    invalid_pubkey_payload = json.dumps(pubkey).encode("utf-8").hex()
+
+    app.router.add_get("", view)
+    client = await aiohttp_client(app)
+    headers = {
+        "X-SignedPubKey": (
+            json.dumps(
+                {
+                    "payload": invalid_pubkey_payload,
+                    "signature": pubkey_signature,
+                }
+            )
+        )
+    }
+    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/"}
+    headers["X-SignedOperation"] = json.dumps(
+        {
+            "payload": bytes.hex(json.dumps(payload).encode("utf-8")),
+            "signature": "96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0ade",
+        }
+    )
+
+    resp = await client.get("/", headers=headers)
+    assert resp.status == 401, resp.text()
+
+    r = await resp.json()
+    assert {"error": "Invalid signature"} == r
+
+
+@pytest.mark.asyncio
+async def test_jwk():
+    payload = "abc123"
+    key = jwk.JWK.generate(
+        kty="EC",
+        crv="P-256",
+    )
+    pubkey = json.loads(key.export_public())
+    jws_signer = jws.JWSCore(alg="ES256", key=key, payload=payload, header=None)
+    signature_and_payload_json_dict = jws_signer.sign()
+    signature = base64url_decode(signature_and_payload_json_dict["signature"])
+
+    # Verify signature
+    pub_jwk = jws.JWK(**pubkey)
+    jws_verifier = jws.JWSCore(
+        alg="ES256",
+        key=pub_jwk,
+        payload=payload,
+        header=None,
+    )
+    assert jws_verifier.verify(signature=signature)
+
+
+@pytest.mark.asyncio
+async def test_require_jwk_authentication_good_key(aiohttp_client, patch_datetime_now):
+    """An HTTP request to a view decorated by `@require_jwk_authentication`
+    auth correctly a temporary key signed by a wallet and an operation signed by that key"""
+    app = web.Application()
+
+    account = eth_account.Account()
+    signer_account = account.create()
+    key = jwk.JWK.generate(
+        kty="EC",
+        crv="P-256",
+        # key_ops=["verify"],
+    )
+
+    pubkey = {
+        "pubkey": json.loads(key.export_public()),
+        "alg": "ECDSA",
+        "domain": "localhost",
+        "address": signer_account.address,
+        "expires": (patch_datetime_now.FAKE_TIME + datetime.timedelta(days=1)).isoformat() + "Z",
+    }
+    pubkey_payload = json.dumps(pubkey).encode("utf-8").hex()
+    signable_message = eth_account.messages.encode_defunct(hexstr=pubkey_payload)
+    signed_message: SignedMessage = signer_account.sign_message(signable_message)
+    pubkey_signature = signed_message.signature.to_0x_hex()
+    pubkey_signature_header = json.dumps(
+        {
+            "payload": pubkey_payload,
+            "signature": pubkey_signature,
+        }
+    )
+
+    @require_jwk_authentication
+    async def view(request, authenticated_sender):
+        assert authenticated_sender == signer_account.address
+        return web.Response(text="ok")
+
+    app.router.add_get("", view)
+    client = await aiohttp_client(app)
+
+    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/"}
+
+    payload_as_bytes = json.dumps(payload).encode("utf-8")
+    headers = {"X-SignedPubKey": pubkey_signature_header}
+    payload_signature = JWA.signing_alg("ES256").sign(key, payload_as_bytes)
+    headers["X-SignedOperation"] = json.dumps(
+        {
+            "payload": payload_as_bytes.hex(),
+            "signature": payload_signature.hex(),
+        }
+    )
+
+    resp = await client.get("/", headers=headers)
+    assert resp.status == 200, resp.text()
+
+    r = await resp.text()
+    assert "ok" == r
+
+
+@pytest.fixture
+def valid_jwk_headers(mocker):
+    mocker.patch("aleph.vm.orchestrator.views.authentication.is_token_still_valid", lambda timestamp: True)
+    return {
+        "X-SignedPubKey": '{"payload":"7b227075626b6579223a7b22616c67223a224553323536222c22637276223a22502d323536222c22657874223a747275652c226b65795f6f7073223a5b22766572696679225d2c226b7479223a224543222c2278223a224b65763844614d7356454673365a6b4679525a4272796344564138566a334f656e49756f34743561374634222c2279223a2279597343556d715978654767673643743736794f47525873545867446444795234644f5639514c6f6b6477227d2c22616c67223a224543445341222c22646f6d61696e223a226c6f63616c686f7374222c2261646472657373223a22307833343932346566393435623933316431653932393337353535366636396365326537666535646363222c2265787069726573223a313638393337353132342e3532317d","signature":"0x58e1498a6c4f88ac1982e7147ff49405ffe1b9633e048bb74cf741abb05ce0b63bb406f3079f641ae89f597654ecd2a704d37ffbf86a28e462140033cc0eedcb1c"}',
+        "X-SignedOperation": '{"time":"2023-07-14T22:14:14.132Z","signature":"96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0add"}',
+    }
+
+
+@pytest.mark.parametrize("missing_header", ["X-SignedPubKey", "X-SignedOperation"])
+@pytest.mark.asyncio
+async def test_missing_headers(valid_jwk_headers: dict[str, Any], mocker, missing_header: str):
+    del valid_jwk_headers[missing_header]
+
+    request = mocker.AsyncMock()
+    request.headers = valid_jwk_headers
+
+    with pytest.raises(web.HTTPBadRequest):
+        await authenticate_jwk(request)

From 7b89523a7290720782fcfc1b55f61d647881acf1 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 16 May 2024 11:28:29 +0200
Subject: [PATCH 774/990] Problem : different version of hexbytes were behaving
 differently

was causing compat problem with other package
Solution :reimplement their hex method to ensure it works on all version
restore eth-account version that was reverted by error
---
 tests/supervisor/test_authentication.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/tests/supervisor/test_authentication.py b/tests/supervisor/test_authentication.py
index 1a1b76ed7..249806f01 100644
--- a/tests/supervisor/test_authentication.py
+++ b/tests/supervisor/test_authentication.py
@@ -16,6 +16,16 @@
 )
 
 
+def to_0x_hex(b: bytes) -> str:
+    """
+    Convert the bytes to a 0x-prefixed hex string
+    """
+
+    # force this for compat between different hexbytes versions which behave differenty
+    # and conflict with other package don't allow us to have the version we want
+    return "0x" + bytes.hex(b)
+
+
 @pytest.mark.asyncio
 async def test_require_jwk_authentication_missing_header(aiohttp_client):
     """An HTTP request to a view decorated by `@require_jwk_authentication` must return an error
@@ -102,7 +112,7 @@ async def test_require_jwk_authentication_expired(aiohttp_client):
     pubkey_payload = json.dumps(pubkey).encode("utf-8").hex()
     signable_message = eth_account.messages.encode_defunct(hexstr=pubkey_payload)
     signed_message: SignedMessage = signer_account.sign_message(signable_message)
-    pubkey_signature = signed_message.signature.to_0x_hex()
+    pubkey_signature = to_0x_hex(signed_message.signature)
 
     pubkey_signature_header = json.dumps(
         {
@@ -154,7 +164,7 @@ async def view(request, authenticated_sender):
     )
 
     resp = await client.get("/", headers=headers)
-    assert resp.status == 401, resp.text()
+    assert resp.status == 401, await resp.text()
 
     r = await resp.json()
     assert {"error": "Invalid signature"} == r
@@ -186,7 +196,7 @@ async def view(request, authenticated_sender):
     pubkey_payload = json.dumps(pubkey).encode("utf-8").hex()
     signable_message = eth_account.messages.encode_defunct(hexstr=pubkey_payload)
     signed_message: SignedMessage = signer_account.sign_message(signable_message)
-    pubkey_signature = signed_message.signature.to_0x_hex()
+    pubkey_signature = to_0x_hex(signed_message.signature)
 
     # Modify the payload to render the signature invalid
     pubkey["domain"] = "baddomain"
@@ -213,7 +223,7 @@ async def view(request, authenticated_sender):
     )
 
     resp = await client.get("/", headers=headers)
-    assert resp.status == 401, resp.text()
+    assert resp.status == 401, await resp.text()
 
     r = await resp.json()
     assert {"error": "Invalid signature"} == r
@@ -266,7 +276,7 @@ async def test_require_jwk_authentication_good_key(aiohttp_client, patch_datetim
     pubkey_payload = json.dumps(pubkey).encode("utf-8").hex()
     signable_message = eth_account.messages.encode_defunct(hexstr=pubkey_payload)
     signed_message: SignedMessage = signer_account.sign_message(signable_message)
-    pubkey_signature = signed_message.signature.to_0x_hex()
+    pubkey_signature = to_0x_hex(signed_message.signature)
     pubkey_signature_header = json.dumps(
         {
             "payload": pubkey_payload,
@@ -295,7 +305,7 @@ async def view(request, authenticated_sender):
     )
 
     resp = await client.get("/", headers=headers)
-    assert resp.status == 200, resp.text()
+    assert resp.status == 200, await resp.text()
 
     r = await resp.text()
     assert "ok" == r

From 75f0742c39d1e11cbf32132e62fe04ae4b75def7 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Wed, 22 May 2024 11:09:32 +0200
Subject: [PATCH 775/990] Added `USE_CONFIDENTIAL_COMPUTING` check (#617)

* Problem: A node operator cannot check or add support for confidential computing.

Solution: Implemented a setting to allow node operator to enable confidential computing. A check ensure that the system is well configured, and it shows that configuration on the /public/config endpoint.

* Fix: Solved PR comments

* Fix: Solved code quality issues.

* Fix: Changed test's mocks after method refactor.

* Fix: Solved linting issue.
---
 src/aleph/vm/conf.py                        | 19 +++++++++++++++++--
 src/aleph/vm/orchestrator/views/__init__.py |  6 ++++++
 src/aleph/vm/utils.py                       |  7 +++++++
 tests/supervisor/test_utils.py              | 19 +++++++++++++++++++
 4 files changed, 49 insertions(+), 2 deletions(-)
 create mode 100644 tests/supervisor/test_utils.py

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 0d22625e8..aa5016aa8 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -16,7 +16,7 @@
 from pydantic.env_settings import DotenvType, env_file_sentinel
 from pydantic.typing import StrPath
 
-from aleph.vm.utils import file_hashes_differ, is_command_available
+from aleph.vm.utils import check_system_module, file_hashes_differ, is_command_available
 
 logger = logging.getLogger(__name__)
 
@@ -261,6 +261,12 @@ class Settings(BaseSettings):
         description="Default hypervisor to use on running instances, can be Firecracker or QEmu",
     )
 
+    ENABLE_CONFIDENTIAL_COMPUTING: bool = Field(
+        default=False,
+        description="Enable Confidential Computing using AMD-SEV. It will test if the host is compatible "
+        "with SEV and SEV-ES",
+    )
+
     # Tests on programs
 
     FAKE_DATA_PROGRAM: Optional[Path] = None
@@ -364,13 +370,22 @@ def check(self):
             "cloud-localds"
         ), "Command `cloud-localds` not found, run `apt install cloud-image-utils`"
 
-        if settings.ENABLE_QEMU_SUPPORT:
+        if self.ENABLE_QEMU_SUPPORT:
             # Qemu support
             assert is_command_available("qemu-img"), "Command `qemu-img` not found, run `apt install qemu-utils`"
             assert is_command_available(
                 "qemu-system-x86_64"
             ), "Command `qemu-system-x86_64` not found, run `apt install qemu-system-x86`"
 
+        if self.ENABLE_CONFIDENTIAL_COMPUTING:
+            assert check_system_module("kvm_amd/parameters/sev") == "Y", "SEV feature isn't enabled, enable it in BIOS"
+            assert (
+                check_system_module("kvm_amd/parameters/sev_es") == "Y"
+            ), "SEV-ES feature isn't enabled, enable it in BIOS"
+
+            assert self.ENABLE_QEMU_SUPPORT, "Qemu Support is needed for confidential computing and it's disabled, "
+            "enable it setting the env variable `ENABLE_QEMU_SUPPORT=True` in configuration"
+
     def setup(self):
         """Setup the environment defined by the settings. Call this method after loading the settings."""
         os.makedirs(self.MESSAGE_CACHE, exist_ok=True)
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 8b7702ba4..31198676c 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -331,6 +331,12 @@ async def status_public_config(request: web.Request):
                 "PAYMENT_RECEIVER_ADDRESS": settings.PAYMENT_RECEIVER_ADDRESS,
                 "PAYMENT_SUPER_TOKEN": settings.PAYMENT_SUPER_TOKEN,
                 "PAYMENT_CHAIN_ID": settings.PAYMENT_CHAIN_ID,
+                "PAYMENT_MONITOR_INTERVAL": settings.PAYMENT_MONITOR_INTERVAL,
+            },
+            "computing": {
+                "ENABLE_QEMU_SUPPORT": settings.ENABLE_QEMU_SUPPORT,
+                "INSTANCE_DEFAULT_HYPERVISOR": settings.INSTANCE_DEFAULT_HYPERVISOR,
+                "ENABLE_CONFIDENTIAL_COMPUTING": settings.ENABLE_CONFIDENTIAL_COMPUTING,
             },
         },
         dumps=dumps_for_json,
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index 63ce18253..296af0c58 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -130,6 +130,13 @@ def is_command_available(command):
         return False
 
 
+def check_system_module(module_path: str) -> str:
+    p = Path("/sys/module") / module_path
+    if not p.exists():
+        return ""
+    return p.open().read().strip()
+
+
 def fix_message_validation(message: dict) -> dict:
     """Patch a fake message program to pass validation."""
     message["item_content"] = json.dumps(message["content"])
diff --git a/tests/supervisor/test_utils.py b/tests/supervisor/test_utils.py
new file mode 100644
index 000000000..8b67fe1ef
--- /dev/null
+++ b/tests/supervisor/test_utils.py
@@ -0,0 +1,19 @@
+from unittest import mock
+
+from aleph.vm.utils import check_system_module
+
+
+def test_check_system_module_enabled():
+
+    with mock.patch(
+        "pathlib.Path.exists",
+        return_value=True,
+    ):
+        expected_value = "Y"
+        with mock.patch(
+            "pathlib.Path.open",
+            mock.mock_open(read_data=expected_value),
+        ):
+
+            output = check_system_module("kvm_amd/parameters/sev_enp")
+            assert output == expected_value

From cc3f292f373dfd0f39084d1f8a47b5059493a893 Mon Sep 17 00:00:00 2001
From: Antony JIN <91880456+Antonyjin@users.noreply.github.com>
Date: Mon, 27 May 2024 14:24:54 +0200
Subject: [PATCH 776/990] 601 creating instance tests (#616)

* Feature: Added automatic tests to check if QEmu runs.

* Fix: Added code quality fixes.

* Testing basic instance
Adding a test that start and stop a simple fake instance

* Adding a test to create a simple instance
Creating an instance locally using fake_data and checks if the execution start
and stop properly.
Doing the same checks with the VM created.

* Fix: Black test did not pass
Black check was failing

Solution: Use black to format bad files

* Fix: isort test did not pass

isort check was failing

Solution: Use isort to format bad files

* Added Qemu automatic tests (#615)

* Feature: Added automatic tests to check if QEmu runs.

* Fix: Added code quality fixes.

* Fix: Changed runtime generation script name.

* Fix: Solve conflicts with main branch.

* Fix: ruff test did not pass

ruff check was failing

Solution: Use ruff to format bad files

* Fix: Branch `main` could not be tested easily

This deploys the main branch automatically on the staging servers for system testing.

* Fix: Unkwnown hashes raised exception

Problem: Many crawlers called URLs that do not exist on CRNs.

The current implementation raises an error when the hash of the VM cannot be found, which fills the logs on Sentry.

Solution: Return an HTTP Not Found status instead.

* Fix: Mypy was failing
mypy was failing because I tried to stop an execution but did not
checked if there was an existing execution

Solution: Check first if there an execution and then stop and shutdown

* Fix: Wrong image name inside the workflow

* Fix: Wrong debian executable name inside workflow

* Feature: Added automatic tests to check if QEmu runs.

* Fix: Added code quality fixes.

* Rebasing from main

* Testing basic instance
Adding a test that start and stop a simple fake instance

* Adding a test to create a simple instance
Creating an instance locally using fake_data and checks if the execution start
and stop properly.
Doing the same checks with the VM created.

* Fix: Black test did not pass
Black check was failing

Solution: Use black to format bad files

* Fix: isort test did not pass

isort check was failing

Solution: Use isort to format bad files

* Fix: ruff test did not pass

ruff check was failing

Solution: Use ruff to format bad files

* Fix: Mypy was failing
mypy was failing because I tried to stop an execution but did not
checked if there was an existing execution

Solution: Check first if there an execution and then stop and shutdown

* Rebasing from main

* Fix: Incorrect path given in the workflow

* Fix: Test use unknown user
Instance test try to chmod a user named jailman that
did not exist

Solution: Creating the user jailman

* unit test job cancelled, push again to check it

* Fix: Job to create user jailman now is inside another job
This job to create a new user do not need hes own job, we could
just insert it in a previous job

---------

Co-authored-by: Andres D. Molins 
Co-authored-by: ajin 
Co-authored-by: Hugo Herter 
---
 .github/workflows/test-on-droplets-matrix.yml |   5 +-
 .github/workflows/test-using-pytest.yml       |  10 +-
 examples/instance_message_from_aleph.json     |   7 -
 src/aleph/vm/conf.py                          |   2 +-
 tests/supervisor/test_instance.py             | 125 ++++++++++++++++++
 5 files changed, 134 insertions(+), 15 deletions(-)
 create mode 100644 tests/supervisor/test_instance.py

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index 4462e60b6..1493af67e 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -19,7 +19,6 @@ on:
   workflow_dispatch:
 
 jobs:
-
   run_on_droplet:
     name: "Test Droplet with ${{ matrix.os_config.os_name }}-${{ matrix.check_vm.alias }}"
     runs-on: ubuntu-latest
@@ -114,10 +113,10 @@ jobs:
         run: |
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
-          
+
           # Wait a few seconds for DigitalOcean to setup the Droplet using apt, which conflicts with our comands:
           sleep 5
-          
+
           # Wait for /var/lib/apt/lists/lock to be unlocked on the remote host via SSH.
           while ssh root@${DROPLET_IPV4} lsof /var/lib/apt/lists/lock; do sleep 1; done
 
diff --git a/.github/workflows/test-using-pytest.yml b/.github/workflows/test-using-pytest.yml
index 732a646ea..e618ab71f 100644
--- a/.github/workflows/test-using-pytest.yml
+++ b/.github/workflows/test-using-pytest.yml
@@ -39,9 +39,10 @@ jobs:
 
       - name: Download and build required files for running tests. Copied from packaging/Makefile.
         run: |
+          sudo useradd jailman
           sudo mkdir --parents /opt/firecracker/
           sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy"
-          
+
           rm -fr /tmp/firecracker-release
           mkdir --parents /tmp/firecracker-release /opt/firecracker
           curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory /tmp/firecracker-release
@@ -50,7 +51,7 @@ jobs:
           cp /tmp/firecracker-release/release-v*/jailer-v*[!.debug] /opt/firecracker/jailer
           chmod +x /opt/firecracker/firecracker
           chmod +x /opt/firecracker/jailer
-          
+
           find /opt
 
       - name: "Build custom runtimes"
@@ -59,12 +60,13 @@ jobs:
           sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86
           cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
           cd runtimes/instance-rootfs && sudo ./create-ubuntu-22-04-qemu-disk.sh && cd ../..
+          cd runtimes/instance-rootfs && sudo ./create-debian-12-disk.sh && cd ../..
 
       - name: "Build example volume"
         run: |
           cd examples/volumes && bash build_squashfs.sh
-
-        # Unit tests create and delete network interfaces, and therefore require to run as root
+        
+      # Unit tests create and delete network interfaces, and therefore require to run as root
       - name: Run unit tests
         run: |
           sudo python3 -m pip install hatch hatch-vcs coverage
diff --git a/examples/instance_message_from_aleph.json b/examples/instance_message_from_aleph.json
index c2218fbaa..a4da80c2b 100644
--- a/examples/instance_message_from_aleph.json
+++ b/examples/instance_message_from_aleph.json
@@ -38,13 +38,6 @@
                 "mount": "/opt/venv",
                 "ref": "5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51",
                 "use_latest": false
-            },
-            {
-                "comment": "Working data persisted on the VM supervisor, not available on other nodes",
-                "mount": "/var/lib/example",
-                "name": "data",
-                "persistence": "host",
-                "size_mib": 5
             }
         ],
         "replaces": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba",
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index aa5016aa8..ef69ffb14 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -287,7 +287,7 @@ class Settings(BaseSettings):
     )
 
     USE_FAKE_INSTANCE_BASE = False
-    FAKE_INSTANCE_BASE = Path(abspath(join(__file__, "../../runtimes/instance-rootfs/rootfs.ext4")))
+    FAKE_INSTANCE_BASE = Path(abspath(join(__file__, "../../../../runtimes/instance-rootfs/debian-12.btrfs")))
     FAKE_QEMU_INSTANCE_BASE = Path(abspath(join(__file__, "../../../../runtimes/instance-rootfs/rootfs.img")))
     FAKE_INSTANCE_ID: str = Field(
         default="decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca",
diff --git a/tests/supervisor/test_instance.py b/tests/supervisor/test_instance.py
new file mode 100644
index 000000000..54c26aee7
--- /dev/null
+++ b/tests/supervisor/test_instance.py
@@ -0,0 +1,125 @@
+import asyncio
+import logging
+from asyncio.subprocess import Process
+from pathlib import Path
+from typing import Optional
+
+import pytest
+from aleph_message.models import ItemHash
+
+from aleph.vm.conf import settings
+from aleph.vm.controllers.__main__ import configuration_from_file, execute_persistent_vm
+from aleph.vm.controllers.firecracker import AlephFirecrackerInstance
+from aleph.vm.hypervisors.firecracker.microvm import MicroVM
+from aleph.vm.models import VmExecution
+from aleph.vm.network.hostnetwork import Network, make_ipv6_allocator
+from aleph.vm.orchestrator import metrics
+from aleph.vm.storage import get_message
+from aleph.vm.systemd import SystemDManager
+from aleph.vm.vm_type import VmType
+
+
+@pytest.mark.asyncio
+class MockSystemDManager(SystemDManager):
+    execution: Optional[MicroVM] = None
+    process: Optional[Process] = None
+
+    async def enable_and_start(self, vm_hash: str):
+        config_path = Path(f"{settings.EXECUTION_ROOT}/{vm_hash}-controller.json")
+        config = configuration_from_file(config_path)
+        self.execution, self.process = await execute_persistent_vm(config)
+        return self.execution, self.process
+
+    def is_service_enabled(self, service: str):
+        return self.process is not None
+
+    def is_service_active(self, service: str):
+        return self.process is not None
+
+    async def stop_and_disable(self, vm_hash: str):
+        if self.execution:
+            await self.execution.shutdown()
+            await self.execution.stop()
+        self.process = None
+        self.execution = None
+        return self.execution, self.process
+
+
+@pytest.mark.asyncio
+async def test_create_instance():
+    """
+    Create a fake instance locally and check that it start / init / stop properly.
+    """
+
+    settings.USE_FAKE_INSTANCE_BASE = True
+    settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM
+    # settings.FAKE_INSTANCE_MESSAGE
+    settings.ALLOW_VM_NETWORKING = True
+    settings.USE_JAILER = True
+
+    logging.basicConfig(level=logging.DEBUG)
+    settings.PRINT_SYSTEM_LOGS = True
+
+    # Ensure that the settings are correct and required files present.
+    settings.setup()
+    settings.check()
+
+    # The database is required for the metrics and is currently not optional.
+    engine = metrics.setup_engine()
+    await metrics.create_tables(engine)
+
+    vm_hash = ItemHash(settings.FAKE_INSTANCE_ID)
+    message = await get_message(ref=vm_hash)
+
+    mock_systemd_manager = MockSystemDManager()
+
+    # Creating a Network to initialize the tap_interface that is needed for the creation of an instance
+    network = Network(
+        vm_ipv4_address_pool_range=settings.IPV4_ADDRESS_POOL,
+        vm_network_size=settings.IPV4_NETWORK_PREFIX_LENGTH,
+        external_interface=settings.NETWORK_INTERFACE,
+        ipv6_allocator=make_ipv6_allocator(
+            allocation_policy=settings.IPV6_ALLOCATION_POLICY,
+            address_pool=settings.IPV6_ADDRESS_POOL,
+            subnet_prefix=settings.IPV6_SUBNET_PREFIX,
+        ),
+        use_ndp_proxy=False,
+        ipv6_forwarding_enabled=False,
+    )
+
+    execution = VmExecution(
+        vm_hash=vm_hash,
+        message=message.content,
+        original=message.content,
+        snapshot_manager=None,
+        systemd_manager=None,
+        persistent=True,
+    )
+
+    # Downloading the resources required may take some time, limit it to 10 seconds
+    await asyncio.wait_for(execution.prepare(), timeout=30)
+
+    vm_id = 3
+    vm_type = VmType.from_message_content(message.content)
+    tap_interface = await network.prepare_tap(vm_id, vm_hash, vm_type)
+    await network.create_tap(vm_id, tap_interface)
+
+    vm = execution.create(vm_id=vm_id, tap_interface=tap_interface)
+
+    # Test that the VM is created correctly. It is not started yet.
+    assert isinstance(vm, AlephFirecrackerInstance)
+    assert vm.vm_id == vm_id
+    assert vm.persistent
+    assert vm.enable_networking
+
+    await execution.start()
+    firecracker_execution, process = await mock_systemd_manager.enable_and_start(execution.vm_hash)
+    assert isinstance(firecracker_execution, MicroVM)
+    assert firecracker_execution.proc is not None
+    await execution.wait_for_init()
+
+    # This sleep is to leave the instance to boot up and prevent disk corruption
+    await asyncio.sleep(60)
+    firecracker_execution, process = await mock_systemd_manager.stop_and_disable(execution.vm_hash)
+    await execution.stop()
+    assert firecracker_execution is None

From 2c2d3f8108f9c7b7150104f538cbe70194fae041 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 29 May 2024 19:06:19 +0200
Subject: [PATCH 777/990] Fix: File was only closed lazily by garbage collector

Problem: Reading a file using `f.open().read()` keeps the file open until the garbage collector deletes the variable.

Since this uses Pathlib already, using `Path(...).read_text()` is cleaner and does not depend on the opening mode being `r`.

A file with no content could not be distinguished from a missing file. Instead, this changes the behaviour to return None when the file is missing instead of an empty string.
---
 src/aleph/vm/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils.py
index 296af0c58..6c114253e 100644
--- a/src/aleph/vm/utils.py
+++ b/src/aleph/vm/utils.py
@@ -130,11 +130,11 @@ def is_command_available(command):
         return False
 
 
-def check_system_module(module_path: str) -> str:
+def check_system_module(module_path: str) -> Optional[str]:
     p = Path("/sys/module") / module_path
     if not p.exists():
-        return ""
-    return p.open().read().strip()
+        return None
+    return p.read_text().strip()
 
 
 def fix_message_validation(message: dict) -> dict:

From b63d24823dde3d8589b0bdad7b453abf81a3d48d Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 30 May 2024 16:41:35 +0200
Subject: [PATCH 778/990] Fix: Some dependencies were inconsistent between
 pyproject.toml and packaging

---
 packaging/Makefile | 2 +-
 pyproject.toml     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 43d8a0017..ca19d6e9a 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.4' 'jwskate==0.8.0' 'eth-account==0.9.0' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.4' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo
diff --git a/pyproject.toml b/pyproject.toml
index 95ed874a5..123e07c91 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,7 +40,7 @@ dependencies = [
   "msgpack==1.0.7",
   "packaging==23.2",
   "jsonschema==4.19.1",
-  "qmp==0.0.1",
+  "qmp==1.1.0",
   "dbus-python==1.3.2",
   "systemd-python==235",
   "systemd-python==235",

From df26f8bb4b99b53139df737b26791651d18c7426 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 30 May 2024 16:40:36 +0200
Subject: [PATCH 779/990] Fix: No .deb was built for Ubuntu 24.04

This adds support in the packaging/Makefile and in the CI to build packages for Ubuntu 24.04
---
 .github/workflows/build-deb-package.yml |  3 +++
 packaging/Makefile                      | 20 ++++++++++++++++++++
 packaging/ubuntu-24.04.dockerfile       | 16 ++++++++++++++++
 3 files changed, 39 insertions(+)
 create mode 100644 packaging/ubuntu-24.04.dockerfile

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 825116667..0c99d47e3 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -19,6 +19,9 @@ jobs:
           - os: "ubuntu-22.04"
             make_target: "all-podman-ubuntu-2204"
             artifact_name: "aleph-vm.ubuntu-22.04.deb"
+          - os: "ubuntu-24.04"
+            make_target: "all-podman-ubuntu-2404"
+            artifact_name: "aleph-vm.ubuntu-24.04.deb"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
diff --git a/packaging/Makefile b/packaging/Makefile
index ca19d6e9a..fffb158dc 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -93,6 +93,17 @@ all-podman-ubuntu-2204: version
 	file target/aleph-vm.deb
 	mv target/aleph-vm.deb target/aleph-vm.ubuntu-22.04.deb
 
+all-podman-ubuntu-2404: version
+	cd .. && podman build -t localhost/aleph-vm-packaging-ubuntu-2404:latest -f ./packaging/ubuntu-24.04.dockerfile .
+	mkdir -p ./target
+	podman run --rm -ti \
+		-w /opt/packaging \
+		-v ./target:/opt/packaging/target \
+		localhost/aleph-vm-packaging-ubuntu-2404:latest \
+		make
+	file target/aleph-vm.deb
+	mv target/aleph-vm.deb target/aleph-vm.ubuntu-24.04.deb
+
 # extract Python requirements from Debian 11 container
 requirements-debian-11: all-podman-debian-11
 	podman run --rm -ti \
@@ -120,6 +131,15 @@ requirements-ubuntu-22.04: all-podman-ubuntu-2204
 		ubuntu:jammy \
 		bash -c "/opt/extract_requirements.sh /mnt/requirements-ubuntu-22.04.txt"
 
+# extract Python requirements from Ubuntu 24.04 container
+requirements-ubuntu-24.04: all-podman-ubuntu-2404
+	podman run --rm -ti \
+		-v ./target/aleph-vm.ubuntu-24.04.deb:/opt/packaging/target/aleph-vm.deb:ro \
+		-v ./extract_requirements.sh:/opt/extract_requirements.sh:ro \
+		-v ./requirements-ubuntu-24.04.txt:/mnt/requirements-ubuntu-24.04.txt \
+		ubuntu:noble \
+		bash -c "/opt/extract_requirements.sh /mnt/requirements-ubuntu-24.04.txt"
+
 # run on host in order to sign with GPG
 repository-bullseye:
 	cd ./repositories/bullseye && reprepro -Vb . includedeb bullseye ../../target/aleph-vm.debian-11.deb && cd ..
diff --git a/packaging/ubuntu-24.04.dockerfile b/packaging/ubuntu-24.04.dockerfile
new file mode 100644
index 000000000..8f892e746
--- /dev/null
+++ b/packaging/ubuntu-24.04.dockerfile
@@ -0,0 +1,16 @@
+FROM ubuntu:24.04
+
+RUN apt-get update && apt-get -y upgrade && apt-get install -y \
+    make \
+    git \
+    curl \
+    sudo \
+    python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt
+COPY ../src/aleph ./src/aleph
+COPY ../packaging ./packaging
+COPY ../kernels ./kernels
+
+COPY ../examples/ ./examples

From b1ca017df0aa42177d2def8ff77bbd8db21758d0 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Wed, 5 Jun 2024 10:04:00 +0200
Subject: [PATCH 780/990] Add platform confidential directory on Settings
 (#618)

Problem: The server don't have a directory to save the platform certificates generated by sevctl.

Solution: Set that directory field on settings class and ensure to create the folder on initialization step.
---
 src/aleph/vm/conf.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index ef69ffb14..17e1d23e7 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -267,6 +267,11 @@ class Settings(BaseSettings):
         "with SEV and SEV-ES",
     )
 
+    CONFIDENTIAL_DIRECTORY: Path = Field(
+        None,
+        description="Confidential Computing default directory. Default to EXECUTION_ROOT/confidential",
+    )
+
     # Tests on programs
 
     FAKE_DATA_PROGRAM: Optional[Path] = None
@@ -409,6 +414,7 @@ def setup(self):
 
         os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True)
         os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True)
+        os.makedirs(self.CONFIDENTIAL_DIRECTORY, exist_ok=True)
 
         self.API_SERVER = self.API_SERVER.rstrip("/")
 
@@ -467,6 +473,8 @@ def __init__(
             self.EXECUTION_LOG_DIRECTORY = self.EXECUTION_ROOT / "executions"
         if not self.JAILER_BASE_DIR:
             self.JAILER_BASE_DIR = self.EXECUTION_ROOT / "jailer"
+        if not self.CONFIDENTIAL_DIRECTORY:
+            self.CONFIDENTIAL_DIRECTORY = self.CACHE_ROOT / "confidential"
 
     class Config:
         env_prefix = "ALEPH_VM_"

From a4e26a83788b1babc779957f4bc11cf9b06a5ce4 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Thu, 6 Jun 2024 13:30:04 +0200
Subject: [PATCH 781/990] Implement get platform certificates endpoint (#619)

Problem: There was no endpoint to get the confidential platform certificates

These are required in order to start the VM key exchange.

Solution: Create that endpoint and return the platform certificates generated by the `sevctl` command.
---
 src/aleph/vm/conf.py                    |  1 +
 src/aleph/vm/orchestrator/resources.py  | 13 +++++++
 src/aleph/vm/orchestrator/supervisor.py | 10 ++++-
 src/aleph/vm/sevclient.py               | 23 ++++++++++++
 tests/supervisor/test_views.py          | 50 +++++++++++++++++++++++++
 5 files changed, 96 insertions(+), 1 deletion(-)
 create mode 100644 src/aleph/vm/sevclient.py

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 17e1d23e7..f2c11a989 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -387,6 +387,7 @@ def check(self):
             assert (
                 check_system_module("kvm_amd/parameters/sev_es") == "Y"
             ), "SEV-ES feature isn't enabled, enable it in BIOS"
+            assert is_command_available("sevctl"), "Command `sevctl` not found, run `cargo install sevctl`"
 
             assert self.ENABLE_QEMU_SUPPORT, "Qemu Support is needed for confidential computing and it's disabled, "
             "enable it setting the env variable `ENABLE_QEMU_SUPPORT=True` in configuration"
diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 448a822c5..fe9deab26 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -11,6 +11,7 @@
 from pydantic import BaseModel, Field
 
 from aleph.vm.conf import settings
+from aleph.vm.sevclient import SevClient
 from aleph.vm.utils import cors_allow_all
 
 
@@ -122,6 +123,18 @@ async def about_system_usage(_: web.Request):
     return web.json_response(text=usage.json(exclude_none=True))
 
 
+@cors_allow_all
+async def about_certificates(request: web.Request):
+    """Public endpoint to expose platform certificates for confidential computing."""
+
+    if not settings.ENABLE_CONFIDENTIAL_COMPUTING:
+        return web.HTTPBadRequest(reason="Confidential computing setting not enabled on that server")
+
+    sev_client: SevClient = request.app["sev_client"]
+
+    return web.FileResponse(await sev_client.get_certificates())
+
+
 class Allocation(BaseModel):
     """An allocation is the set of resources that are currently allocated on this orchestrator.
     It contains the item_hashes of all persistent VMs, instances, on-demand VMs and jobs.
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 892106ba0..a2a712445 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -18,10 +18,11 @@
 
 from aleph.vm.conf import settings
 from aleph.vm.pool import VmPool
+from aleph.vm.sevclient import SevClient
 from aleph.vm.version import __version__
 
 from .metrics import create_tables, setup_engine
-from .resources import about_system_usage
+from .resources import about_certificates, about_system_usage
 from .tasks import (
     start_payment_monitoring_task,
     start_watch_for_messages_task,
@@ -95,6 +96,7 @@ def setup_webapp():
         web.get("/about/executions/details", about_executions),
         web.get("/about/executions/records", about_execution_records),
         web.get("/about/usage/system", about_system_usage),
+        web.get("/about/certificates", about_certificates),
         web.get("/about/config", about_config),
         # /control APIs are used to control the VMs and access their logs
         web.post("/control/allocation/notify", notify_allocation),
@@ -159,6 +161,12 @@ def run():
     app["secret_token"] = secret_token
     app["vm_pool"] = pool
 
+    # Store sevctl app singleton only if confidential feature is enabled
+    if settings.ENABLE_CONFIDENTIAL_COMPUTING:
+        sev_client = SevClient(settings.CONFIDENTIAL_DIRECTORY)
+        app["sev_client"] = sev_client
+        # TODO: Review and check sevctl first initialization steps, like (sevctl generate and sevctl provision)
+
     logger.debug(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
 
     try:
diff --git a/src/aleph/vm/sevclient.py b/src/aleph/vm/sevclient.py
new file mode 100644
index 000000000..fe9eb1c00
--- /dev/null
+++ b/src/aleph/vm/sevclient.py
@@ -0,0 +1,23 @@
+from pathlib import Path
+
+from aleph.vm.utils import run_in_subprocess
+
+
+class SevClient:
+    def __init__(self, sev_dir: Path):
+        self.sev_dir = sev_dir
+        self.certificates_dir = sev_dir / "platform"
+        self.certificates_dir.mkdir(exist_ok=True, parents=True)
+        self.certificates_archive = self.certificates_dir / "certs_export.cert"
+
+    async def sevctl_cmd(self, *args) -> bytes:
+        return await run_in_subprocess(
+            ["sevctl", *args],
+            check=True,
+        )
+
+    async def get_certificates(self) -> Path:
+        if not self.certificates_archive.is_file():
+            _ = await self.sevctl_cmd("export", str(self.certificates_archive))
+
+        return self.certificates_archive
diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
index 254e326df..52426d48c 100644
--- a/tests/supervisor/test_views.py
+++ b/tests/supervisor/test_views.py
@@ -1,8 +1,14 @@
+import tempfile
+from pathlib import Path
+from unittest import mock
+from unittest.mock import call
+
 import pytest
 from aiohttp import web
 
 from aleph.vm.conf import settings
 from aleph.vm.orchestrator.supervisor import setup_webapp
+from aleph.vm.sevclient import SevClient
 
 
 @pytest.mark.asyncio
@@ -121,3 +127,47 @@ def get_persistent_executions(self):
     )
     assert response.status == 200
     assert await response.json() == {"success": True, "successful": [], "failing": [], "errors": {}}
+
+
+@pytest.mark.asyncio
+async def test_about_certificates_missing_setting(aiohttp_client):
+    """Test that the certificates system endpoint returns an error if the setting isn't enabled"""
+    settings.ENABLE_CONFIDENTIAL_COMPUTING = False
+
+    app = setup_webapp()
+    app["sev_client"] = SevClient(Path().resolve())
+    client = await aiohttp_client(app)
+    response: web.Response = await client.get("/about/certificates")
+    assert response.status == 400
+    assert await response.text() == "400: Confidential computing setting not enabled on that server"
+
+
+@pytest.mark.asyncio
+async def test_about_certificates(aiohttp_client):
+    """Test that the certificates system endpoint responds. No auth needed"""
+
+    settings.ENABLE_QEMU_SUPPORT = True
+    settings.ENABLE_CONFIDENTIAL_COMPUTING = True
+    settings.setup()
+
+    with mock.patch(
+        "pathlib.Path.is_file",
+        return_value=False,
+    ) as is_file_mock:
+        with mock.patch(
+            "aleph.vm.sevclient.run_in_subprocess",
+            return_value=True,
+        ) as export_mock:
+            with tempfile.TemporaryDirectory() as tmp_dir:
+                app = setup_webapp()
+                sev_client = SevClient(Path(tmp_dir))
+                app["sev_client"] = sev_client
+                # Create mock file to return it
+                Path(sev_client.certificates_archive).touch(exist_ok=True)
+
+                client = await aiohttp_client(app)
+                response: web.Response = await client.get("/about/certificates")
+                assert response.status == 200
+                is_file_mock.assert_has_calls([call(), call()])
+                certificates_expected_dir = sev_client.certificates_archive
+                export_mock.assert_called_once_with(["sevctl", "export", str(certificates_expected_dir)], check=True)

From 30d3284f167af67ba5b94fde07cf2806bd9f8493 Mon Sep 17 00:00:00 2001
From: lyam 
Date: Wed, 12 Jun 2024 14:08:18 +0200
Subject: [PATCH 782/990] Fix: Correct string formatting in VM startup response

---
 src/aleph/vm/orchestrator/views/operator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 298486b73..7617a3bb5 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -182,7 +182,7 @@ async def operate_reboot(request: web.Request, authenticated_sender: str) -> web
             await create_vm_execution(vm_hash=vm_hash, pool=pool)
         return web.Response(status=200, body=f"Rebooted VM with ref {vm_hash}")
     else:
-        return web.Response(status=200, body="Starting VM (was not running) with ref {vm_hash}")
+        return web.Response(status=200, body=f"Starting VM (was not running) with ref {vm_hash}")
 
 
 @cors_allow_all

From b7e0ccf62dc30e7cffc835f052d15e6d5ab7ff0e Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 5 Jun 2024 10:39:03 +0200
Subject: [PATCH 783/990] Fix: Orchestrator failed with `assert
 result["result"] == HTTPOk.status_code`

Problem:

The diagnostic VM returned HTTP 200 with {"result": False} when it could not connect to the internet.

Since this is an OK return code, `raise_for_status` did not raise an error and an assertion error was raised.

Solution:

Test that the returned status code also corresponds to HTTP OK.
---
 src/aleph/vm/orchestrator/status.py | 13 +++++++---
 tests/supervisor/test_status.py     | 39 +++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 4 deletions(-)
 create mode 100644 tests/supervisor/test_status.py

diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py
index b0d76554d..90cf15d2f 100644
--- a/src/aleph/vm/orchestrator/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -118,10 +118,15 @@ async def check_ipv6(session: ClientSession, vm_id: ItemHash) -> bool:
 async def check_internet(session: ClientSession, vm_id: ItemHash) -> bool:
     """Check that the VM has internet connectivity. This requires DNS, IP, HTTP and TLS to work."""
     try:
-        result: dict = await get_json_from_vm(session, vm_id, "/internet")
-        assert result["result"] == HTTPOk.status_code
-        assert "Server" in result["headers"]
-        return True
+        response: dict = await get_json_from_vm(session, vm_id, "/internet")
+
+        # The HTTP Header "Server" must always be present in the result.
+        if "Server" not in response["headers"]:
+            raise ValueError("Server header not found in the result.")
+
+        # The diagnostic VM returns HTTP 200 with {"result": False} when cannot connect to the internet.
+        # else it forwards the return code if its own test endpoint.
+        return response.get("result") == HTTPOk.status_code
     except ClientResponseError:
         return False
 
diff --git a/tests/supervisor/test_status.py b/tests/supervisor/test_status.py
new file mode 100644
index 000000000..e3232f1bd
--- /dev/null
+++ b/tests/supervisor/test_status.py
@@ -0,0 +1,39 @@
+from unittest.mock import AsyncMock, MagicMock, Mock
+
+import pytest
+from aleph_message.models import ItemHash
+
+from aleph.vm.orchestrator.status import check_internet
+
+
+@pytest.mark.asyncio
+async def test_check_internet_no_server_header():
+    vm_id = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")
+
+    mock_session = Mock()
+    mock_session.get = MagicMock()
+    mock_session.get.__aenter__.return_value.json = AsyncMock(return_value={"result": 200})
+
+    # A "Server" header is always expected in the result from the VM.
+    # If it is not present, the diagnostic VM is not working correctly
+    # and an error must be raised.
+    with pytest.raises(ValueError):
+        await check_internet(mock_session, vm_id)
+
+
+@pytest.mark.asyncio
+async def test_check_internet_wrong_result_code():
+    vm_id = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")
+
+    mock_session = Mock()
+    mock_session.get = MagicMock()
+
+    mock_session.get.return_value.__aenter__.return_value.json = AsyncMock(
+        return_value={"result": 200, "headers": {"Server": "nginx"}}
+    )
+    assert await check_internet(mock_session, vm_id) is True
+
+    mock_session.get.return_value.__aenter__.return_value.json = AsyncMock(
+        return_value={"result": 400, "headers": {"Server": "nginx"}}
+    )
+    assert await check_internet(mock_session, vm_id) is False

From 1095e3f859c9c61bc3afc33a2cdeafac74ea3d16 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 13 Jun 2024 13:46:33 +0200
Subject: [PATCH 784/990] Problem: Crash when trying to auth via websocket
 (#630)

The auth function in websocket was crashing when the header  "X-Auth-Signature" wasn't passed, even  that authentification method wasn't used.

This caused issues when header "X-Auth-Signature" was not passed.

Endpoint authenticate_api_request is not adequate for Websocket connections (#632)

---------

Co-authored-by: Hugo Herter 
---
 src/aleph/vm/orchestrator/views/operator.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 7617a3bb5..148ecd092 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -10,7 +10,6 @@
 
 from aleph.vm.models import VmExecution
 from aleph.vm.orchestrator.run import create_vm_execution
-from aleph.vm.orchestrator.views import authenticate_api_request
 from aleph.vm.orchestrator.views.authentication import (
     authenticate_websocket_message,
     require_jwk_authentication,
@@ -68,7 +67,7 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
         ws = web.WebSocketResponse()
         await ws.prepare(request)
         try:
-            await authenticate_for_vm_or_403(execution, request, vm_hash, ws)
+            await authenticate_websocket_for_vm_or_403(execution, vm_hash, ws)
             await ws.send_json({"status": "connected"})
 
             queue = execution.vm.get_log_queue()
@@ -88,12 +87,12 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
             execution.vm.unregister_queue(queue)
 
 
-async def authenticate_for_vm_or_403(execution, request, vm_hash, ws):
-    """Allow authentication via HEADER or via websocket"""
-    if authenticate_api_request(request):
-        logger.debug(f"Accepted request to access logs via the allocatioan api key on {vm_hash}")
-        return True
+async def authenticate_websocket_for_vm_or_403(execution: VmExecution, vm_hash: ItemHash, ws: web.WebSocketResponse):
+    """Authenticate a websocket connection.
 
+    Web browsers do not allow setting headers in WebSocket requests, so the authentication
+    relies on the first message sent by the client.
+    """
     first_message = await ws.receive_json()
     credentials = first_message["auth"]
     authenticated_sender = await authenticate_websocket_message(credentials)

From b82450340254a52db57f7f7546caab545edcaadc Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 14 Jun 2024 11:58:27 +0200
Subject: [PATCH 785/990] Fix: CI Droplet cleanup failed when same name was
 used

When there were multiple Droplets with the same name, cleanup using doctl compute droplet delete -f $NAME would not work.

Error: There are 3 Droplets with the name "aleph-vm-ci-XXX"; please provide a specific Droplet ID. [425559566, 425702949, 425703724]
---
 .github/workflows/test-on-droplets-matrix.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index 1493af67e..c6d048d4f 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -176,4 +176,9 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          doctl compute droplet delete -f aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }}
+          DROPLET_IDS=$(doctl compute droplet list --format "ID,Name" --no-header | grep "aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }}" | awk '{print $1}')
+
+          for DROPLET_ID in $DROPLET_IDS; do
+            echo "Deleting droplet with ID: $DROPLET_ID"
+            doctl compute droplet delete --force $DROPLET_ID
+          done

From d1a1497371f349ffccecebd94b55c490dc19d3d3 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Mon, 17 Jun 2024 09:59:38 +0200
Subject: [PATCH 786/990] Fix: `make clean` did not cleanup all resources

---
 packaging/Makefile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packaging/Makefile b/packaging/Makefile
index fffb158dc..d1581df9a 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -59,6 +59,9 @@ clean:
 	rm -fr ./target/*
 	rm -fr ./build/*
 	rm -fr ./aleph-vm/opt/aleph-vm/
+	rm -fr ./aleph-vm/opt/firecracker/
+	rm -fr ./aleph-vm/opt/kubo/
+	rm -fr ./aleph-vm/opt/aleph-vm/
 
 all-podman-debian-11: version
 	cd .. && podman build -t localhost/aleph-vm-packaging-debian-11:latest -f ./packaging/debian-11.dockerfile .

From 3f978c5d98a2a34e83e6f6391d7e6a007938b32f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 29 May 2024 20:31:16 +0200
Subject: [PATCH 787/990] Fix: Test dependencies were outdated

This updates test dependencies in pyproject.toml.

Support for Python 3.9 is removed in favour of Python 3.10 minimum.

Dependencies for running the software are untouched.
---
 pyproject.toml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 123e07c91..067de413b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,7 +7,7 @@ name = "aleph-vm"
 dynamic = ["version"]
 description = "Aleph.im VM execution engine"
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 license = {file = "LICENSE"}
 keywords = []
 authors = [
@@ -86,10 +86,10 @@ check = "aleph-vm controller run {args:--help}"
 type = "virtual"
 system-packages = true
 dependencies = [
-  "pytest==8.0.1",
-  "pytest-cov==4.1.0",
-  "pytest-mock==3.12.0",
-  "pytest-asyncio==0.23.5",
+  "pytest==8.2.1",
+  "pytest-cov==5.0.0",
+  "pytest-mock==3.14.0",
+  "pytest-asyncio==0.23.7",
   "pytest-aiohttp==1.0.5",
 ]
 [tool.hatch.envs.testing.scripts]
@@ -105,14 +105,14 @@ cov = [
 ]
 
 [[tool.hatch.envs.all.matrix]]
-python = ["3.9", "3.10", "3.11", "3.12"]
+python = ["3.10", "3.11", "3.12"]
 
 [tool.hatch.envs.lint]
 detached = true
 dependencies = [
-  "black==24.1.1",
+  "black==24.3.0",
   "mypy==1.8.0",
-  "ruff==0.1.15",
+  "ruff==0.4.6",
   "isort==5.13.2",
 ]
 [tool.hatch.envs.lint.scripts]
@@ -151,7 +151,7 @@ line-length = 120
 #skip-string-normalization = true
 
 [tool.mypy]
-python_version = "3.9"
+python_version = "3.10"
 install_types = true
 non_interactive = true
 ignore_missing_imports = true
@@ -159,8 +159,9 @@ explicit_package_bases = true
 check_untyped_defs = true
 
 [tool.ruff]
-target-version = "py39"
+target-version = "py310"
 line-length = 120
+[tool.ruff.lint]
 select = [
   "A",
   "ARG",
@@ -205,13 +206,12 @@ ignore = [
 #  "F401",
 #]
 
-[tool.ruff.isort]
-known-first-party = ["aleph.vm"]
+isort.known-first-party = ["aleph.vm"]
 
 #[tool.ruff.flake8-tidy-imports]
 #ban-relative-imports = "all"
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
 "tests/**/*" = ["PLR2004", "S101", "TID252"]
 

From 18b0f88f65cb293e79f90bc8e92380168f92eede Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 5 Jun 2024 12:01:39 +0200
Subject: [PATCH 788/990] Fix: Binary `sevctl` was absent from debian packages

Solution: Build `sevctl` using an upstream version of Rust (the version in Debian is not supported), and bundle it in the Debian packages.

Add a setting in aleph-vm with the path of the bundled binary.
---
 .dockerignore                                 |  1 +
 .github/workflows/build-deb-package.yml       |  4 ++++
 .github/workflows/test-on-droplets-matrix.yml |  1 +
 .gitignore                                    |  1 +
 .gitmodules                                   |  0
 packaging/Makefile                            | 11 ++++++++++-
 packaging/debian-11.dockerfile                |  2 +-
 packaging/debian-12.dockerfile                |  2 +-
 packaging/ubuntu-22.04.dockerfile             |  1 +
 packaging/ubuntu-24.04.dockerfile             |  1 +
 src/aleph/vm/conf.py                          |  1 +
 11 files changed, 22 insertions(+), 3 deletions(-)
 create mode 100644 .gitmodules

diff --git a/.dockerignore b/.dockerignore
index 6937fd0ff..9f4bf0390 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -14,3 +14,4 @@
 **/data.tgz
 /pydantic/
 **/target
+/packaging/sevctl/target
diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 0c99d47e3..25de1acc7 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -26,9 +26,13 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
+          submodules: true
           # Fetch the whole history for all tags and branches (required for aleph.__version__)
           fetch-depth: 0
 
+      - name: Initialize git submodules
+        run: git submodule init
+
       - run: |
           cd packaging && make ${{ matrix.make_target }} && cd ..
           ls packaging/target
diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index c6d048d4f..4033f8eee 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -68,6 +68,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          submodules: true
 
       - name: Install doctl
         uses: digitalocean/action-doctl@v2
diff --git a/.gitignore b/.gitignore
index cb2fd28d8..af67f6e22 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,7 @@ node_modules
 /runtimes/aleph-debian-11-python/rootfs/
 /packaging/aleph-vm/opt/
 /packaging/target/
+/packaging/sevctl/target/
 /packaging/repositories/*/db/
 /packaging/repositories/*/dists/
 /packaging/repositories/*/pool/
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 000000000..e69de29bb
diff --git a/packaging/Makefile b/packaging/Makefile
index d1581df9a..a1df5ecbd 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -18,13 +18,14 @@ debian-package-code:
 	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.4' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
-debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo
+debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo  target/bin/sevctl
 	rm -fr ./aleph-vm/opt/firecracker
 	mkdir -p ./aleph-vm/opt/firecracker
 	cp -pr ./target/vmlinux.bin ./aleph-vm/opt/firecracker/
 	cp -pr ./target/firecracker ./aleph-vm/opt/firecracker/
 	cp -pr ./target/jailer ./aleph-vm/opt/firecracker/
 	cp -pr ./target/kubo/kubo ./aleph-vm/opt/kubo
+	cp -pr ./target/bin/sevctl ./aleph-vm/opt/sevctl
 
 firecracker-bins: target-dir build-dir
 	mkdir -p ./build/firecracker-release
@@ -45,6 +46,11 @@ download-ipfs-kubo: target-dir build-dir
 	mkdir -p ./target/kubo
 	curl -fsSL https://github.com/ipfs/kubo/releases/download/v0.23.0/kubo_v0.23.0_linux-amd64.tar.gz | tar -xz --directory ./target/kubo
 
+target/bin/sevctl:
+	# Release 0.4.3 matches revision c41c9172be013d6f10b9e1d7286fcb021805d5a5
+	cargo install --git https://github.com/virtee/sevctl.git --rev c41c9172be013d6f10b9e1d7286fcb021805d5a5 --target x86_64-unknown-linux-gnu --root ./target
+	./target/bin/sevctl -V
+
 version:
 	python3 ./version_from_git.py --inplace deb aleph-vm/DEBIAN/control
 	python3 ./version_from_git.py --inplace __version__ ../src/aleph/vm/version.py
@@ -62,6 +68,7 @@ clean:
 	rm -fr ./aleph-vm/opt/firecracker/
 	rm -fr ./aleph-vm/opt/kubo/
 	rm -fr ./aleph-vm/opt/aleph-vm/
+	rm -fr ./sevctl/target/
 
 all-podman-debian-11: version
 	cd .. && podman build -t localhost/aleph-vm-packaging-debian-11:latest -f ./packaging/debian-11.dockerfile .
@@ -86,6 +93,8 @@ all-podman-debian-12: version
 	mv target/aleph-vm.deb target/aleph-vm.debian-12.deb
 
 all-podman-ubuntu-2204: version
+	# Ensure the sevctl submodule is checked out first.
+	git submodule init
 	cd .. && podman build -t localhost/aleph-vm-packaging-ubuntu-2204:latest -f ./packaging/ubuntu-22.04.dockerfile .
 	mkdir -p ./target
 	podman run --rm -ti \
diff --git a/packaging/debian-11.dockerfile b/packaging/debian-11.dockerfile
index 677c28827..ebe903ef6 100644
--- a/packaging/debian-11.dockerfile
+++ b/packaging/debian-11.dockerfile
@@ -1,4 +1,4 @@
-FROM debian:bullseye
+FROM rust:1.79.0-bullseye
 
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     make \
diff --git a/packaging/debian-12.dockerfile b/packaging/debian-12.dockerfile
index f4177b128..eb130d7dc 100644
--- a/packaging/debian-12.dockerfile
+++ b/packaging/debian-12.dockerfile
@@ -1,4 +1,4 @@
-FROM debian:bookworm
+FROM rust:1.79.0-bookworm
 
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     make \
diff --git a/packaging/ubuntu-22.04.dockerfile b/packaging/ubuntu-22.04.dockerfile
index 32467a5e9..34d6bef55 100644
--- a/packaging/ubuntu-22.04.dockerfile
+++ b/packaging/ubuntu-22.04.dockerfile
@@ -6,6 +6,7 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     curl \
     sudo \
     python3-pip \
+    cargo \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /opt
diff --git a/packaging/ubuntu-24.04.dockerfile b/packaging/ubuntu-24.04.dockerfile
index 8f892e746..d5685f729 100644
--- a/packaging/ubuntu-24.04.dockerfile
+++ b/packaging/ubuntu-24.04.dockerfile
@@ -6,6 +6,7 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     curl \
     sudo \
     python3-pip \
+    cargo \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /opt
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index f2c11a989..48c78248f 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -177,6 +177,7 @@ class Settings(BaseSettings):
 
     FIRECRACKER_PATH = Path("/opt/firecracker/firecracker")
     JAILER_PATH = Path("/opt/firecracker/jailer")
+    SEV_CTL_PATH = Path("/opt/sevctl")
     LINUX_PATH = Path("/opt/firecracker/vmlinux.bin")
     INIT_TIMEOUT: float = 20.0
 

From 98d6f1a5a45dac62b90931566c180b0a6d505819 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 14 Jun 2024 13:26:24 +0200
Subject: [PATCH 789/990] Fix: CI `fail fast' prevented debug on other OS

---
 .github/workflows/build-deb-package.yml       | 1 +
 .github/workflows/test-on-droplets-matrix.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 25de1acc7..a55a20982 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -7,6 +7,7 @@ jobs:
     name: "Build ${{ matrix.os }} Package"
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         os: ["debian-11", "debian-12", "ubuntu-22.04"]
         include:
diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index 4033f8eee..33ff38005 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -26,6 +26,7 @@ jobs:
     timeout-minutes: 10
 
     strategy:
+      fail-fast: false
       matrix:
 
         # Check compatibility with all supported OSes.

From 377b77f389629ad7422b16f8b7ea15e965984bbf Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 18 Jun 2024 09:31:00 +0200
Subject: [PATCH 790/990] Fix: No test ensured binaries copied to .deb

---
 .github/workflows/build-deb-package.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index a55a20982..ca4776b8b 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -38,6 +38,14 @@ jobs:
           cd packaging && make ${{ matrix.make_target }} && cd ..
           ls packaging/target
 
+      - name: Ensure that the relevant files are present in the package
+        run: |
+          dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/kubo/ipfs
+          dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/firecracker/firecracker
+          dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/firecracker/jailer
+          dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/firecracker/vmlinux.bin
+          dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/sevctl
+
       - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.artifact_name }}

From 4060e001d5d2f5064e13774ab5bf677eb5ee8120 Mon Sep 17 00:00:00 2001
From: aliel 
Date: Wed, 19 Jun 2024 20:25:55 +0200
Subject: [PATCH 791/990] Disable printing system logs on deb package as per
 default configuration recommendation.

---
 packaging/aleph-vm/etc/aleph-vm/supervisor.env | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packaging/aleph-vm/etc/aleph-vm/supervisor.env b/packaging/aleph-vm/etc/aleph-vm/supervisor.env
index 9793f2422..b661ee90f 100644
--- a/packaging/aleph-vm/etc/aleph-vm/supervisor.env
+++ b/packaging/aleph-vm/etc/aleph-vm/supervisor.env
@@ -1,3 +1,4 @@
-ALEPH_VM_PRINT_SYSTEM_LOGS=True
+# System logs make boot ~2x slower
+ALEPH_VM_PRINT_SYSTEM_LOGS=False
 ALEPH_VM_DOMAIN_NAME=vm.example.org
 ALEPH_VM_PAYMENT_RECEIVER_ADDRESS=

From 834ae4dbc40b802502f48bd07cf16dfcd6160b33 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 20 Jun 2024 09:59:58 +0200
Subject: [PATCH 792/990] Add more information on testing for devs

---
 TESTING.md | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/TESTING.md b/TESTING.md
index 67f9143b5..7adbae0c6 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -44,3 +44,30 @@ hatch env create testing
 ```shell
 hatch run testing:test
 ```
+
+
+## Debugging the tests
+Some tricks and options that might help debugging problematic tests.
+
+Only launch pytest with a test name and more verbose debugging
+```shell
+hatch run testing:pytest -vv --log-level=DEBUG --full-trace -o log_cli=true -k 
+```
+
+
+Specify `--capture=no` to pytest so it launch. This way you get the full output, including firecracker logs
+
+## Debugging runtimes
+If the error is in the runtime:
+Modify the #!  to pass the -v option to python, which will print all the debugging info
+`#!/usr/bin/python3 -vOO`
+
+To have these modification take effect you need to rebuild the runtime file using `create_disk_image.sh` as _root_
+
+```shell
+sudo bash create_disk_image.sh
+```
+
+Don't forget to have the print system log option set `ALEPH_VM_PRINT_SYSTEM_LOGS=1`
+
+`aleph-debian-12-python` is used in test_create_execution
\ No newline at end of file

From 4e350d6451f9fab93f9af81d71e06438ad2ac38e Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 17 Jun 2024 14:04:35 +0200
Subject: [PATCH 793/990] Add test for the reboot endpoint

---
 tests/supervisor/views/test_operator.py | 38 +++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 tests/supervisor/views/test_operator.py

diff --git a/tests/supervisor/views/test_operator.py b/tests/supervisor/views/test_operator.py
new file mode 100644
index 000000000..3e50fe53e
--- /dev/null
+++ b/tests/supervisor/views/test_operator.py
@@ -0,0 +1,38 @@
+import pytest
+
+from aleph.vm.orchestrator.supervisor import setup_webapp
+
+
+@pytest.mark.asyncio
+async def test_reboot_ok(aiohttp_client, mocker):
+    mock_address = "mock_address"
+    mock_hash = "fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_"
+    mocker.patch(
+        "aleph.vm.orchestrator.views.authentication.authenticate_jwk",
+        return_value=mock_address,
+    )
+
+    class FakeVmPool:
+        executions = {
+            mock_hash: mocker.Mock(
+                vm_hash=mock_hash,
+                message=mocker.Mock(address=mock_address),
+                is_confidential=False,
+                is_running=True,
+            ),
+        }
+        systemd_manager = mocker.Mock(restart=mocker.Mock())
+
+    app = setup_webapp()
+    pool = FakeVmPool()
+    app["vm_pool"] = pool
+    app["pubsub"] = FakeVmPool()
+    client = await aiohttp_client(app)
+    response = await client.post(
+        f"/control/machine/{mock_hash}/reboot",
+    )
+    assert response.status == 200
+    assert (
+        await response.text() == "Rebooted VM with ref fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_"
+    )
+    assert pool.systemd_manager.restart.call_count == 1

From 7e76cf1887566e943ff794ab18e5afe801b96a18 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 20 Jun 2024 14:07:53 +0200
Subject: [PATCH 794/990] fix VM log in systemd controller for qemu

Modification still needed for firecracker instances
---
 src/aleph/vm/controllers/__main__.py          |  2 +-
 src/aleph/vm/controllers/configuration.py     |  1 +
 .../vm/controllers/firecracker/executable.py  |  1 +
 src/aleph/vm/controllers/qemu/instance.py     |  9 ++-
 src/aleph/vm/hypervisors/qemu/qemuvm.py       | 63 ++++++-------------
 5 files changed, 30 insertions(+), 46 deletions(-)

diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index 39d606784..34d5a71c2 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -72,7 +72,7 @@ async def execute_persistent_vm(config: Configuration):
         process = await execution.start(config.vm_configuration.config_file_path)
     else:
         assert isinstance(config.vm_configuration, QemuVMConfiguration)
-        execution = QemuVM(config.vm_configuration)
+        execution = QemuVM(config.vm_hash, config.vm_configuration)
         process = await execution.start()
 
     return execution, process
diff --git a/src/aleph/vm/controllers/configuration.py b/src/aleph/vm/controllers/configuration.py
index be8d1986c..f54acbcad 100644
--- a/src/aleph/vm/controllers/configuration.py
+++ b/src/aleph/vm/controllers/configuration.py
@@ -36,6 +36,7 @@ class HypervisorType(str, Enum):
 
 class Configuration(BaseModel):
     vm_id: int
+    vm_hash: str
     settings: Settings
     vm_configuration: Union[QemuVMConfiguration, VMConfiguration]
     hypervisor: HypervisorType = HypervisorType.firecracker
diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index ad1a8364e..472235213 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -285,6 +285,7 @@ async def configure(self):
 
             configuration = Configuration(
                 vm_id=self.vm_id,
+                vm_hash=self.vm_hash,
                 settings=settings,
                 vm_configuration=vm_configuration,
             )
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index 7d6128e20..7ddc33e38 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -92,7 +92,7 @@ class EntryDict(TypedDict):
     MESSAGE: str
 
 
-def make_logs_queue(stdout_identifier, stderr_identifier, skip_past=True) -> tuple[asyncio.Queue, Callable[[], None]]:
+def make_logs_queue(stdout_identifier, stderr_identifier, skip_past=False) -> tuple[asyncio.Queue, Callable[[], None]]:
     """Create a queue which streams the logs for the process.
 
     @param stdout_identifier: journald identifier for process stdout
@@ -131,6 +131,7 @@ def _ready_for_read() -> None:
     loop.add_reader(r.fileno(), _ready_for_read)
 
     def do_cancel():
+        logger.info(f"cancelling reader {r}")
         loop.remove_reader(r.fileno())
         r.close()
 
@@ -244,7 +245,11 @@ async def configure(self):
         )
 
         configuration = Configuration(
-            vm_id=self.vm_id, settings=settings, vm_configuration=vm_configuration, hypervisor=HypervisorType.qemu
+            vm_id=self.vm_id,
+            vm_hash=self.vm_hash,
+            settings=settings,
+            vm_configuration=vm_configuration,
+            hypervisor=HypervisorType.qemu,
         )
 
         save_controller_configuration(self.vm_hash, configuration)
diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index 87ba9724e..c20c17b77 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -3,9 +3,10 @@
 from asyncio import Task
 from asyncio.subprocess import Process
 from pathlib import Path
-from typing import Optional
+from typing import Optional, TextIO
 
 import qmp
+from systemd import journal
 
 from aleph.vm.controllers.configuration import QemuVMConfiguration
 from aleph.vm.controllers.qemu.instance import logger
@@ -29,7 +30,7 @@ def __repr__(self) -> str:
         else:
             return ""
 
-    def __init__(self, config: QemuVMConfiguration):
+    def __init__(self, vm_hash, config: QemuVMConfiguration):
         self.qemu_bin_path = config.qemu_bin_path
         self.cloud_init_drive_path = config.cloud_init_drive_path
         self.image_path = config.image_path
@@ -39,6 +40,15 @@ def __init__(self, config: QemuVMConfiguration):
         self.mem_size_mb = config.mem_size_mb
         self.interface_name = config.interface_name
         self.log_queues = []
+        self.vm_hash = vm_hash
+
+    @property
+    def _journal_stdout_name(self) -> str:
+        return f"vm-{self.vm_hash}-stdout"
+
+    @property
+    def _journal_stderr_name(self) -> str:
+        return f"vm-{self.vm_hash}-stderr"
 
     def prepare_start(self):
         pass
@@ -49,6 +59,9 @@ async def start(
         # Based on the command
         #  qemu-system-x86_64 -enable-kvm -m 2048 -net nic,model=virtio
         # -net tap,ifname=tap0,script=no,downscript=no -drive file=alpine.qcow2,media=disk,if=virtio -nographic
+
+        journal_stdout: TextIO = journal.stream(self._journal_stdout_name)
+        journal_stderr: TextIO = journal.stream(self._journal_stderr_name)
         # hardware_resources.published ports -> not implemented at the moment
         # hardware_resources.seconds -> only for microvm
         args = [
@@ -89,51 +102,15 @@ async def start(
         self.qemu_process = proc = await asyncio.create_subprocess_exec(
             *args,
             stdin=asyncio.subprocess.DEVNULL,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
+            stdout=journal_stdout,
+            stderr=journal_stderr,
         )
 
-        logger.debug(f"started qemu vm {self}, {proc}")
+        print(
+            f"Started QemuVm {self}, {proc}. Log available with: journalctl -t  {self._journal_stdout_name} -t {self._journal_stderr_name}"
+        )
         return proc
 
-    # TODO : convert when merging with log fixing branch
-    async def _process_stderr(self):
-        while not self.qemu_process:
-            await asyncio.sleep(0.01)  # Todo: Use signal here
-        while True:
-            assert self.qemu_process.stderr, "Qemu process stderr is missing"
-            line = await self.qemu_process.stderr.readline()
-            if not line:  # FD is closed nothing more will come
-                print(self, "EOF")
-                return
-            for queue in self.log_queues:
-                await queue.put(("stderr", line))
-            print(self, line.decode().strip(), file=sys.stderr)
-
-    def start_printing_logs(self) -> tuple[Task, Task]:
-        """Start two tasks to process the stdout and stderr
-
-        It will stream their content to queues registered on self.log_queues
-        It will also print them"""
-
-        loop = asyncio.get_running_loop()
-        stdout_task = loop.create_task(self._process_stdout())
-        stderr_task = loop.create_task(self._process_stderr())
-        return stdout_task, stderr_task
-
-    async def _process_stdout(self):
-        while not self.qemu_process:
-            await asyncio.sleep(0.01)  # Todo: Use signal here
-        while True:
-            assert self.qemu_process.stdout, "Qemu process stdout is missing"
-            line = await self.qemu_process.stdout.readline()
-            if not line:  # FD is closed nothing more will come
-                print(self, "EOF")
-                return
-            for queue in self.log_queues:
-                await queue.put(("stdout", line))
-            print(self, line.decode().strip())
-
     def _get_qmpclient(self) -> Optional[qmp.QEMUMonitorProtocol]:
         if not (self.qmp_socket_path and self.qmp_socket_path.exists()):
             return None

From 2e0ddc7a11199739636f988184095b9752be03a4 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 20 Jun 2024 14:10:30 +0200
Subject: [PATCH 795/990] remove useless method

---
 src/aleph/vm/controllers/qemu/instance.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index 7ddc33e38..62fa83cca 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -320,18 +320,6 @@ def print_logs(self) -> None:
         """Print logs to our output for debugging"""
         queue = self.get_log_queue()
 
-        async def print_logs():
-            try:
-                while True:
-                    log_type, message = await queue.get()
-                    fd = sys.stderr if log_type == "stderr" else sys.stdout
-                    print(self, message, file=fd)
-            finally:
-                self.unregister_queue(queue)
-
-        loop = asyncio.get_running_loop()
-        self.print_task = loop.create_task(print_logs(), name=f"{self}-print-logs")
-
     def get_log_queue(self) -> asyncio.Queue:
         queue, canceller = make_logs_queue(self._journal_stdout_name, self._journal_stderr_name)
         self._queue_cancellers[queue] = canceller

From 83b35be24794f76fa8f7aadff911b23e4c739c0a Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 20 Jun 2024 14:11:04 +0200
Subject: [PATCH 796/990] update doc

---
 src/aleph/vm/controllers/qemu/QEMU.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/controllers/qemu/QEMU.md b/src/aleph/vm/controllers/qemu/QEMU.md
index 66b63e75b..d1d11059f 100644
--- a/src/aleph/vm/controllers/qemu/QEMU.md
+++ b/src/aleph/vm/controllers/qemu/QEMU.md
@@ -107,8 +107,11 @@ import aiohttp
 def on_message(content):
     try:
         msg = json.loads(content)
-        fd = sys.stderr if msg["type"] == "stderr" else sys.stdout
-        print("<", msg["message"], file=fd, end="")
+        if msg.get('status'):
+            print(msg)
+        else:
+            fd = sys.stderr if msg["type"] == "stderr" else sys.stdout
+            print("<", msg["message"], file=fd, end="")
     except:
         print("unable to parse", content)
 
@@ -125,7 +128,6 @@ async def tail_websocket(url):
                     break
                 elif msg.type == aiohttp.WSMsgType.ERROR:
                     print("Error", msg)
-                    break
 
 
 vm_hash = "decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca"

From a9b25a10ea09e29b882641c445a9f502b48e965e Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 20 Jun 2024 14:13:55 +0200
Subject: [PATCH 797/990] useless import

---
 src/aleph/vm/controllers/qemu/instance.py | 1 -
 src/aleph/vm/network/utils/__init__.py    | 0
 2 files changed, 1 deletion(-)
 create mode 100644 src/aleph/vm/network/utils/__init__.py

diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index 62fa83cca..68758e361 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -2,7 +2,6 @@
 import json
 import logging
 import shutil
-import sys
 from asyncio import Task
 from asyncio.subprocess import Process
 from pathlib import Path
diff --git a/src/aleph/vm/network/utils/__init__.py b/src/aleph/vm/network/utils/__init__.py
new file mode 100644
index 000000000..e69de29bb

From 138f4d3289c1dd2301ff45434a48923d6737616c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 20 Jun 2024 14:53:44 +0200
Subject: [PATCH 798/990] Move make_logs_queue in utils/logs.py

---
 src/aleph/vm/controllers/qemu/instance.py    | 56 +------------------
 src/aleph/vm/network/utils/__init__.py       |  0
 src/aleph/vm/{utils.py => utils/__init__.py} |  0
 src/aleph/vm/utils/logs.py                   | 58 ++++++++++++++++++++
 4 files changed, 61 insertions(+), 53 deletions(-)
 delete mode 100644 src/aleph/vm/network/utils/__init__.py
 rename src/aleph/vm/{utils.py => utils/__init__.py} (100%)
 create mode 100644 src/aleph/vm/utils/logs.py

diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index 68758e361..d56d9d0c3 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -5,14 +5,15 @@
 from asyncio import Task
 from asyncio.subprocess import Process
 from pathlib import Path
-from typing import Callable, Generic, Optional, TypedDict, TypeVar, Union
+from typing import Callable, Generic, Optional, TypeVar, Union
 
 import psutil
+
+from aleph.vm.utils.logs import make_logs_queue
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
 from aleph_message.models.execution.instance import RootfsVolume
 from aleph_message.models.execution.volume import PersistentVolume, VolumePersistence
-from systemd import journal
 
 from aleph.vm.conf import settings
 from aleph.vm.controllers.configuration import (
@@ -86,57 +87,6 @@ async def make_writable_volume(self, parent_image_path, volume: Union[Persistent
 ConfigurationType = TypeVar("ConfigurationType")
 
 
-class EntryDict(TypedDict):
-    SYSLOG_IDENTIFIER: str
-    MESSAGE: str
-
-
-def make_logs_queue(stdout_identifier, stderr_identifier, skip_past=False) -> tuple[asyncio.Queue, Callable[[], None]]:
-    """Create a queue which streams the logs for the process.
-
-    @param stdout_identifier: journald identifier for process stdout
-    @param stderr_identifier: journald identifier for process stderr
-    @param skip_past: Skip past history.
-    @return: queue and function to cancel the queue.
-
-    The consumer is required to call the queue cancel function when it's done consuming the queue.
-
-    Works by creating a journald reader, and using `add_reader` to call a callback when
-    data is available for reading.
-    In the callback we check the message type and fill the queue accordingly
-
-    For more information refer to the sd-journal(3) manpage
-    and systemd.journal module documentation.
-    """
-    r = journal.Reader()
-    r.add_match(SYSLOG_IDENTIFIER=stdout_identifier)
-    r.add_match(SYSLOG_IDENTIFIER=stderr_identifier)
-    queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
-
-    def _ready_for_read() -> None:
-        change_type = r.process()  # reset fd status
-        if change_type != journal.APPEND:
-            return
-        entry: EntryDict
-        for entry in r:
-            log_type = "stdout" if entry["SYSLOG_IDENTIFIER"] == stdout_identifier else "stderr"
-            msg = entry["MESSAGE"]
-            asyncio.create_task(queue.put((log_type, msg)))
-
-    if skip_past:
-        r.seek_tail()
-
-    loop = asyncio.get_event_loop()
-    loop.add_reader(r.fileno(), _ready_for_read)
-
-    def do_cancel():
-        logger.info(f"cancelling reader {r}")
-        loop.remove_reader(r.fileno())
-        r.close()
-
-    return queue, do_cancel
-
-
 class AlephQemuInstance(Generic[ConfigurationType], CloudInitMixin, AlephVmControllerInterface):
     vm_id: int
     vm_hash: ItemHash
diff --git a/src/aleph/vm/network/utils/__init__.py b/src/aleph/vm/network/utils/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/src/aleph/vm/utils.py b/src/aleph/vm/utils/__init__.py
similarity index 100%
rename from src/aleph/vm/utils.py
rename to src/aleph/vm/utils/__init__.py
diff --git a/src/aleph/vm/utils/logs.py b/src/aleph/vm/utils/logs.py
new file mode 100644
index 000000000..6baca5268
--- /dev/null
+++ b/src/aleph/vm/utils/logs.py
@@ -0,0 +1,58 @@
+import asyncio
+import logging
+from typing import TypedDict, Callable
+
+from systemd import journal
+
+logger = logging.getLogger(__name__)
+
+
+class EntryDict(TypedDict):
+    SYSLOG_IDENTIFIER: str
+    MESSAGE: str
+
+
+def make_logs_queue(stdout_identifier, stderr_identifier, skip_past=False) -> tuple[asyncio.Queue, Callable[[], None]]:
+    """Create a queue which streams the logs for the process.
+
+    @param stdout_identifier: journald identifier for process stdout
+    @param stderr_identifier: journald identifier for process stderr
+    @param skip_past: Skip past history.
+    @return: queue and function to cancel the queue.
+
+    The consumer is required to call the queue cancel function when it's done consuming the queue.
+
+    Works by creating a journald reader, and using `add_reader` to call a callback when
+    data is available for reading.
+    In the callback we check the message type and fill the queue accordingly
+
+    For more information refer to the sd-journal(3) manpage
+    and systemd.journal module documentation.
+    """
+    r = journal.Reader()
+    r.add_match(SYSLOG_IDENTIFIER=stdout_identifier)
+    r.add_match(SYSLOG_IDENTIFIER=stderr_identifier)
+    queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
+
+    def _ready_for_read() -> None:
+        change_type = r.process()  # reset fd status
+        if change_type != journal.APPEND:
+            return
+        entry: EntryDict
+        for entry in r:
+            log_type = "stdout" if entry["SYSLOG_IDENTIFIER"] == stdout_identifier else "stderr"
+            msg = entry["MESSAGE"]
+            asyncio.create_task(queue.put((log_type, msg)))
+
+    if skip_past:
+        r.seek_tail()
+
+    loop = asyncio.get_event_loop()
+    loop.add_reader(r.fileno(), _ready_for_read)
+
+    def do_cancel():
+        logger.info(f"cancelling reader {r}")
+        loop.remove_reader(r.fileno())
+        r.close()
+
+    return queue, do_cancel

From 9feb2fd3a9806a325248bdd5d9c9cd86da86565b Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 21 Jun 2024 12:01:32 +0200
Subject: [PATCH 799/990] Use journald logging for firecracker too

---
 src/aleph/vm/controllers/__main__.py          |  4 +-
 .../vm/controllers/firecracker/executable.py  | 19 +-----
 src/aleph/vm/controllers/interface.py         | 37 +++++++++--
 src/aleph/vm/controllers/qemu/instance.py     | 35 +---------
 .../vm/hypervisors/firecracker/microvm.py     | 66 +++++++------------
 src/aleph/vm/hypervisors/qemu/qemuvm.py       |  2 -
 6 files changed, 58 insertions(+), 105 deletions(-)

diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index 34d5a71c2..32640bf6e 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -61,6 +61,7 @@ async def execute_persistent_vm(config: Configuration):
         assert isinstance(config.vm_configuration, VMConfiguration)
         execution = MicroVM(
             vm_id=config.vm_id,
+            vm_hash=config.vm_hash,
             firecracker_bin_path=config.vm_configuration.firecracker_bin_path,
             jailer_base_directory=config.settings.JAILER_BASE_DIR,
             use_jailer=config.vm_configuration.use_jailer,
@@ -83,9 +84,6 @@ async def handle_persistent_vm(config: Configuration, execution: Union[MicroVM,
     loop = asyncio.get_event_loop()
     loop.add_signal_handler(signal.SIGTERM, execution.send_shutdown_message)
 
-    if config.settings.PRINT_SYSTEM_LOGS:
-        execution.start_printing_logs()
-
     await process.wait()
     logger.info(f"Process terminated with {process.returncode}")
 
diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index 472235213..e8c463920 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -188,6 +188,7 @@ def __init__(
 
         self.fvm = MicroVM(
             vm_id=self.vm_id,
+            vm_hash=vm_hash,
             firecracker_bin_path=settings.FIRECRACKER_PATH,
             jailer_base_directory=settings.JAILER_BASE_DIR,
             use_jailer=settings.USE_JAILER,
@@ -259,9 +260,6 @@ async def start(self):
                 await self.tap_interface.delete()
             raise
 
-        if self.enable_console:
-            self.fvm.start_printing_logs()
-
         await self.wait_for_init()
         logger.debug(f"started fvm {self.vm_id}")
         await self.load_configuration()
@@ -331,18 +329,3 @@ async def teardown(self):
 
     async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
         raise NotImplementedError()
-
-    def get_log_queue(self) -> asyncio.Queue:
-        queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
-        # Limit the number of queues per VM
-
-        if len(self.fvm.log_queues) > 20:
-            logger.warning("Too many log queues, dropping the oldest one")
-            self.fvm.log_queues.pop(0)
-        self.fvm.log_queues.append(queue)
-        return queue
-
-    def unregister_queue(self, queue: asyncio.Queue):
-        if queue in self.fvm.log_queues:
-            self.fvm.log_queues.remove(queue)
-        queue.empty()
diff --git a/src/aleph/vm/controllers/interface.py b/src/aleph/vm/controllers/interface.py
index b7afb32bf..5ee710663 100644
--- a/src/aleph/vm/controllers/interface.py
+++ b/src/aleph/vm/controllers/interface.py
@@ -3,8 +3,9 @@
 from abc import ABC
 from asyncio.subprocess import Process
 from collections.abc import Coroutine
-from typing import Any, Optional
+from typing import Any, Optional, Callable
 
+from aleph.vm.utils.logs import make_logs_queue
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
 
@@ -15,6 +16,9 @@
 
 
 class AlephVmControllerInterface(ABC):
+    log_queues: list[asyncio.Queue] = []
+    _queue_cancellers: dict[asyncio.Queue, Callable] = {}
+
     vm_id: int
     """id in the VMPool, attributed at execution"""
     vm_hash: ItemHash
@@ -89,8 +93,29 @@ async def create_snapshot(self) -> CompressedDiskVolumeSnapshot:
         """Must be implement if self.support_snapshot is True"""
         raise NotImplementedError()
 
-    async def get_log_queue(self) -> asyncio.Queue:
-        raise NotImplementedError()
-
-    async def unregister_queue(self, queue: asyncio.Queue):
-        raise NotImplementedError()
+    def get_log_queue(self) -> asyncio.Queue:
+        queue, canceller = make_logs_queue(self._journal_stdout_name, self._journal_stderr_name)
+        self._queue_cancellers[queue] = canceller
+        # Limit the number of queues per VM
+        # TODO : fix
+        if len(self.log_queues) > 20:
+            logger.warning("Too many log queues, dropping the oldest one")
+            self.unregister_queue(self.log_queues[1])
+        self.log_queues.append(queue)
+        return queue
+
+    def unregister_queue(self, queue: asyncio.Queue) -> None:
+        if queue in self.log_queues:
+            self._queue_cancellers[queue]()
+            del self._queue_cancellers[queue]
+            self.log_queues.remove(queue)
+        queue.empty()
+
+
+    @property
+    def _journal_stdout_name(self) -> str:
+        return f"vm-{self.vm_hash}-stdout"
+
+    @property
+    def _journal_stderr_name(self) -> str:
+        return f"vm-{self.vm_hash}-stderr"
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index d56d9d0c3..d23c2cb4b 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -1,15 +1,13 @@
-import asyncio
 import json
 import logging
 import shutil
 from asyncio import Task
 from asyncio.subprocess import Process
 from pathlib import Path
-from typing import Callable, Generic, Optional, TypeVar, Union
+from typing import Generic, Optional, TypeVar, Union
 
 import psutil
 
-from aleph.vm.utils.logs import make_logs_queue
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
 from aleph_message.models.execution.instance import RootfsVolume
@@ -101,7 +99,6 @@ class AlephQemuInstance(Generic[ConfigurationType], CloudInitMixin, AlephVmContr
     support_snapshot = False
     qmp_socket_path = None
     persistent = True
-    _queue_cancellers: dict[asyncio.Queue, Callable] = {}
     controller_configuration: Configuration
 
     def __repr__(self):
@@ -121,7 +118,6 @@ def __init__(
         tap_interface: Optional[TapInterface] = None,
     ):
         self.vm_id = vm_id
-        self.vm_hash = vm_hash
         self.resources = resources
         if enable_console is None:
             enable_console = settings.PRINT_SYSTEM_LOGS
@@ -131,6 +127,8 @@ def __init__(
         self.tap_interface = tap_interface
         self.qemu_process = None
 
+        self.vm_hash = vm_hash
+
     # TODO : wait for andress soltion for pid handling
     def to_dict(self):
         """Dict representation of the virtual machine. Used to record resource usage and for JSON serialization."""
@@ -210,14 +208,6 @@ def save_controller_configuration(self):
         path.chmod(0o644)
         return path
 
-    @property
-    def _journal_stdout_name(self) -> str:
-        return f"vm-{self.vm_hash}-stdout"
-
-    @property
-    def _journal_stderr_name(self) -> str:
-        return f"vm-{self.vm_hash}-stderr"
-
     async def start(self):
         # Start via systemd not here
         raise NotImplementedError()
@@ -253,7 +243,6 @@ async def stop_guest_api(self):
         pass
 
     print_task: Optional[Task] = None
-    log_queues: list[asyncio.Queue] = []
 
     async def teardown(self):
         if self.print_task:
@@ -268,21 +257,3 @@ async def teardown(self):
     def print_logs(self) -> None:
         """Print logs to our output for debugging"""
         queue = self.get_log_queue()
-
-    def get_log_queue(self) -> asyncio.Queue:
-        queue, canceller = make_logs_queue(self._journal_stdout_name, self._journal_stderr_name)
-        self._queue_cancellers[queue] = canceller
-        # Limit the number of queues per VM
-        # TODO : fix
-        if len(self.log_queues) > 20:
-            logger.warning("Too many log queues, dropping the oldest one")
-            self.unregister_queue(self.log_queues[1])
-        self.log_queues.append(queue)
-        return queue
-
-    def unregister_queue(self, queue: asyncio.Queue) -> None:
-        if queue in self.log_queues:
-            self._queue_cancellers[queue]()
-            del self._queue_cancellers[queue]
-            self.log_queues.remove(queue)
-        queue.empty()
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 888aeae1b..278103fd6 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -13,10 +13,12 @@
 from pathlib import Path
 from pwd import getpwnam
 from tempfile import NamedTemporaryFile
-from typing import Any, Optional
+from typing import Any, Optional, TextIO
 
 import msgpack
+from systemd import journal
 
+from aleph_message.models import ItemHash
 from .config import Drive, FirecrackerConfig
 
 logger = logging.getLogger(__name__)
@@ -84,7 +86,6 @@ class MicroVM:
     proc: Optional[asyncio.subprocess.Process] = None
     stdout_task: Optional[Task] = None
     stderr_task: Optional[Task] = None
-    log_queues: list[asyncio.Queue]
     config_file_path: Optional[Path] = None
     drives: list[Drive]
     init_timeout: float
@@ -124,6 +125,7 @@ def vsock_path(self) -> str:
     def __init__(
         self,
         vm_id: int,
+        vm_hash: ItemHash,
         firecracker_bin_path: Path,
         jailer_base_directory: Path,
         use_jailer: bool = True,
@@ -131,6 +133,7 @@ def __init__(
         init_timeout: float = 5.0,
     ):
         self.vm_id = vm_id
+        self.vm_hash = vm_hash
         self.use_jailer = use_jailer
         self.jailer_base_directory = jailer_base_directory
         self.firecracker_bin_path = firecracker_bin_path
@@ -138,7 +141,6 @@ def __init__(
         self.drives = []
         self.init_timeout = init_timeout
         self.runtime_config = None
-        self.log_queues: list[asyncio.Queue] = []
 
     def to_dict(self) -> dict:
         return {
@@ -213,16 +215,27 @@ async def start_firecracker(self, config_path: Path) -> asyncio.subprocess.Proce
             str(config_path),
         )
 
+        journal_stdout: TextIO = journal.stream(self._journal_stdout_name)
+        journal_stderr: TextIO = journal.stream(self._journal_stderr_name)
+
         logger.debug(" ".join(options))
 
         self.proc = await asyncio.create_subprocess_exec(
             *options,
             stdin=asyncio.subprocess.PIPE,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
+            stdout=journal_stdout,
+            stderr=journal_stderr,
         )
         return self.proc
 
+    @property
+    def _journal_stdout_name(self) -> str:
+        return f"vm-{self.vm_hash}-stdout"
+
+    @property
+    def _journal_stderr_name(self) -> str:
+        return f"vm-{self.vm_hash}-stderr"
+
     async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subprocess.Process:
         if not self.jailer_bin_path:
             msg = "Jailer binary path is missing"
@@ -231,6 +244,8 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces
         gid = str(getpwnam("jailman").pw_gid)
 
         self.config_file_path = config_path
+        journal_stdout: TextIO = journal.stream(self._journal_stdout_name)
+        journal_stderr: TextIO = journal.stream(self._journal_stderr_name)
 
         options = (
             str(self.jailer_bin_path),
@@ -254,8 +269,8 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces
         self.proc = await asyncio.create_subprocess_exec(
             *options,
             stdin=asyncio.subprocess.PIPE,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
+            stdout=journal_stdout,
+            stderr=journal_stderr,
         )
         return self.proc
 
@@ -361,43 +376,6 @@ def enable_drive(self, drive_path: Path, read_only: bool = True) -> Drive:
         self.drives.append(drive)
         return drive
 
-    async def print_logs(self):
-        while not self.proc:
-            await asyncio.sleep(0.01)  # Todo: Use signal here
-        while True:
-            assert self.proc.stdout, "Process stdout is missing"
-            line = await self.proc.stdout.readline()
-            if not line:  # EOF, FD is closed nothing more will come
-                return
-            for queue in self.log_queues:
-                if queue.full():
-                    logger.warning("Log queue is full")
-                else:
-                    await queue.put(("stdout", line))
-            print(self, line.decode().strip())
-
-    async def print_logs_stderr(self):
-        while not self.proc:
-            await asyncio.sleep(0.01)  # Todo: Use signal here
-        while True:
-            assert self.proc.stderr, "Process stderr is missing"
-            line = await self.proc.stderr.readline()
-            if not line:  # EOF, FD is closed nothing more will come
-                return
-            for queue in self.log_queues:
-                if queue.full():
-                    logger.warning("Log queue is full")
-                else:
-                    await queue.put(("stderr", line))
-                await queue.put(("stderr", line))
-            print(self, line.decode().strip(), file=sys.stderr)
-
-    def start_printing_logs(self) -> tuple[Task, Task]:
-        loop = asyncio.get_running_loop()
-        self.stdout_task = loop.create_task(self.print_logs())
-        self.stderr_task = loop.create_task(self.print_logs_stderr())
-        return self.stdout_task, self.stderr_task
-
     async def wait_for_init(self) -> None:
         """Wait for a connection from the init in the VM"""
         logger.debug("Waiting for init...")
diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index c20c17b77..b4e577dde 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -22,7 +22,6 @@ class QemuVM:
     mem_size_mb: int
     interface_name: str
     qemu_process = None
-    log_queues: list[asyncio.Queue]
 
     def __repr__(self) -> str:
         if self.qemu_process:
@@ -39,7 +38,6 @@ def __init__(self, vm_hash, config: QemuVMConfiguration):
         self.vcpu_count = config.vcpu_count
         self.mem_size_mb = config.mem_size_mb
         self.interface_name = config.interface_name
-        self.log_queues = []
         self.vm_hash = vm_hash
 
     @property

From b5a70c30818c30d47385aff23f3fc4bf8ee561be Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 21 Jun 2024 12:06:18 +0200
Subject: [PATCH 800/990] Make Firecracker program respect the log settings

---
 .../vm/controllers/firecracker/executable.py  |  1 +
 .../vm/hypervisors/firecracker/microvm.py     | 20 ++++++++++++++-----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index e8c463920..1d9fe6360 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -194,6 +194,7 @@ def __init__(
             use_jailer=settings.USE_JAILER,
             jailer_bin_path=settings.JAILER_PATH,
             init_timeout=settings.INIT_TIMEOUT,
+            enable_log=enable_console,
         )
         if prepare_jailer:
             self.fvm.prepare_jailer()
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 278103fd6..3439814ce 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -92,6 +92,7 @@ class MicroVM:
     runtime_config: Optional[RuntimeConfiguration]
     mounted_rootfs: Optional[Path] = None
     _unix_socket: Optional[Server] = None
+    enable_log: bool
 
     def __repr__(self):
         return f""
@@ -131,6 +132,7 @@ def __init__(
         use_jailer: bool = True,
         jailer_bin_path: Optional[Path] = None,
         init_timeout: float = 5.0,
+        enable_log: bool = True,
     ):
         self.vm_id = vm_id
         self.vm_hash = vm_hash
@@ -141,6 +143,7 @@ def __init__(
         self.drives = []
         self.init_timeout = init_timeout
         self.runtime_config = None
+        self.enable_log = enable_log
 
     def to_dict(self) -> dict:
         return {
@@ -214,9 +217,12 @@ async def start_firecracker(self, config_path: Path) -> asyncio.subprocess.Proce
             "--config-file",
             str(config_path),
         )
-
-        journal_stdout: TextIO = journal.stream(self._journal_stdout_name)
-        journal_stderr: TextIO = journal.stream(self._journal_stderr_name)
+        if self.enable_log:
+            journal_stdout: Optional[TextIO] = journal.stream(self._journal_stdout_name)
+            journal_stderr: Optional[TextIO] = journal.stream(self._journal_stderr_name)
+        else:
+            journal_stdout = None
+            journal_stderr = None
 
         logger.debug(" ".join(options))
 
@@ -244,8 +250,12 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces
         gid = str(getpwnam("jailman").pw_gid)
 
         self.config_file_path = config_path
-        journal_stdout: TextIO = journal.stream(self._journal_stdout_name)
-        journal_stderr: TextIO = journal.stream(self._journal_stderr_name)
+        if self.enable_log:
+            journal_stdout: Optional[TextIO] = journal.stream(self._journal_stdout_name)
+            journal_stderr: Optional[TextIO] = journal.stream(self._journal_stderr_name)
+        else:
+            journal_stdout = None
+            journal_stderr = None
 
         options = (
             str(self.jailer_bin_path),

From dc2004de4d380525127cb4c202b13dd3264f69e6 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 21 Jun 2024 14:21:01 +0200
Subject: [PATCH 801/990] unused

---
 src/aleph/vm/controllers/qemu/instance.py | 5 -----
 tests/supervisor/test_qemu_instance.py    | 2 --
 2 files changed, 7 deletions(-)

diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index d23c2cb4b..e9539d59b 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -89,7 +89,6 @@ class AlephQemuInstance(Generic[ConfigurationType], CloudInitMixin, AlephVmContr
     vm_id: int
     vm_hash: ItemHash
     resources: AlephQemuResources
-    enable_console: bool
     enable_networking: bool
     hardware_resources: MachineResources
     tap_interface: Optional[TapInterface] = None
@@ -113,15 +112,11 @@ def __init__(
         vm_hash: ItemHash,
         resources: AlephQemuResources,
         enable_networking: bool = False,
-        enable_console: Optional[bool] = None,
         hardware_resources: MachineResources = MachineResources(),
         tap_interface: Optional[TapInterface] = None,
     ):
         self.vm_id = vm_id
         self.resources = resources
-        if enable_console is None:
-            enable_console = settings.PRINT_SYSTEM_LOGS
-        self.enable_console = enable_console
         self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING
         self.hardware_resources = hardware_resources
         self.tap_interface = tap_interface
diff --git a/tests/supervisor/test_qemu_instance.py b/tests/supervisor/test_qemu_instance.py
index 3792aaa87..0d7f08fcd 100644
--- a/tests/supervisor/test_qemu_instance.py
+++ b/tests/supervisor/test_qemu_instance.py
@@ -57,7 +57,6 @@ async def test_create_qemu_instance():
     settings.USE_JAILER = False
 
     logging.basicConfig(level=logging.DEBUG)
-    settings.PRINT_SYSTEM_LOGS = True
 
     # Ensure that the settings are correct and required files present.
     settings.setup()
@@ -112,7 +111,6 @@ async def test_create_qemu_instance_online():
     settings.USE_JAILER = False
 
     logging.basicConfig(level=logging.DEBUG)
-    settings.PRINT_SYSTEM_LOGS = True
 
     # Ensure that the settings are correct and required files present.
     settings.setup()

From ed28be7f8a74a718f79f13a853e9a5798016dc89 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 21 Jun 2024 14:44:40 +0200
Subject: [PATCH 802/990] connect to /dev/null if not printing

---
 src/aleph/vm/controllers/interface.py          |  5 ++---
 src/aleph/vm/controllers/qemu/instance.py      |  1 -
 .../vm/hypervisors/firecracker/microvm.py      | 18 +++++++++---------
 src/aleph/vm/utils/logs.py                     |  2 +-
 4 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/src/aleph/vm/controllers/interface.py b/src/aleph/vm/controllers/interface.py
index 5ee710663..efce726e2 100644
--- a/src/aleph/vm/controllers/interface.py
+++ b/src/aleph/vm/controllers/interface.py
@@ -3,14 +3,14 @@
 from abc import ABC
 from asyncio.subprocess import Process
 from collections.abc import Coroutine
-from typing import Any, Optional, Callable
+from typing import Any, Callable, Optional
 
-from aleph.vm.utils.logs import make_logs_queue
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
 
 from aleph.vm.controllers.firecracker.snapshots import CompressedDiskVolumeSnapshot
 from aleph.vm.network.interfaces import TapInterface
+from aleph.vm.utils.logs import make_logs_queue
 
 logger = logging.getLogger(__name__)
 
@@ -111,7 +111,6 @@ def unregister_queue(self, queue: asyncio.Queue) -> None:
             self.log_queues.remove(queue)
         queue.empty()
 
-
     @property
     def _journal_stdout_name(self) -> str:
         return f"vm-{self.vm_hash}-stdout"
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index e9539d59b..2353162b6 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -7,7 +7,6 @@
 from typing import Generic, Optional, TypeVar, Union
 
 import psutil
-
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
 from aleph_message.models.execution.instance import RootfsVolume
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 3439814ce..8d66ac257 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -16,9 +16,9 @@
 from typing import Any, Optional, TextIO
 
 import msgpack
+from aleph_message.models import ItemHash
 from systemd import journal
 
-from aleph_message.models import ItemHash
 from .config import Drive, FirecrackerConfig
 
 logger = logging.getLogger(__name__)
@@ -218,11 +218,11 @@ async def start_firecracker(self, config_path: Path) -> asyncio.subprocess.Proce
             str(config_path),
         )
         if self.enable_log:
-            journal_stdout: Optional[TextIO] = journal.stream(self._journal_stdout_name)
-            journal_stderr: Optional[TextIO] = journal.stream(self._journal_stderr_name)
+            journal_stdout = journal.stream(self._journal_stdout_name)
+            journal_stderr = journal.stream(self._journal_stderr_name)
         else:
-            journal_stdout = None
-            journal_stderr = None
+            journal_stdout = asyncio.subprocess.DEVNULL
+            journal_stderr = asyncio.subprocess.DEVNULL
 
         logger.debug(" ".join(options))
 
@@ -251,11 +251,11 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces
 
         self.config_file_path = config_path
         if self.enable_log:
-            journal_stdout: Optional[TextIO] = journal.stream(self._journal_stdout_name)
-            journal_stderr: Optional[TextIO] = journal.stream(self._journal_stderr_name)
+            journal_stdout = journal.stream(self._journal_stdout_name)
+            journal_stderr = journal.stream(self._journal_stderr_name)
         else:
-            journal_stdout = None
-            journal_stderr = None
+            journal_stdout = asyncio.subprocess.DEVNULL
+            journal_stderr = asyncio.subprocess.DEVNULL
 
         options = (
             str(self.jailer_bin_path),
diff --git a/src/aleph/vm/utils/logs.py b/src/aleph/vm/utils/logs.py
index 6baca5268..d95adbac6 100644
--- a/src/aleph/vm/utils/logs.py
+++ b/src/aleph/vm/utils/logs.py
@@ -1,6 +1,6 @@
 import asyncio
 import logging
-from typing import TypedDict, Callable
+from typing import Callable, TypedDict
 
 from systemd import journal
 

From 4f9b3e6cedb7e3b36ee6d2b6c93b9506e0057bf4 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 27 Jun 2024 12:46:52 +0200
Subject: [PATCH 803/990] Fix: Path to `sevctl` was not from settings (#637)

Fix: Path to `sevctl` was not from settings
---
 src/aleph/vm/conf.py                    |  3 +--
 src/aleph/vm/orchestrator/supervisor.py |  2 +-
 src/aleph/vm/sevclient.py               | 15 +++++++++++----
 tests/supervisor/test_views.py          | 10 ++++++----
 4 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 48c78248f..7743c44da 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -388,8 +388,7 @@ def check(self):
             assert (
                 check_system_module("kvm_amd/parameters/sev_es") == "Y"
             ), "SEV-ES feature isn't enabled, enable it in BIOS"
-            assert is_command_available("sevctl"), "Command `sevctl` not found, run `cargo install sevctl`"
-
+            assert self.SEV_CTL_PATH.is_file(), f"File not found {self.SEV_CTL_PATH}"
             assert self.ENABLE_QEMU_SUPPORT, "Qemu Support is needed for confidential computing and it's disabled, "
             "enable it setting the env variable `ENABLE_QEMU_SUPPORT=True` in configuration"
 
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index a2a712445..31f7fe42f 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -163,7 +163,7 @@ def run():
 
     # Store sevctl app singleton only if confidential feature is enabled
     if settings.ENABLE_CONFIDENTIAL_COMPUTING:
-        sev_client = SevClient(settings.CONFIDENTIAL_DIRECTORY)
+        sev_client = SevClient(settings.CONFIDENTIAL_DIRECTORY, settings.SEV_CTL_PATH)
         app["sev_client"] = sev_client
         # TODO: Review and check sevctl first initialization steps, like (sevctl generate and sevctl provision)
 
diff --git a/src/aleph/vm/sevclient.py b/src/aleph/vm/sevclient.py
index fe9eb1c00..a5b4ed154 100644
--- a/src/aleph/vm/sevclient.py
+++ b/src/aleph/vm/sevclient.py
@@ -4,20 +4,27 @@
 
 
 class SevClient:
-    def __init__(self, sev_dir: Path):
+    sev_dir: Path
+    sev_ctl_executable: Path
+    certificates_dir: Path
+    certificates_archive: Path
+
+    def __init__(self, sev_dir: Path, sev_ctl_executable: Path):
         self.sev_dir = sev_dir
+        self.sev_ctl_executable = sev_ctl_executable
         self.certificates_dir = sev_dir / "platform"
         self.certificates_dir.mkdir(exist_ok=True, parents=True)
         self.certificates_archive = self.certificates_dir / "certs_export.cert"
 
-    async def sevctl_cmd(self, *args) -> bytes:
+    async def sev_ctl_cmd(self, *args) -> bytes:
+        """Run a command of the 'sevctl' tool."""
         return await run_in_subprocess(
-            ["sevctl", *args],
+            [self.sev_ctl_executable, *args],
             check=True,
         )
 
     async def get_certificates(self) -> Path:
         if not self.certificates_archive.is_file():
-            _ = await self.sevctl_cmd("export", str(self.certificates_archive))
+            _ = await self.sev_ctl_cmd("export", str(self.certificates_archive))
 
         return self.certificates_archive
diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
index 52426d48c..4e1f2746f 100644
--- a/tests/supervisor/test_views.py
+++ b/tests/supervisor/test_views.py
@@ -1,5 +1,5 @@
 import tempfile
-from pathlib import Path
+from pathlib import Path, PosixPath
 from unittest import mock
 from unittest.mock import call
 
@@ -135,7 +135,7 @@ async def test_about_certificates_missing_setting(aiohttp_client):
     settings.ENABLE_CONFIDENTIAL_COMPUTING = False
 
     app = setup_webapp()
-    app["sev_client"] = SevClient(Path().resolve())
+    app["sev_client"] = SevClient(Path().resolve(), Path("/opt/sevctl").resolve())
     client = await aiohttp_client(app)
     response: web.Response = await client.get("/about/certificates")
     assert response.status == 400
@@ -160,7 +160,7 @@ async def test_about_certificates(aiohttp_client):
         ) as export_mock:
             with tempfile.TemporaryDirectory() as tmp_dir:
                 app = setup_webapp()
-                sev_client = SevClient(Path(tmp_dir))
+                sev_client = SevClient(Path(tmp_dir), Path("/opt/sevctl"))
                 app["sev_client"] = sev_client
                 # Create mock file to return it
                 Path(sev_client.certificates_archive).touch(exist_ok=True)
@@ -170,4 +170,6 @@ async def test_about_certificates(aiohttp_client):
                 assert response.status == 200
                 is_file_mock.assert_has_calls([call(), call()])
                 certificates_expected_dir = sev_client.certificates_archive
-                export_mock.assert_called_once_with(["sevctl", "export", str(certificates_expected_dir)], check=True)
+                export_mock.assert_called_once_with(
+                    [PosixPath("/opt/sevctl"), "export", str(certificates_expected_dir)], check=True
+                )

From 45c14c501f3eaa10506561ac597f00e8cf13f64b Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 27 Jun 2024 17:43:54 +0200
Subject: [PATCH 804/990] Fix: CRN API did not expose CPU features for trusted
 computing (#622)

Trusted computing requires CPU features such as `sev`, `sev_es` and `sev_snp`.

This adds the field `properties.cpu.features` `/about/usage/system` as a list of CPU features.

Currently, only SEV related features are present, but more can be added, for example `avx2`, `fma` and `f16c`. Adding them will require ensuring that they are actually active and not just present on the CPU via `/proc/cpuinfo`.

This work is based on a proposal to add the relevant field on aleph-message: https://github.com/aleph-im/aleph-message/pull/100
---
 docker/vm_supervisor-dev.dockerfile    |  2 +-
 examples/volumes/Dockerfile            |  2 +-
 packaging/Makefile                     |  2 +-
 pyproject.toml                         |  2 +-
 src/aleph/vm/conf.py                   | 15 +++++++++-----
 src/aleph/vm/orchestrator/README.md    |  2 +-
 src/aleph/vm/orchestrator/resources.py | 17 +++++++++++++++-
 src/aleph/vm/utils/__init__.py         | 28 ++++++++++++++++++++++++++
 tests/supervisor/test_utils.py         | 24 ++++++++++++++++++++--
 9 files changed, 81 insertions(+), 13 deletions(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index da730aca8..77718f3da 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -19,7 +19,7 @@ RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://s3.amazonaws.com/spec.ccf
 RUN ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker
 RUN ln /opt/firecracker/release-*/jailer-v* /opt/firecracker/jailer
 
-RUN pip3 install typing-extensions 'aleph-message==0.4.4'
+RUN pip3 install typing-extensions 'aleph-message==0.4.7'
 
 RUN mkdir -p /var/lib/aleph/vm/jailer
 
diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile
index c5e67993f..f3aad1e18 100644
--- a/examples/volumes/Dockerfile
+++ b/examples/volumes/Dockerfile
@@ -6,6 +6,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 RUN python3 -m venv /opt/venv
-RUN /opt/venv/bin/pip install 'aleph-message==0.4.4'
+RUN /opt/venv/bin/pip install 'aleph-message==0.4.7'
 
 CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs
diff --git a/packaging/Makefile b/packaging/Makefile
index a1df5ecbd..8b497da13 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.4' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.7' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo  target/bin/sevctl
diff --git a/pyproject.toml b/pyproject.toml
index 067de413b..4a6d57148 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,7 +29,7 @@ dependencies = [
   "aiodns==3.1.0",
   "setproctitle==1.3.3",
   "pyyaml==6.0.1",
-  "aleph-message==0.4.4",
+  "aleph-message==0.4.7",
   "eth-account~=0.10",
   "sentry-sdk==1.31.0",
   "aioredis==1.3.1",
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 7743c44da..86f83cca8 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -16,7 +16,12 @@
 from pydantic.env_settings import DotenvType, env_file_sentinel
 from pydantic.typing import StrPath
 
-from aleph.vm.utils import check_system_module, file_hashes_differ, is_command_available
+from aleph.vm.utils import (
+    check_amd_sev_es_supported,
+    check_amd_sev_supported,
+    file_hashes_differ,
+    is_command_available,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -384,11 +389,11 @@ def check(self):
             ), "Command `qemu-system-x86_64` not found, run `apt install qemu-system-x86`"
 
         if self.ENABLE_CONFIDENTIAL_COMPUTING:
-            assert check_system_module("kvm_amd/parameters/sev") == "Y", "SEV feature isn't enabled, enable it in BIOS"
-            assert (
-                check_system_module("kvm_amd/parameters/sev_es") == "Y"
-            ), "SEV-ES feature isn't enabled, enable it in BIOS"
             assert self.SEV_CTL_PATH.is_file(), f"File not found {self.SEV_CTL_PATH}"
+            assert check_amd_sev_supported(), "SEV feature isn't enabled, enable it in BIOS"
+            assert check_amd_sev_es_supported(), "SEV-ES feature isn't enabled, enable it in BIOS"
+            # Not available on the test machine yet
+            # assert check_amd_sev_snp_supported(), "SEV-SNP feature isn't enabled, enable it in BIOS"
             assert self.ENABLE_QEMU_SUPPORT, "Qemu Support is needed for confidential computing and it's disabled, "
             "enable it setting the env variable `ENABLE_QEMU_SUPPORT=True` in configuration"
 
diff --git a/src/aleph/vm/orchestrator/README.md b/src/aleph/vm/orchestrator/README.md
index a9d9a3136..95430423a 100644
--- a/src/aleph/vm/orchestrator/README.md
+++ b/src/aleph/vm/orchestrator/README.md
@@ -86,7 +86,7 @@ is used to parse and validate Aleph messages.
 ```shell
 apt install -y --no-install-recommends --no-install-suggests python3-pip
 pip3 install pydantic[dotenv]
-pip3 install 'aleph-message==0.4.4'
+pip3 install 'aleph-message==0.4.7'
 ```
 
 ### 2.f. Create the jailer working directory:
diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index fe9deab26..0f6ba0966 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -12,7 +12,12 @@
 
 from aleph.vm.conf import settings
 from aleph.vm.sevclient import SevClient
-from aleph.vm.utils import cors_allow_all
+from aleph.vm.utils import (
+    check_amd_sev_es_supported,
+    check_amd_sev_snp_supported,
+    check_amd_sev_supported,
+    cors_allow_all,
+)
 
 
 class Period(BaseModel):
@@ -90,6 +95,16 @@ def get_machine_properties() -> MachineProperties:
         cpu=CpuProperties(
             architecture=cpu_info.get("raw_arch_string", cpu_info.get("arch_string_raw")),
             vendor=cpu_info.get("vendor_id", cpu_info.get("vendor_id_raw")),
+            features=list(
+                filter(
+                    None,
+                    (
+                        "sev" if check_amd_sev_supported() else None,
+                        "sev_es" if check_amd_sev_es_supported() else None,
+                        "sev_snp" if check_amd_sev_snp_supported() else None,
+                    ),
+                )
+            ),
         ),
     )
 
diff --git a/src/aleph/vm/utils/__init__.py b/src/aleph/vm/utils/__init__.py
index 6c114253e..35bbf2ff7 100644
--- a/src/aleph/vm/utils/__init__.py
+++ b/src/aleph/vm/utils/__init__.py
@@ -137,6 +137,34 @@ def check_system_module(module_path: str) -> Optional[str]:
     return p.read_text().strip()
 
 
+def check_amd_sev_supported() -> bool:
+    """Check if AMD SEV is supported on the system.
+
+    AMD Secure Encrypted Virtualization (SEV)
+    Uses one key per virtual machine to isolate guests and the hypervisor from one another.
+    """
+    return (check_system_module("kvm_amd/parameters/sev") == "Y") and Path("/dev/sev").exists()
+
+
+def check_amd_sev_es_supported() -> bool:
+    """Check if AMD SEV-ES is supported on the system.
+
+    AMD Secure Encrypted Virtualization-Encrypted State (SEV-ES)
+    Encrypts all CPU register contents when a VM stops running.
+    """
+    return (check_system_module("kvm_amd/parameters/sev_es") == "Y") and Path("/dev/sev").exists()
+
+
+def check_amd_sev_snp_supported() -> bool:
+    """Check if AMD SEV-SNP is supported on the system.
+
+    AMD Secure Encrypted Virtualization-Secure Nested Paging (SEV-SNP)
+    Adds strong memory integrity protection to help prevent malicious hypervisor-based attacks like data replay,
+    memory re-mapping, and more in order to create an isolated execution environment.
+    """
+    return check_system_module("kvm_amd/parameters/sev_snp") == "Y"
+
+
 def fix_message_validation(message: dict) -> dict:
     """Patch a fake message program to pass validation."""
     message["item_content"] = json.dumps(message["content"])
diff --git a/tests/supervisor/test_utils.py b/tests/supervisor/test_utils.py
index 8b67fe1ef..0451d9607 100644
--- a/tests/supervisor/test_utils.py
+++ b/tests/supervisor/test_utils.py
@@ -1,6 +1,11 @@
 from unittest import mock
 
-from aleph.vm.utils import check_system_module
+from aleph.vm.utils import (
+    check_amd_sev_es_supported,
+    check_amd_sev_snp_supported,
+    check_amd_sev_supported,
+    check_system_module,
+)
 
 
 def test_check_system_module_enabled():
@@ -11,9 +16,24 @@ def test_check_system_module_enabled():
     ):
         expected_value = "Y"
         with mock.patch(
-            "pathlib.Path.open",
+            "aleph.vm.utils.Path.open",
             mock.mock_open(read_data=expected_value),
         ):
 
             output = check_system_module("kvm_amd/parameters/sev_enp")
             assert output == expected_value
+
+            assert check_amd_sev_supported() is True
+            assert check_amd_sev_es_supported() is True
+            assert check_amd_sev_snp_supported() is True
+
+        with mock.patch(
+            "aleph.vm.utils.Path.open",
+            mock.mock_open(read_data="N"),
+        ):
+            output = check_system_module("kvm_amd/parameters/sev_enp")
+            assert output == "N"
+
+            assert check_amd_sev_supported() is False
+            assert check_amd_sev_es_supported() is False
+            assert check_amd_sev_snp_supported() is False

From 4a9eabd157ec99edfb3a16ce2f62c28c29a9efcb Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 4 Jul 2024 08:55:12 +0200
Subject: [PATCH 805/990] Problem: Websocket were required to fetch logs (#645)
 Endpoint renamed

Solution: Provide a view that works with GET


*Breaking change* : the previous logs endpoint has be renamed to `stream_logs`

Renamed ~/control/machine/{ref}/logs` => `/control/machine/{ref}/stream_logs` For websocket
New endpoint: `/control/machine/{ref}/logs` For GET

Add test for Webocket log endpoint, new endpoint, test authentification properly for the websocket endpoint
---
 src/aleph/vm/controllers/interface.py       |   5 +-
 src/aleph/vm/controllers/qemu/instance.py   |   4 -
 src/aleph/vm/orchestrator/supervisor.py     |   4 +-
 src/aleph/vm/orchestrator/views/operator.py |  30 ++-
 src/aleph/vm/utils/logs.py                  |  26 ++-
 src/aleph/vm/utils/test_helpers.py          |  86 +++++++
 tests/supervisor/test_authentication.py     |  81 +------
 tests/supervisor/views/test_operator.py     | 236 ++++++++++++++++++++
 8 files changed, 390 insertions(+), 82 deletions(-)
 create mode 100644 src/aleph/vm/utils/test_helpers.py

diff --git a/src/aleph/vm/controllers/interface.py b/src/aleph/vm/controllers/interface.py
index efce726e2..d5a290173 100644
--- a/src/aleph/vm/controllers/interface.py
+++ b/src/aleph/vm/controllers/interface.py
@@ -10,7 +10,7 @@
 
 from aleph.vm.controllers.firecracker.snapshots import CompressedDiskVolumeSnapshot
 from aleph.vm.network.interfaces import TapInterface
-from aleph.vm.utils.logs import make_logs_queue
+from aleph.vm.utils.logs import get_past_vm_logs, make_logs_queue
 
 logger = logging.getLogger(__name__)
 
@@ -118,3 +118,6 @@ def _journal_stdout_name(self) -> str:
     @property
     def _journal_stderr_name(self) -> str:
         return f"vm-{self.vm_hash}-stderr"
+
+    def past_logs(self):
+        yield from get_past_vm_logs(self._journal_stdout_name, self._journal_stderr_name)
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index 2353162b6..81dea8ff3 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -247,7 +247,3 @@ async def teardown(self):
             if self.tap_interface:
                 await self.tap_interface.delete()
         await self.stop_guest_api()
-
-    def print_logs(self) -> None:
-        """Print logs to our output for debugging"""
-        queue = self.get_log_queue()
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 31f7fe42f..a3905cae2 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -49,6 +49,7 @@
 from .views.operator import (
     operate_erase,
     operate_expire,
+    operate_logs,
     operate_reboot,
     operate_stop,
     stream_logs,
@@ -100,7 +101,8 @@ def setup_webapp():
         web.get("/about/config", about_config),
         # /control APIs are used to control the VMs and access their logs
         web.post("/control/allocation/notify", notify_allocation),
-        web.get("/control/machine/{ref}/logs", stream_logs),
+        web.get("/control/machine/{ref}/stream_logs", stream_logs),
+        web.get("/control/machine/{ref}/logs", operate_logs),
         web.post("/control/machine/{ref}/expire", operate_expire),
         web.post("/control/machine/{ref}/stop", operate_stop),
         web.post("/control/machine/{ref}/erase", operate_erase),
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 148ecd092..cd8fbae14 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -65,6 +65,7 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
     queue = None
     try:
         ws = web.WebSocketResponse()
+        logger.info(f"starting websocket: {request.path}")
         await ws.prepare(request)
         try:
             await authenticate_websocket_for_vm_or_403(execution, vm_hash, ws)
@@ -75,6 +76,7 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
             while True:
                 log_type, message = await queue.get()
                 assert log_type in ("stdout", "stderr")
+                logger.debug(message)
 
                 await ws.send_json({"type": log_type, "message": message})
 
@@ -87,15 +89,41 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
             execution.vm.unregister_queue(queue)
 
 
+@cors_allow_all
+@require_jwk_authentication
+async def operate_logs(request: web.Request, authenticated_sender: str) -> web.StreamResponse:
+    """Logs of a VM (not streaming)"""
+    vm_hash = get_itemhash_or_400(request.match_info)
+    pool: VmPool = request.app["vm_pool"]
+    execution = get_execution_or_404(vm_hash, pool=pool)
+    if not is_sender_authorized(authenticated_sender, execution.message):
+        return web.Response(status=403, body="Unauthorized sender")
+
+    response = web.StreamResponse()
+    response.headers["Content-Type"] = "text/plain"
+    await response.prepare(request)
+
+    for entry in execution.vm.past_logs():
+        msg = f'{entry["__REALTIME_TIMESTAMP"].isoformat()}> {entry["MESSAGE"]}'
+        await response.write(msg.encode())
+    await response.write_eof()
+    return response
+
+
 async def authenticate_websocket_for_vm_or_403(execution: VmExecution, vm_hash: ItemHash, ws: web.WebSocketResponse):
     """Authenticate a websocket connection.
 
     Web browsers do not allow setting headers in WebSocket requests, so the authentication
     relies on the first message sent by the client.
     """
-    first_message = await ws.receive_json()
+    try:
+        first_message = await ws.receive_json()
+    except TypeError as error:
+        logging.exception(error)
+        raise web.HTTPForbidden(body="Invalid auth package")
     credentials = first_message["auth"]
     authenticated_sender = await authenticate_websocket_message(credentials)
+
     if is_sender_authorized(authenticated_sender, execution.message):
         logger.debug(f"Accepted request to access logs by {authenticated_sender} on {vm_hash}")
         return True
diff --git a/src/aleph/vm/utils/logs.py b/src/aleph/vm/utils/logs.py
index d95adbac6..a112cfc0a 100644
--- a/src/aleph/vm/utils/logs.py
+++ b/src/aleph/vm/utils/logs.py
@@ -1,6 +1,7 @@
 import asyncio
 import logging
-from typing import Callable, TypedDict
+from datetime import datetime
+from typing import Callable, Generator, TypedDict
 
 from systemd import journal
 
@@ -10,6 +11,7 @@
 class EntryDict(TypedDict):
     SYSLOG_IDENTIFIER: str
     MESSAGE: str
+    __REALTIME_TIMESTAMP: datetime
 
 
 def make_logs_queue(stdout_identifier, stderr_identifier, skip_past=False) -> tuple[asyncio.Queue, Callable[[], None]]:
@@ -56,3 +58,25 @@ def do_cancel():
         r.close()
 
     return queue, do_cancel
+
+
+def get_past_vm_logs(stdout_identifier, stderr_identifier) -> Generator[EntryDict, None, None]:
+    """Get existing log for the VM identifiers.
+
+    @param stdout_identifier: journald identifier for process stdout
+    @param stderr_identifier: journald identifier for process stderr
+    @return: an iterator of log entry
+
+    Works by creating a journald reader, and using `add_reader` to call a callback when
+    data is available for reading.
+
+    For more information refer to the sd-journal(3) manpage
+    and systemd.journal module documentation.
+    """
+    r = journal.Reader()
+    r.add_match(SYSLOG_IDENTIFIER=stdout_identifier)
+    r.add_match(SYSLOG_IDENTIFIER=stderr_identifier)
+
+    r.seek_head()
+    for entry in r:
+        yield entry
diff --git a/src/aleph/vm/utils/test_helpers.py b/src/aleph/vm/utils/test_helpers.py
new file mode 100644
index 000000000..ecdf4f40b
--- /dev/null
+++ b/src/aleph/vm/utils/test_helpers.py
@@ -0,0 +1,86 @@
+import datetime
+import json
+
+import eth_account.messages
+import pytest
+from eth_account.datastructures import SignedMessage
+from eth_account.signers.local import LocalAccount
+from jwcrypto import jwk
+from jwcrypto.jwa import JWA
+
+
+@pytest.fixture
+def patch_datetime_now(monkeypatch):
+    """Fixture for patching the datetime.now() and datetime.utcnow() methods
+    to return a fixed datetime object.
+    This fixture creates a subclass of `datetime.datetime` called `mydatetime`,
+    which overrides the `now()` and `utcnow()` class methods to return a fixed
+    datetime object specified by `FAKE_TIME`.
+    """
+
+    class MockDateTime(datetime.datetime):
+        FAKE_TIME = datetime.datetime(2010, 12, 25, 17, 5, 55)
+
+        @classmethod
+        def now(cls, tz=None, *args, **kwargs):
+            return cls.FAKE_TIME.replace(tzinfo=tz)
+
+        @classmethod
+        def utcnow(cls, *args, **kwargs):
+            return cls.FAKE_TIME
+
+    monkeypatch.setattr(datetime, "datetime", MockDateTime)
+    return MockDateTime
+
+
+async def generate_signer_and_signed_headers_for_operation(
+    patch_datetime_now, operation_payload: dict
+) -> tuple[LocalAccount, dict]:
+    """Generate a temporary eth_account for testing and sign the operation with it"""
+    account = eth_account.Account()
+    signer_account = account.create()
+    key = jwk.JWK.generate(
+        kty="EC",
+        crv="P-256",
+        # key_ops=["verify"],
+    )
+    pubkey = {
+        "pubkey": json.loads(key.export_public()),
+        "alg": "ECDSA",
+        "domain": "localhost",
+        "address": signer_account.address,
+        "expires": (patch_datetime_now.FAKE_TIME + datetime.timedelta(days=1)).isoformat() + "Z",
+    }
+    pubkey_payload = json.dumps(pubkey).encode("utf-8").hex()
+    signable_message = eth_account.messages.encode_defunct(hexstr=pubkey_payload)
+    signed_message: SignedMessage = signer_account.sign_message(signable_message)
+    pubkey_signature = to_0x_hex(signed_message.signature)
+    pubkey_signature_header = json.dumps(
+        {
+            "payload": pubkey_payload,
+            "signature": pubkey_signature,
+        }
+    )
+    payload_as_bytes = json.dumps(operation_payload).encode("utf-8")
+
+    payload_signature = JWA.signing_alg("ES256").sign(key, payload_as_bytes)
+    headers = {
+        "X-SignedPubKey": pubkey_signature_header,
+        "X-SignedOperation": json.dumps(
+            {
+                "payload": payload_as_bytes.hex(),
+                "signature": payload_signature.hex(),
+            }
+        ),
+    }
+    return signer_account, headers
+
+
+def to_0x_hex(b: bytes) -> str:
+    """
+    Convert the bytes to a 0x-prefixed hex string
+    """
+
+    # force this for compat between different hexbytes versions which behave differenty
+    # and conflict with other package don't allow us to have the version we want
+    return "0x" + bytes.hex(b)
diff --git a/tests/supervisor/test_authentication.py b/tests/supervisor/test_authentication.py
index 249806f01..f4269a4ad 100644
--- a/tests/supervisor/test_authentication.py
+++ b/tests/supervisor/test_authentication.py
@@ -1,4 +1,3 @@
-import datetime
 import json
 from typing import Any
 
@@ -8,22 +7,16 @@
 from eth_account.datastructures import SignedMessage
 from jwcrypto import jwk, jws
 from jwcrypto.common import base64url_decode
-from jwcrypto.jwa import JWA
 
 from aleph.vm.orchestrator.views.authentication import (
     authenticate_jwk,
     require_jwk_authentication,
 )
-
-
-def to_0x_hex(b: bytes) -> str:
-    """
-    Convert the bytes to a 0x-prefixed hex string
-    """
-
-    # force this for compat between different hexbytes versions which behave differenty
-    # and conflict with other package don't allow us to have the version we want
-    return "0x" + bytes.hex(b)
+from aleph.vm.utils.test_helpers import (
+    generate_signer_and_signed_headers_for_operation,
+    patch_datetime_now,
+    to_0x_hex,
+)
 
 
 @pytest.mark.asyncio
@@ -67,30 +60,6 @@ async def view(request, authenticated_sender):
     assert {"error": "Invalid X-SignedPubKey format"} == r
 
 
-@pytest.fixture
-def patch_datetime_now(monkeypatch):
-    """Fixture for patching the datetime.now() and datetime.utcnow() methods
-    to return a fixed datetime object.
-    This fixture creates a subclass of `datetime.datetime` called `mydatetime`,
-    which overrides the `now()` and `utcnow()` class methods to return a fixed
-    datetime object specified by `FAKE_TIME`.
-    """
-
-    class MockDateTime(datetime.datetime):
-        FAKE_TIME = datetime.datetime(2010, 12, 25, 17, 5, 55)
-
-        @classmethod
-        def now(cls, tz=None, *args, **kwargs):
-            return cls.FAKE_TIME.replace(tzinfo=tz)
-
-        @classmethod
-        def utcnow(cls, *args, **kwargs):
-            return cls.FAKE_TIME
-
-    monkeypatch.setattr(datetime, "datetime", MockDateTime)
-    return MockDateTime
-
-
 @pytest.mark.asyncio
 async def test_require_jwk_authentication_expired(aiohttp_client):
     app = web.Application()
@@ -257,32 +226,8 @@ async def test_require_jwk_authentication_good_key(aiohttp_client, patch_datetim
     """An HTTP request to a view decorated by `@require_jwk_authentication`
     auth correctly a temporary key signed by a wallet and an operation signed by that key"""
     app = web.Application()
-
-    account = eth_account.Account()
-    signer_account = account.create()
-    key = jwk.JWK.generate(
-        kty="EC",
-        crv="P-256",
-        # key_ops=["verify"],
-    )
-
-    pubkey = {
-        "pubkey": json.loads(key.export_public()),
-        "alg": "ECDSA",
-        "domain": "localhost",
-        "address": signer_account.address,
-        "expires": (patch_datetime_now.FAKE_TIME + datetime.timedelta(days=1)).isoformat() + "Z",
-    }
-    pubkey_payload = json.dumps(pubkey).encode("utf-8").hex()
-    signable_message = eth_account.messages.encode_defunct(hexstr=pubkey_payload)
-    signed_message: SignedMessage = signer_account.sign_message(signable_message)
-    pubkey_signature = to_0x_hex(signed_message.signature)
-    pubkey_signature_header = json.dumps(
-        {
-            "payload": pubkey_payload,
-            "signature": pubkey_signature,
-        }
-    )
+    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/"}
+    signer_account, headers = await generate_signer_and_signed_headers_for_operation(patch_datetime_now, payload)
 
     @require_jwk_authentication
     async def view(request, authenticated_sender):
@@ -292,18 +237,6 @@ async def view(request, authenticated_sender):
     app.router.add_get("", view)
     client = await aiohttp_client(app)
 
-    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/"}
-
-    payload_as_bytes = json.dumps(payload).encode("utf-8")
-    headers = {"X-SignedPubKey": pubkey_signature_header}
-    payload_signature = JWA.signing_alg("ES256").sign(key, payload_as_bytes)
-    headers["X-SignedOperation"] = json.dumps(
-        {
-            "payload": payload_as_bytes.hex(),
-            "signature": payload_signature.hex(),
-        }
-    )
-
     resp = await client.get("/", headers=headers)
     assert resp.status == 200, await resp.text()
 
diff --git a/tests/supervisor/views/test_operator.py b/tests/supervisor/views/test_operator.py
index 3e50fe53e..72a42ae09 100644
--- a/tests/supervisor/views/test_operator.py
+++ b/tests/supervisor/views/test_operator.py
@@ -1,6 +1,19 @@
+import asyncio
+import datetime
+import json
+from asyncio import Queue
+
+import aiohttp
 import pytest
+from aiohttp.test_utils import TestClient
 
 from aleph.vm.orchestrator.supervisor import setup_webapp
+from aleph.vm.pool import VmPool
+from aleph.vm.utils.logs import EntryDict
+from aleph.vm.utils.test_helpers import (
+    generate_signer_and_signed_headers_for_operation,
+    patch_datetime_now,
+)
 
 
 @pytest.mark.asyncio
@@ -36,3 +49,226 @@ class FakeVmPool:
         await response.text() == "Rebooted VM with ref fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_"
     )
     assert pool.systemd_manager.restart.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_logs(aiohttp_client, mocker):
+    mock_address = "mock_address"
+    mock_hash = "fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_"
+    mocker.patch(
+        "aleph.vm.orchestrator.views.authentication.authenticate_jwk",
+        return_value=mock_address,
+    )
+
+    # noinspection PyMissingConstructor
+    class FakeVmPool(VmPool):
+        def __init__(self):
+            pass
+
+        executions = {
+            mock_hash: mocker.Mock(
+                vm_hash=mock_hash,
+                message=mocker.Mock(address=mock_address),
+                is_confidential=False,
+                is_running=True,
+                vm=mocker.Mock(
+                    past_logs=mocker.Mock(
+                        return_value=[
+                            EntryDict(
+                                SYSLOG_IDENTIFIER="stdout",
+                                MESSAGE="logline1",
+                                __REALTIME_TIMESTAMP=datetime.datetime(2020, 10, 12, 1, 2),
+                            ),
+                            EntryDict(
+                                SYSLOG_IDENTIFIER="stdout",
+                                MESSAGE="logline2",
+                                __REALTIME_TIMESTAMP=datetime.datetime(2020, 10, 12, 1, 3),
+                            ),
+                        ]
+                    )
+                ),
+            ),
+        }
+        systemd_manager = mocker.Mock(restart=mocker.Mock())
+
+    app = setup_webapp()
+    pool = FakeVmPool()
+    app["vm_pool"] = pool
+    app["pubsub"] = FakeVmPool()
+    client = await aiohttp_client(app)
+    response = await client.get(
+        f"/control/machine/{mock_hash}/logs",
+    )
+    assert response.status == 200
+    assert await response.text() == "2020-10-12T01:02:00> logline12020-10-12T01:03:00> logline2"
+
+
+@pytest.mark.asyncio
+async def test_websocket_logs(aiohttp_client, mocker):
+    mock_address = "mock_address"
+    mock_hash = "fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_"
+    mocker.patch(
+        "aleph.vm.orchestrator.views.operator.authenticate_websocket_message",
+        return_value=mock_address,
+    )
+    fake_queue: Queue[tuple[str, str]] = asyncio.Queue()
+    await fake_queue.put(("stdout", "this is a first log entry"))
+
+    fakeVmPool = mocker.Mock(
+        executions={
+            mock_hash: mocker.Mock(
+                vm_hash=mock_hash,
+                message=mocker.Mock(address=mock_address),
+                is_confidential=False,
+                is_running=True,
+                vm=mocker.Mock(
+                    get_log_queue=mocker.Mock(return_value=fake_queue),
+                ),
+            ),
+        },
+    )
+    app = setup_webapp()
+    app["vm_pool"] = fakeVmPool
+    app["pubsub"] = None
+    client = await aiohttp_client(app)
+    websocket = await client.ws_connect(
+        f"/control/machine/{mock_hash}/stream_logs",
+    )
+    await websocket.send_json({"auth": "auth is disabled"})
+    response = await websocket.receive_json()
+    assert response == {"status": "connected"}
+
+    response = await websocket.receive_json()
+    assert response == {"message": "this is a first log entry", "type": "stdout"}
+
+    await fake_queue.put(("stdout", "this is a second log entry"))
+    response = await websocket.receive_json()
+    assert response == {"message": "this is a second log entry", "type": "stdout"}
+    await websocket.close()
+    assert websocket.closed
+
+
+@pytest.mark.asyncio
+async def test_websocket_logs_missing_auth(aiohttp_client, mocker):
+    mock_address = "mock_address"
+    mock_hash = "fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_"
+
+    fake_queue: Queue[tuple[str, str]] = asyncio.Queue()
+    await fake_queue.put(("stdout", "this is a first log entry"))
+
+    fakeVmPool = mocker.Mock(
+        executions={
+            mock_hash: mocker.Mock(
+                vm_hash=mock_hash,
+                message=mocker.Mock(address=mock_address),
+                is_confidential=False,
+                is_running=True,
+                vm=mocker.Mock(
+                    get_log_queue=mocker.Mock(return_value=fake_queue),
+                ),
+            ),
+        },
+    )
+    app = setup_webapp()
+    app["vm_pool"] = fakeVmPool
+    app["pubsub"] = None
+    client = await aiohttp_client(app)
+    websocket = await client.ws_connect(
+        f"/control/machine/{mock_hash}/stream_logs",
+    )
+    # Wait for message without sending an auth package.
+    # Test with a timeout because we receive nothing
+    with pytest.raises((TimeoutError, asyncio.exceptions.TimeoutError)):
+        response = await websocket.receive_json(timeout=1)
+        assert False
+
+    # It's totally reachable with the pytest.raises
+    # noinspection PyUnreachableCode
+    await websocket.close()
+    assert websocket.closed
+
+
+@pytest.mark.asyncio
+async def test_websocket_logs_invalid_auth(aiohttp_client, mocker):
+    mock_address = "mock_address"
+    mock_hash = "fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_"
+
+    fake_queue: Queue[tuple[str, str]] = asyncio.Queue()
+    await fake_queue.put(("stdout", "this is a first log entry"))
+
+    fakeVmPool = mocker.Mock(
+        executions={
+            mock_hash: mocker.Mock(
+                vm_hash=mock_hash,
+                message=mocker.Mock(address=mock_address),
+                is_confidential=False,
+                is_running=True,
+                vm=mocker.Mock(
+                    get_log_queue=mocker.Mock(return_value=fake_queue),
+                ),
+            ),
+        },
+    )
+    app = setup_webapp()
+    app["vm_pool"] = fakeVmPool
+    app["pubsub"] = None
+    client: TestClient = await aiohttp_client(app)
+    websocket = await client.ws_connect(
+        f"/control/machine/{mock_hash}/stream_logs",
+    )
+
+    await websocket.send_json({"auth": "invalid auth package"})
+    response = await websocket.receive()
+    # Subject to change in the future, for now the connexion si broken and closed
+    assert response.type == aiohttp.WSMsgType.CLOSE
+    assert websocket.closed
+
+
+@pytest.mark.asyncio
+async def test_websocket_logs_good_auth(aiohttp_client, mocker, patch_datetime_now):
+    "Test valid authentification for websocket logs endpoint"
+    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/"}
+    signer_account, headers = await generate_signer_and_signed_headers_for_operation(patch_datetime_now, payload)
+
+    mock_address = signer_account.address
+    mock_hash = "fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_"
+
+    fake_queue: Queue[tuple[str, str]] = asyncio.Queue()
+    await fake_queue.put(("stdout", "this is a first log entry"))
+
+    fakeVmPool = mocker.Mock(
+        executions={
+            mock_hash: mocker.Mock(
+                vm_hash=mock_hash,
+                message=mocker.Mock(address=mock_address),
+                is_confidential=False,
+                is_running=True,
+                vm=mocker.Mock(
+                    get_log_queue=mocker.Mock(return_value=fake_queue),
+                ),
+            ),
+        },
+    )
+    app = setup_webapp()
+    app["vm_pool"] = fakeVmPool
+    app["pubsub"] = None
+    client = await aiohttp_client(app)
+    websocket = await client.ws_connect(
+        f"/control/machine/{mock_hash}/stream_logs",
+    )
+    # Need to deserialize since we pass a json otherwhise it get double json encoded
+    # which is not what the endpoint expect
+    auth_package = {
+        "X-SignedPubKey": json.loads(headers["X-SignedPubKey"]),
+        "X-SignedOperation": json.loads(headers["X-SignedOperation"]),
+    }
+
+    await websocket.send_json({"auth": auth_package})
+    response = await websocket.receive_json()
+    assert response == {"status": "connected"}
+
+    response = await websocket.receive_json()
+    assert response == {"message": "this is a first log entry", "type": "stdout"}
+
+    await websocket.close()
+    assert websocket.closed

From 1fedcd76198f57f6913fa252c8e3b1268218fb4e Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Fri, 5 Jul 2024 13:17:13 +0200
Subject: [PATCH 806/990] Improve instances code (#654)

Problem: If an instance is created and removed, one of the base mounted devices remains on the system. Also, a user cannot create a qemu disk from Debian image.

Solution: Remove the useless device from the system and create the two needed script to create Qemu Debian runtimes.
---
 .../create-debian-11-qemu-disk.sh              | 18 ++++++++++++++++++
 .../create-debian-12-qemu-disk.sh              | 18 ++++++++++++++++++
 .../vm/hypervisors/firecracker/microvm.py      |  1 +
 3 files changed, 37 insertions(+)
 create mode 100755 runtimes/instance-rootfs/create-debian-11-qemu-disk.sh
 create mode 100755 runtimes/instance-rootfs/create-debian-12-qemu-disk.sh

diff --git a/runtimes/instance-rootfs/create-debian-11-qemu-disk.sh b/runtimes/instance-rootfs/create-debian-11-qemu-disk.sh
new file mode 100755
index 000000000..59bef20b5
--- /dev/null
+++ b/runtimes/instance-rootfs/create-debian-11-qemu-disk.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euf
+
+# Variables
+ROOTFS_FILENAME="./rootfs.img"
+IMAGE_URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-genericcloud-amd64.qcow2"
+IMAGE_NAME="debian-11-genericcloud-amd64.qcow2"
+
+# Cleanup previous run
+rm -f "$ROOTFS_FILENAME"
+
+# Download Ubuntu image
+echo "Downloading Debian 11 image"
+curl -L "$IMAGE_URL" -o "$IMAGE_NAME"
+
+# Rename final file
+mv "$IMAGE_NAME" "$ROOTFS_FILENAME"
diff --git a/runtimes/instance-rootfs/create-debian-12-qemu-disk.sh b/runtimes/instance-rootfs/create-debian-12-qemu-disk.sh
new file mode 100755
index 000000000..1a4df4235
--- /dev/null
+++ b/runtimes/instance-rootfs/create-debian-12-qemu-disk.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euf
+
+# Variables
+ROOTFS_FILENAME="./rootfs.img"
+IMAGE_URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2"
+IMAGE_NAME="debian-12-genericcloud-amd64.qcow2"
+
+# Cleanup previous run
+rm -f "$ROOTFS_FILENAME"
+
+# Download Ubuntu image
+echo "Downloading Debian 12 image"
+curl -L "$IMAGE_URL" -o "$IMAGE_NAME"
+
+# Rename final file
+mv "$IMAGE_NAME" "$ROOTFS_FILENAME"
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 8d66ac257..bae76b95f 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -477,6 +477,7 @@ async def teardown(self):
             await asyncio.sleep(1)
             root_fs = self.mounted_rootfs.name
             system(f"dmsetup remove {root_fs}")
+            system(f"dmsetup remove {root_fs}_base")
             if self.use_jailer and Path(self.jailer_path).is_dir():
                 shutil.rmtree(self.jailer_path)
 

From a3c5a8ca9904bc92c9434c26661777878a1297dc Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 5 Jul 2024 15:37:03 +0200
Subject: [PATCH 807/990] Fix: AttributeError: 'MicroVM' object has no
 attribute 'send_shutdown_message' (#653)

* Fix: AttributeError: 'MicroVM' object has no attribute 'send_shutdown_message'

Error reported on https://github.com/aleph-im/aleph-vm/issues/652


Co-authored-by: Olivier Le Thanh Duong 
---
 src/aleph/vm/controllers/__main__.py    | 7 ++++++-
 src/aleph/vm/hypervisors/qemu/qemuvm.py | 4 ++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index 32640bf6e..3232c82bf 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -82,7 +82,12 @@ async def execute_persistent_vm(config: Configuration):
 async def handle_persistent_vm(config: Configuration, execution: Union[MicroVM, QemuVM], process: Process):
     # Catch the terminating signal and send a proper message to the vm to stop it so it close files properly
     loop = asyncio.get_event_loop()
-    loop.add_signal_handler(signal.SIGTERM, execution.send_shutdown_message)
+
+    def callback():
+        """Callback for the signal handler to stop the VM and cleanup properly on SIGTERM."""
+        loop.create_task(execution.teardown())
+
+    loop.add_signal_handler(signal.SIGTERM, callback)
 
     await process.wait()
     logger.info(f"Process terminated with {process.returncode}")
diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index b4e577dde..0d49403c8 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -125,3 +125,7 @@ def send_shutdown_message(self):
                 logger.warning("unexpected answer from VM", resp)
             print("shutdown message sent")
             client.close()
+
+    async def teardown(self):
+        """Stop the VM."""
+        self.send_shutdown_message()

From 67ae73246d90fd399fde048dc6530f604f2b047e Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 5 Jul 2024 16:47:18 +0200
Subject: [PATCH 808/990] Implement Confidential Computing (#650)

* Implement Start Confidential endpoint (#627)

* Problem: The server don't have a directory to save the platform certificates generated by sevctl.

Solution: Set that directory field on settings class and ensure to create the folder on initialization step.

* Problem: The aren't an endpoint to be able to get the confidential platform certificates to start the VM key exchange.

Solution: Create that endpoint and return the platform certificates generated by the `sevctl` command.

* Fix: Solved code quality issues.

* Fix: Added 2 test cases for that endpoint.

* Fix: Added PR suggestions.

* Fix: Modified test mock to let the tests work

* Problem: Now isn't possible as a VM operator to get the client session certificates to initialize a confidential VM.

Solution: Create an operator start endpoint that receive the confidential session files and starts the qemu VM to continue with the certificate exchange methods.

* Fix: Remove useless aiofiles import

* Fix: Solve test issues after code quality fixes

* Fix: Solve code quality issues.

* Fix: Solve code quality issues.

* Fix: Write file in sync mode to avoid adding a new dependency. Files to write should be so small, so any blocking issue should be here.

* Fix: Solved PR comments and wrong conditionals.

* Fix: Solved more PR comments.

* Fix: Removed unexisting import

* Fix: Added useless command requested on the PR review.

* Fix: Changed endpoint path and added automatic tests for that endpoint.

* Fix: Solved settings singleton issue with testing, adding an `initialize_settings` method.

* Fix: Just disable the setting that is failing and remove previous method to initialize the singleton.

* Fix: CI Droplet cleanup failed when same name was used

When there were multiple Droplets with the same name, cleanup using doctl compute droplet delete -f $NAME would not work.

Error: There are 3 Droplets with the name "aleph-vm-ci-XXX"; please provide a specific Droplet ID. [425559566, 425702949, 425703724]
(cherry picked from commit b82450340254a52db57f7f7546caab545edcaadc)

* Problem: Could not install on Python 12 via pip install -e

because of deps problem.
Solution : upgrade aiohttp version

* Problem: Crash in log when VM was printing control char

* Raise log level for VM termination in controller so we always display when it finish

* comment

* Problem: Error were not properly returned in allocation endpoint

* Add Qemu confidential controler implementation

* remove duplicate endpoint

* fix test in test_about_certificates

* Add TODO comment

Co-authored-by: nesitor 

* Only run Confidentifial if is_confidential

* Add script to build OVMF file for confidential VMs (#636)

* Rename the confidential endpoints (#641)

* Rename confidential endpoints

* Rename the function too and reorder

* isort

* Use unified logging system for confidential

* Provide an example confidential image construction script and instruction

* Prevent the cleanup being run twice

but still works with -e

* Problem: sudo command was not working inside the VM

ensure the setuid bit stay preserved when copying the file"

* remove unecesary step

* More example instruction

* Problem: A user cannot specify which OVMF firmware want to use for they instances.

Solution: Use new aleph-message version that includes that data schema and implement it on the qemu confidential resources.

* Adapt example confidential message

* fix host volume for confidential

* Force aleph-message minimal version

* Correct problem in HostVolume code

* Merge both test_operator

* Fix options for HostVolume

* Move `domain` payload field to Operation token instead PubKey one (#647)

* Problem: If a user wants to manage different operations for a different CRNs, they have to sign a new pubkey token for every CRN, and this is so bad for the user experience.

Solution: Move the `domain` field to the operation token payload instead the pubkey one, just to improve the user experience and maintain the security integrity.

* Fix: Solved test error message failing.

---------

Co-authored-by: Andres D. Molins 

* Problem: Failing initialization of AlephQemuConfidentialInstance

due to merge problem

* fix invoking of sevctl

* Update src/aleph/vm/pool.py

---------

Co-authored-by: nesitor 
Co-authored-by: Hugo Herter 
Co-authored-by: Andres D. Molins 
---
 ...fidential_instance_message_from_aleph.json |  55 +++++++
 examples/example_confidential_image/README.md |  76 +++++++++
 .../build_debian_image.sh                     | 151 ++++++++++++++++++
 .../setup_debian_rootfs.sh                    | 114 +++++++++++++
 pyproject.toml                                |   3 +-
 .../create_disk_image.sh                      |   2 +-
 runtimes/ovmf/README.md                       |  24 +++
 runtimes/ovmf/build_ovmf.sh                   |  35 ++++
 runtimes/ovmf/download_dependencies.sh        |  39 +++++
 ...okation-of-cryptomount-s-for-AMD-SEV.patch |  58 +++++++
 src/aleph/vm/conf.py                          |   9 +-
 src/aleph/vm/controllers/__main__.py          |   8 +-
 src/aleph/vm/controllers/configuration.py     |  27 +++-
 src/aleph/vm/controllers/qemu/client.py       |  75 +++++++++
 src/aleph/vm/controllers/qemu/instance.py     |  33 +++-
 .../controllers/qemu_confidential/__init__.py |   0
 .../controllers/qemu_confidential/instance.py | 145 +++++++++++++++++
 src/aleph/vm/hypervisors/qemu/qemuvm.py       |  26 ++-
 .../hypervisors/qemu_confidential/__init__.py |   0
 .../hypervisors/qemu_confidential/qemuvm.py   | 135 ++++++++++++++++
 src/aleph/vm/models.py                        |  46 ++++--
 src/aleph/vm/orchestrator/run.py              |  33 +---
 src/aleph/vm/orchestrator/supervisor.py       |   6 +
 .../vm/orchestrator/views/authentication.py   |  11 +-
 src/aleph/vm/orchestrator/views/operator.py   | 127 ++++++++++++++-
 src/aleph/vm/pool.py                          |   4 +-
 src/aleph/vm/sevclient.py                     |   2 +-
 src/aleph/vm/utils/__init__.py                |   1 +
 tests/supervisor/test_authentication.py       |  18 +--
 tests/supervisor/test_qemu_instance.py        |   2 +
 tests/supervisor/test_views.py                |   4 +-
 tests/supervisor/views/test_operator.py       | 131 ++++++++++++++-
 32 files changed, 1321 insertions(+), 79 deletions(-)
 create mode 100644 examples/confidential_instance_message_from_aleph.json
 create mode 100644 examples/example_confidential_image/README.md
 create mode 100644 examples/example_confidential_image/build_debian_image.sh
 create mode 100644 examples/example_confidential_image/setup_debian_rootfs.sh
 create mode 100644 runtimes/ovmf/README.md
 create mode 100644 runtimes/ovmf/build_ovmf.sh
 create mode 100644 runtimes/ovmf/download_dependencies.sh
 create mode 100644 runtimes/ovmf/patches/edk2/0001-Fix-invokation-of-cryptomount-s-for-AMD-SEV.patch
 create mode 100644 src/aleph/vm/controllers/qemu/client.py
 create mode 100644 src/aleph/vm/controllers/qemu_confidential/__init__.py
 create mode 100644 src/aleph/vm/controllers/qemu_confidential/instance.py
 create mode 100644 src/aleph/vm/hypervisors/qemu_confidential/__init__.py
 create mode 100644 src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py

diff --git a/examples/confidential_instance_message_from_aleph.json b/examples/confidential_instance_message_from_aleph.json
new file mode 100644
index 000000000..6b130c65c
--- /dev/null
+++ b/examples/confidential_instance_message_from_aleph.json
@@ -0,0 +1,55 @@
+{
+  "chain": "ETH",
+  "item_hash": "fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-fake-hash-hash",
+  "sender": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba",
+  "type": "INSTANCE",
+  "channel": "Fun-dApps",
+  "confirmed": true,
+  "content": {
+    "address": "0x2b0eE984F821C710104e575953634b3f3d364Ec4",
+    "allow_amend": false,
+    "variables": {
+      "VM_CUSTOM_NUMBER": "32"
+    },
+    "environment": {
+      "hypervisor": "qemu",
+      "reproducible": true,
+      "internet": true,
+      "aleph_api": true,
+      "shared_cache": true,
+      "trusted_execution": {
+        "firmware": "88978bb4c2ff54400ce5f51c3a109e1af1ab03d1ea4409666917317ac513846b",
+        "policy": 1
+      }
+    },
+    "resources": {
+      "vcpus": 1,
+      "memory": 512,
+      "seconds": 30
+    },
+    "rootfs": {
+      "parent": {
+        "ref": "549ec451d9b099cad112d4aaa2c00ac40fb6729a92ff252ff22eef0b5c3cb613",
+        "use_latest": true
+      },
+      "persistence": "host",
+      "size_mib": 5000
+    },
+    "authorized_keys": [
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDj95BHGUx0/z2G/tTrEi8o49i70xvjcEUdSs3j4A33jE7pAphrfRVbuFMgFubcm8n9r5ftd/H8SjjTL4hY9YvWV5ZuMf92GUga3n4wgevvPlBszYZCy/idxFl0vtHYC1CcK9v4tVb9onhDt8FOJkf2m6PmDyvC+6tl6LwoerXTeeiKr5VnTB4KOBkammtFmix3d1X1SZd/cxdwZIHcQ7BNsqBm2w/YzVba6Z4ZnFUelBkQtMQqNs2aV51O1pFFqtZp2mM71D5d8vn9pOtqJ5QmY5IW6NypcyqKJZg5o6QguK5rdXLkc7AWro27BiaHIENl3w0wazp9EDO9zPAGJ6lz olivier@lanius"
+    ],
+    "time": 1619017773.8950517
+  },
+  "item_content": "{\"address\":\"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\",\"allow_amend\":false,\"variables\":{\"VM_CUSTOM_NUMBER\":\"32\"},\"environment\":{\"reproducible\":true,\"internet\":true,\"aleph_api\":true,\"shared_cache\":true},\"resources\":{\"vcpus\":1,\"memory\":128,\"seconds\":30},\"rootfs\":{\"parent\":{\"ref\":\"549ec451d9b099cad112d4aaa2c00ac40fb6729a92ff252ff22eef0b5c3cb613\",\"use_latest\":true},\"persistence\":\"host\",\"size_mib\":20000},\"cloud_config\":{\"password\":\"password\",\"chpasswd\":{\"expire\":\"False\"}},\"volumes\":[{\"mount\":\"/opt/venv\",\"ref\":\"5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51\",\"use_latest\":false},{\"comment\":\"Working data persisted on the VM supervisor, not available on other nodes\",\"mount\":\"/var/lib/example\",\"name\":\"data\",\"persistence\":\"host\",\"size_mib\":5}],\"replaces\":\"0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba\",\"time\":1619017773.8950517}",
+  "item_type": "inline",
+  "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b",
+  "size": 749,
+  "time": 1619017773.8950577,
+  "confirmations": [
+    {
+      "chain": "ETH",
+      "height": 12284734,
+      "hash": "0x67f2f3cde5e94e70615c92629c70d22dc959a118f46e9411b29659c2fce87cdc"
+    }
+  ]
+}
diff --git a/examples/example_confidential_image/README.md b/examples/example_confidential_image/README.md
new file mode 100644
index 000000000..a4d3b2275
--- /dev/null
+++ b/examples/example_confidential_image/README.md
@@ -0,0 +1,76 @@
+# Create an encrypted VM image
+Theses samples scripts create an encrypted VM image suitable be used for confidential computing.
+
+They will create an encrypted partition, a boot partition and the necessary initramfs to decrypt the partition. The created image is designed to work in tandem with the custom OVMF found in `runtimes/ovmf` which can receive the decryption key in a secure channel via QMP and pass it to grub to decrypt the disk.  
+
+You can customise your VM by modifying the `setup_debian_rootfs.sh` script and  adding your instructions at the end. This script is run "inside" the VM chroot. For examples: add your user, ssh key or install additional software. 
+
+
+## Procedure to create the image
+### Requirements
+* guestmount
+* parted
+* cryptsetup
+
+On debian they can be installed via their respective packages :
+`apt install guestmount parted cryptsetup`
+
+### Procure a debian image
+Your image need to have cloud-init installed in it for the network setup. It is recommended to start from the genericcloud image. Experiment with using the nocloud image then installing cloud-init have failed to work.
+
+```shell
+wget https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2
+```
+
+### Extract the root filesystem
+To do so, we simply need to mount the raw image with `guestmount`.
+
+> Make sure that you stop the VM before exporting the root filesystem.
+
+```shell
+sudo mkdir -p /mnt/debian
+sudo guestmount \
+  --format=qcow2 \
+  -a ./debian-12-genericcloud-amd64.qcow2 \
+  -o allow_other \
+  -i /mnt/debian
+```
+
+Then, you can simply copy the root file system to any directory, take caution to preserve the proper permission like the setuid bit with the --archive option.
+
+```shell
+export ROOT_DIR=./extracted
+mkdir ${ROOT_DIR}
+sudo cp --archive /mnt/debian/* ${ROOT_DIR}
+```
+
+Clean up the mount
+```shell
+sudo guestunmount /mnt/debian
+sudo rm -r /mnt/debian
+```
+
+
+Run the build_debian_image.sh that will create the image with the encrypted disk 
+> This script will require sudo for certain commands
+```shell
+bash ./build_debian_image.sh -o ~/destination-image.img --password your-password -r $ROOT_DIR
+```
+
+> If you need debuging you can pass the -x option to bash before the script name
+
+## To test and further customise you image you can also boot it inside qemu
+```shell
+sudo qemu-system-x86_64 \
+  -drive format=raw,file= \
+  -enable-kvm \
+  -m 2048 \
+  -nic user,model=virtio \
+  -nographic \
+  -serial mon:stdio \
+  -drive if=pflash,format=raw,unit=0,file=/usr/share/ovmf/OVMF.fd,readonly=on
+  ```
+
+> Once you have entered your password you might have to wait a minute or so for the disk to decrypt and boot.
+
+To exit qemu : press Ctrl a, x and then [Enter]
\ No newline at end of file
diff --git a/examples/example_confidential_image/build_debian_image.sh b/examples/example_confidential_image/build_debian_image.sh
new file mode 100644
index 000000000..3be89a77e
--- /dev/null
+++ b/examples/example_confidential_image/build_debian_image.sh
@@ -0,0 +1,151 @@
+#!/bin/bash
+
+set -eo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+ROOTFS_DIR=""
+IMAGE_SIZE="4GB"
+IMAGE_FILE=""
+MAPPER_NAME="cr_root"
+LOOP_DEVICE_ID=""
+MAPPED_DEVICE_ID=""
+MOUNT_POINT=""
+CLEANUP_DONE=false
+
+cleanup() {
+  if [ "$CLEANUP_DONE" = true ]; then
+    return
+  fi
+  CLEANUP_DONE=true
+  echo "Cleaning up..."
+  if mountpoint -q "${MOUNT_POINT}"; then
+    sudo umount --recursive "${MOUNT_POINT}" || echo "Failed to unmount ${MOUNT_POINT}"
+  fi
+  if [ -n "${MAPPED_DEVICE_ID}" ]; then
+    sudo cryptsetup close "${MAPPED_DEVICE_ID}" || echo "Failed to close encrypted device ${MAPPED_DEVICE_ID}"
+  fi
+  if [ -n "${LOOP_DEVICE_ID}" ]; then
+    sudo losetup -d "${LOOP_DEVICE_ID}" || echo "Failed to detach loop device ${LOOP_DEVICE_ID}"
+  fi
+  if [ -f "${KEY_FILE}" ]; then
+    rm -f "${KEY_FILE}" || echo "Failed to remove key file ${KEY_FILE}"
+  fi
+}
+
+
+# Trap command to catch and handle various signals:
+# - EXIT: Triggered when the script exits (normal completion or an error).
+# - HUP (SIGHUP): Signal 1, sent when the controlling terminal is closed (e.g., terminal window closed or SSH session logout).
+# - INT (SIGINT): Signal 2, sent when the user interrupts the process (e.g., pressing Ctrl+C).
+# - QUIT (SIGQUIT): Signal 3, sent when the user requests the process to quit and perform a core dump (e.g., pressing Ctrl+\).
+# - PIPE (SIGPIPE): Signal 13, sent when attempting to write to a pipe without a reader (e.g., in scripts using pipelines if a command in the pipeline exits prematurely).
+# - TERM (SIGTERM): Signal 15, sent by the kill command to request the process to terminate gracefully.
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+usage() {
+  cat <&2
+Usage:
+    $0 --rootfs-dir ROOTFS_DIR [--image-size IMAGE_SIZE] [--password DISK_PASSWORD] [--mapper-name MAPPER_NAME]
+    -o IMAGE_FILE | --output IMAGE_FILE       Image file to use. Defaults to ".img."
+    -p DISK_PASSWORD | --password=DISK_PASSWORD   Password to use for the encrypted disk. Automatically generated if not specified.
+    -r ROOTFS_DIR | --rootfs-dir=ROOTFS_DIR   Directory containing the original rootfs.
+    -s IMAGE_SIZE | --image-size IMAGE_SIZE   Size of the target image, ex: 20GB. Defaults to 4GB.
+    -m MAPPER_NAME | --mapper-name=MAPPER_NAME   Device mapped name for encrypted disk. Default to "cr_root" if not specified.
+USAGE
+}
+
+while true; do
+  case "$1" in
+  -o | --output)
+    IMAGE_FILE=$2
+    shift 2
+    ;;
+  -p | --password)
+    DISK_PASSWORD=$2
+    shift 2
+    ;;
+  -r | --rootfs-dir)
+    ROOTFS_DIR=$2
+    shift 2
+    ;;
+  -s | --image-size)
+    IMAGE_SIZE=$2
+    shift 2
+    ;;
+  -m | --mapper-name)
+      MAPPER_NAME=$2
+      shift 2
+      ;;
+  *)
+    break
+    ;;
+  esac
+done
+
+if [ -z "${ROOTFS_DIR}" ]; then
+  usage
+  exit 1
+fi
+
+if [ -z "${DISK_PASSWORD}" ]; then
+  echo "No disk password provided. Generating one..."
+  DISK_PASSWORD=$(
+    tr "${KEY_FILE}"
+
+sudo cryptsetup --batch-mode --type luks1 --key-file "${KEY_FILE}" luksFormat "${OS_PARTITION_DEVICE_ID}"
+sudo cryptsetup open --key-file "${KEY_FILE}" "${OS_PARTITION_DEVICE_ID}" "${MAPPER_NAME}"
+sudo mkfs.ext4 "${MAPPED_DEVICE_ID}"
+
+echo "Copying root file system to the new OS partition..."
+sudo mkdir -p "${MOUNT_POINT}"
+sudo mount "${MAPPED_DEVICE_ID}" "${MOUNT_POINT}"
+sudo cp --archive "${ROOTFS_DIR}"/* "${MOUNT_POINT}"
+
+echo "Configuring root file system..."
+for m in run sys proc dev; do sudo mount --bind /$m ${MOUNT_POINT}/$m; done
+sudo cp "${SCRIPT_DIR}/setup_debian_rootfs.sh" "${KEY_FILE}" "${MOUNT_POINT}"
+sudo chroot "${MOUNT_POINT}" bash setup_debian_rootfs.sh --loop-device-id "${LOOP_DEVICE_ID}" --mapper-name "${MAPPER_NAME}"
+sudo rm "${MOUNT_POINT}/setup_debian_rootfs.sh" "${KEY_FILE}"
+
+cleanup
+
+echo "Done! The new image is available as ${IMAGE_FILE}."
+echo "Disk password: ${DISK_PASSWORD}"
diff --git a/examples/example_confidential_image/setup_debian_rootfs.sh b/examples/example_confidential_image/setup_debian_rootfs.sh
new file mode 100644
index 000000000..f87683569
--- /dev/null
+++ b/examples/example_confidential_image/setup_debian_rootfs.sh
@@ -0,0 +1,114 @@
+#! /bin/bash
+# This script sets up the Debian root file system to boot from an encrypted OS partition.
+# In details:
+# * Configure crypttab to add a second key to the OS partition to make the kernel unlock
+#   the partition by itself without requiring user input
+# * Configure /etc/fstab to point to the correct devices
+# * Regenerate Grub in removable so that the only unencrypted script just points to
+#   the Grub scripts inside the encrypted partition
+# * Update the initramfs to take the modifications to the config files into account.
+
+set -eo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+LOOP_DEVICE_ID=""
+MAPPER_NAME=""
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $0 --loop-device LOOP_DEVICE_ID [--mapper-name MAPPER_NAME]
+    -d LOOP_DEVICE_ID | --loop-device-id=LOOP_DEVICE_ID   Device ID of the disk image.
+    -m MAPPER_NAME | --mapper-name=MAPPER_NAME   Device mapped name for encrypted disk. Automatically set to "cr_root" if not specified.
+USAGE
+}
+
+while test -n "$1"; do
+  case "$1" in
+  -d | --loop-device-id)
+    LOOP_DEVICE_ID=$2
+    shift 2
+    ;;
+  -p | --mapper-name)
+      MAPPER_NAME=$2
+      shift 2
+      ;;
+  esac
+done
+
+if [ -z "${LOOP_DEVICE_ID}" ]; then
+  usage
+  exit 1
+fi
+
+if [ -z "${MAPPER_NAME}" ]; then
+  MAPPER_NAME=cr_root
+fi
+
+# Temporary tmp is needed for apt
+mount -t tmpfs  -o size=100M tmpfs /tmp
+#  Install crypsetup and openssh
+DEBIAN_FRONTEND=noninteractive apt update
+DEBIAN_FRONTEND=noninteractive apt install -y -f openssh-server openssh-client cryptsetup cryptsetup-initramfs
+
+# The original password of the OS partition. Must be provided by the caller of the script.
+BOOT_KEY_FILE="${SCRIPT_DIR}/os_partition.key"
+
+BOOT_PARTITION_DEVICE_ID="${LOOP_DEVICE_ID}p1"
+OS_PARTITION_DEVICE_ID="${LOOP_DEVICE_ID}p2"
+
+BOOT_PARTITION_UUID=$(blkid --match-tag=UUID --output=value "${BOOT_PARTITION_DEVICE_ID}" )
+OS_PARTITION_UUID=$(blkid --match-tag=UUID --output=value "${OS_PARTITION_DEVICE_ID}" )
+
+MAPPED_DEVICE_ID="/dev/mapper/${MAPPER_NAME}"
+
+# Create key file to unlock the disk at boot
+mkdir -p /etc/cryptsetup-keys.d
+KEY_FILE="/etc/cryptsetup-keys.d/luks-${OS_PARTITION_UUID}.key"
+dd if=/dev/urandom bs=1 count=33|base64 -w 0 > "${KEY_FILE}"
+chmod 0600 "${KEY_FILE}"
+cryptsetup \
+  --key-slot 1 \
+  --iter-time 1 \
+  --key-file "${BOOT_KEY_FILE}" \
+  luksAddKey "${OS_PARTITION_DEVICE_ID}" \
+  "${KEY_FILE}"
+
+# Tell the kernel to look for keys in /etc/cryptsetup-keys.d
+echo "KEYFILE_PATTERN=\"/etc/cryptsetup-keys.d/*\"" >>/etc/cryptsetup-initramfs/conf-hook
+
+# Reduce the accessibility of the initramfs
+echo "UMASK=0077" >> /etc/initramfs-tools/initramfs.conf
+
+# Configure Grub and crypttab
+echo "GRUB_ENABLE_CRYPTODISK=y" >> /etc/default/grub
+echo 'GRUB_PRELOAD_MODULES="luks cryptodisk lvm ext2"' >> /etc/default/grub
+echo "${MAPPER_NAME} UUID=${OS_PARTITION_UUID} ${KEY_FILE} luks" >> /etc/crypttab
+cat << EOF > /etc/fstab
+${MAPPED_DEVICE_ID} / ext4 rw,discard,errors=remount-ro 0 1
+UUID=${BOOT_PARTITION_UUID} /boot/efi vfat defaults 0 0
+EOF
+
+# Install Grub and regenerate grub.cfg
+mount /boot/efi
+grub-install --target=x86_64-efi --removable
+grub-install --target=x86_64-efi --recheck
+grub-mkconfig -o /boot/grub/grub.cfg
+umount /boot/efi
+
+# Force Grub config to use a crypt device
+sed -i "s+root=PARTUUID= +cryptdevice=UUID=${OS_PARTITION_UUID}:${MAPPER_NAME} root=${MAPPED_DEVICE_ID} +g" /boot/grub/grub.cfg
+
+# Update initramfs after changes to fstab and crypttab
+update-initramfs -u
+
+# Generate system SSH keys
+ssh-keygen -A
+
+# Example to add a sudo user
+useradd -m -s /bin/bash username
+echo 'username:password' | chpasswd
+usermod -aG sudo username
+
+umount /tmp
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 4a6d57148..67850ab88 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,7 +25,7 @@ classifiers = [
 ]
 dependencies = [
   "pydantic[dotenv]~=1.10.13",
-  "aiohttp==3.8.6",
+  "aiohttp==3.9.5",
   "aiodns==3.1.0",
   "setproctitle==1.3.3",
   "pyyaml==6.0.1",
@@ -51,6 +51,7 @@ dependencies = [
   "aiohttp_cors~=0.7.0",
   "pyroute2==0.7.12",
   "jwcrypto==1.5.6",
+  "python-cpuid==0.1.0"
 ]
 
 [project.urls]
diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index bfaf050f1..859d678d5 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -36,7 +36,7 @@ locale-gen en_US.UTF-8
 
 echo "Pip installing aleph-sdk-python"
 mkdir -p /opt/aleph/libs
-pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.9.0' 'aleph-message==0.4.4' 'fastapi~=0.109.2'
+pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.9.0' 'aleph-message==0.4.7' 'fastapi~=0.109.2'
 
 # Compile Python code to bytecode for faster execution
 # -o2 is needed to compile with optimization level 2 which is what we launch init1.py ("python -OO")
diff --git a/runtimes/ovmf/README.md b/runtimes/ovmf/README.md
new file mode 100644
index 000000000..83d028779
--- /dev/null
+++ b/runtimes/ovmf/README.md
@@ -0,0 +1,24 @@
+# OVMF build for Confidential VMs
+
+The files in this directory build a version of OVMF able to store SEV secrets
+in a physical memory region that will then be accessible by Grub. The final OVMF image
+also include Grub in order to measure OVMF+Grub before loading secrets inside
+the VM.
+
+This process relies on the patch sets produced by James Bottomley:
+https://listman.redhat.com/archives/edk2-devel-archive/2020-November/msg01247.html
+
+## Build instructions
+
+As this requires a patched version of Grub, it is advised to build both tools inside a container.
+
+
+e.g using podman
+```
+# Clone grub and edk2, and apply the patches
+bash ./download_dependencies.sh
+podman run -v ./build_ovmf.sh:/opt/build_ovmf.sh  -v ./downloads:/opt/downloads\
+ ubuntu:22.04  bash /opt/download_dependencies.sh
+# The OVMF.fd file will be in `downloads/edk2/Build/AmdSev/RELEASE_GCC5/FV/OVMF.fd
+cp downloads/edk2/Build/AmdSev/RELEASE_GCC5/FV/OVMF.fd confidential-OVMF.fd
+```
diff --git a/runtimes/ovmf/build_ovmf.sh b/runtimes/ovmf/build_ovmf.sh
new file mode 100644
index 000000000..3b31bbf0f
--- /dev/null
+++ b/runtimes/ovmf/build_ovmf.sh
@@ -0,0 +1,35 @@
+#! /bin/bash
+# Script to build OVMF + Grub for confidential computing. The resulting image will be
+# a single firmware image containing OVMF and Grub so that the entirety of the unencrypted
+# boot code can be measured before feeding secrets to the VM.
+
+set -eo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+GRUB_DIR="${SCRIPT_DIR}/downloads/grub"
+EDK2_DIR="${SCRIPT_DIR}/downloads/edk2"
+
+if [ ! -d "${GRUB_DIR}" ]; then
+  echo "Grub directory not found: ${GRUB_DIR}" >&2
+fi
+
+if [ ! -d "${EDK2_DIR}" ]; then
+  echo "EDK2 directory not found: ${EDK2_DIR}" >&2
+fi
+
+apt-get update
+# Packages for Grub
+apt-get install -y autoconf autopoint binutils bison flex gcc gettext git make pkg-config python3 python-is-python3
+# Packages for OVMF (there are some duplicates with Grub, kept for documentation)
+apt-get install -y bison build-essential dosfstools flex iasl libgmp3-dev libmpfr-dev mtools nasm subversion texinfo uuid-dev
+
+cd $GRUB_DIR
+./bootstrap
+./configure --prefix /usr/ --with-platform=efi --target=x86_64
+make
+make install
+
+# Build OVMF
+cd $EDK2_DIR
+OvmfPkg/build.sh -b RELEASE -p OvmfPkg/AmdSev/AmdSevX64.dsc
diff --git a/runtimes/ovmf/download_dependencies.sh b/runtimes/ovmf/download_dependencies.sh
new file mode 100644
index 000000000..178820d99
--- /dev/null
+++ b/runtimes/ovmf/download_dependencies.sh
@@ -0,0 +1,39 @@
+#! /bin/bash
+
+set -eo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+DOWNLOAD_DIR="${SCRIPT_DIR}/downloads"
+PATCH_DIR="${SCRIPT_DIR}/patches"
+
+GRUB_GIT_REPOSITORY="https://github.com/aleph-im/grub.git"
+GRUB_COMMIT="aleph/efi-secrets"
+GRUB_DIR="${DOWNLOAD_DIR}/grub"
+
+EDK2_GIT_REPOSITORY="https://github.com/tianocore/edk2.git"
+EDK2_COMMIT="edk2-stable202205"
+EDK2_DIR="${DOWNLOAD_DIR}/edk2"
+
+# Download Grub
+git clone  --depth 1 --branch "${GRUB_COMMIT}" ${GRUB_GIT_REPOSITORY} "${GRUB_DIR}"
+
+# Download EDK2 (=OVMF)
+git clone --recurse-submodules "${EDK2_GIT_REPOSITORY}" "${EDK2_DIR}"
+
+
+
+
+# Apply patches to EDK2
+EDK2_PATCH_DIR="${PATCH_DIR}/edk2"
+pushd "${EDK2_DIR}" > /dev/null
+git checkout "${EDK2_COMMIT}"
+git submodule update
+# Default user is needed by git am. only set it for the repo if not set already
+if ! git config user.name > /dev/null; then
+    git config --local user.name "Your Name"
+fi
+if ! git config user.email > /dev/null; then
+    git config  --local user.email "you@example.com"
+fi
+git am --ignore-space-change --ignore-whitespace "${EDK2_PATCH_DIR}/0001-Fix-invokation-of-cryptomount-s-for-AMD-SEV.patch"
+popd > /dev/null
diff --git a/runtimes/ovmf/patches/edk2/0001-Fix-invokation-of-cryptomount-s-for-AMD-SEV.patch b/runtimes/ovmf/patches/edk2/0001-Fix-invokation-of-cryptomount-s-for-AMD-SEV.patch
new file mode 100644
index 000000000..5c4f5e290
--- /dev/null
+++ b/runtimes/ovmf/patches/edk2/0001-Fix-invokation-of-cryptomount-s-for-AMD-SEV.patch
@@ -0,0 +1,58 @@
+From b3f1d358cc4098fb59a778d5340018a4e73ff87f Mon Sep 17 00:00:00 2001
+From: Olivier Desenfans 
+Date: Thu, 30 Jun 2022 10:38:18 +0200
+Subject: [PATCH] Fix invokation of cryptomount -s for AMD SEV
+
+The current implementation targeted the first version of James
+Bottomley's Grub patches. These patches have since been updated
+to move the secret loading part from a dedicated command to
+a secret-finding module that must be invoked with
+
+cryptomount -s MOD
+
+Fixed the name of the Grub module which was renamed from sevsecret
+to efisecret.
+---
+ OvmfPkg/AmdSev/Grub/grub.cfg | 10 ++--------
+ OvmfPkg/AmdSev/Grub/grub.sh  |  2 +-
+ 2 files changed, 3 insertions(+), 9 deletions(-)
+
+diff --git a/OvmfPkg/AmdSev/Grub/grub.cfg b/OvmfPkg/AmdSev/Grub/grub.cfg
+index 17be94277a..331baf798c 100644
+--- a/OvmfPkg/AmdSev/Grub/grub.cfg
++++ b/OvmfPkg/AmdSev/Grub/grub.cfg
+@@ -10,16 +10,10 @@
+ ##
+ 
+ echo "Entering grub config"
+-sevsecret
++cryptomount -s efisecret
+ if [ $? -ne 0 ]; then
+-    echo "Failed to locate anything in the SEV secret area, prompting for password"
++    echo "Failed to mount root securely, retrying with password prompt"
+     cryptomount -a
+-else
+-    cryptomount -s
+-    if [ $? -ne 0 ]; then
+-        echo "Failed to mount root securely, retrying with password prompt"
+-        cryptomount -a
+-    fi
+ fi
+ set root=
+ for f in (crypto*); do
+diff --git a/OvmfPkg/AmdSev/Grub/grub.sh b/OvmfPkg/AmdSev/Grub/grub.sh
+index 99807d7291..abec80d7da 100644
+--- a/OvmfPkg/AmdSev/Grub/grub.sh
++++ b/OvmfPkg/AmdSev/Grub/grub.sh
+@@ -44,7 +44,7 @@ GRUB_MODULES="
+             linux
+             linuxefi
+             reboot
+-            sevsecret
++            efisecret
+             "
+ basedir=$(dirname -- "$0")
+ 
+-- 
+2.25.1
+
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 86f83cca8..a192a9b4e 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -278,6 +278,8 @@ class Settings(BaseSettings):
         description="Confidential Computing default directory. Default to EXECUTION_ROOT/confidential",
     )
 
+    CONFIDENTIAL_SESSION_DIRECTORY: Path = Field(None, description="Default to EXECUTION_ROOT/sessions")
+
     # Tests on programs
 
     FAKE_DATA_PROGRAM: Optional[Path] = None
@@ -421,6 +423,7 @@ def setup(self):
         os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True)
         os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True)
         os.makedirs(self.CONFIDENTIAL_DIRECTORY, exist_ok=True)
+        os.makedirs(self.CONFIDENTIAL_SESSION_DIRECTORY, exist_ok=True)
 
         self.API_SERVER = self.API_SERVER.rstrip("/")
 
@@ -469,6 +472,8 @@ def __init__(
             self.RUNTIME_CACHE = self.CACHE_ROOT / "runtime"
         if not self.DATA_CACHE:
             self.DATA_CACHE = self.CACHE_ROOT / "data"
+        if not self.CONFIDENTIAL_DIRECTORY:
+            self.CONFIDENTIAL_DIRECTORY = self.CACHE_ROOT / "confidential"
         if not self.JAILER_BASE_DIRECTORY:
             self.JAILER_BASE_DIRECTORY = self.EXECUTION_ROOT / "jailer"
         if not self.PERSISTENT_VOLUMES_DIR:
@@ -479,8 +484,8 @@ def __init__(
             self.EXECUTION_LOG_DIRECTORY = self.EXECUTION_ROOT / "executions"
         if not self.JAILER_BASE_DIR:
             self.JAILER_BASE_DIR = self.EXECUTION_ROOT / "jailer"
-        if not self.CONFIDENTIAL_DIRECTORY:
-            self.CONFIDENTIAL_DIRECTORY = self.CACHE_ROOT / "confidential"
+        if not self.CONFIDENTIAL_SESSION_DIRECTORY:
+            self.CONFIDENTIAL_SESSION_DIRECTORY = self.EXECUTION_ROOT / "sessions"
 
     class Config:
         env_prefix = "ALEPH_VM_"
diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index 3232c82bf..2ba923abd 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -10,11 +10,13 @@
 
 from aleph.vm.hypervisors.firecracker.microvm import MicroVM
 from aleph.vm.hypervisors.qemu.qemuvm import QemuVM
+from aleph.vm.hypervisors.qemu_confidential.qemuvm import QemuConfidentialVM
 from aleph.vm.network.hostnetwork import Network, make_ipv6_allocator
 
 from .configuration import (
     Configuration,
     HypervisorType,
+    QemuConfidentialVMConfiguration,
     QemuVMConfiguration,
     VMConfiguration,
 )
@@ -71,6 +73,10 @@ async def execute_persistent_vm(config: Configuration):
 
         execution.prepare_start()
         process = await execution.start(config.vm_configuration.config_file_path)
+    elif isinstance(config.vm_configuration, QemuConfidentialVMConfiguration):  # FIXME
+        assert isinstance(config.vm_configuration, QemuConfidentialVMConfiguration)
+        execution = QemuConfidentialVM(config.vm_hash, config.vm_configuration)
+        process = await execution.start()
     else:
         assert isinstance(config.vm_configuration, QemuVMConfiguration)
         execution = QemuVM(config.vm_hash, config.vm_configuration)
@@ -90,7 +96,7 @@ def callback():
     loop.add_signal_handler(signal.SIGTERM, callback)
 
     await process.wait()
-    logger.info(f"Process terminated with {process.returncode}")
+    logger.warning(f"Process terminated with {process.returncode}")
 
 
 async def run_persistent_vm(config: Configuration):
diff --git a/src/aleph/vm/controllers/configuration.py b/src/aleph/vm/controllers/configuration.py
index f54acbcad..32f869bea 100644
--- a/src/aleph/vm/controllers/configuration.py
+++ b/src/aleph/vm/controllers/configuration.py
@@ -1,7 +1,7 @@
 import logging
 from enum import Enum
 from pathlib import Path
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 from pydantic import BaseModel
 
@@ -18,6 +18,12 @@ class VMConfiguration(BaseModel):
     init_timeout: float
 
 
+class QemuVMHostVolume(BaseModel):
+    mount: str
+    path_on_host: Path
+    read_only: bool
+
+
 class QemuVMConfiguration(BaseModel):
     qemu_bin_path: str
     cloud_init_drive_path: Optional[str]
@@ -27,6 +33,23 @@ class QemuVMConfiguration(BaseModel):
     vcpu_count: int
     mem_size_mb: int
     interface_name: Optional[str]
+    host_volumes: List[QemuVMHostVolume]
+
+
+class QemuConfidentialVMConfiguration(BaseModel):
+    qemu_bin_path: str
+    cloud_init_drive_path: Optional[str]
+    image_path: str
+    monitor_socket_path: Path
+    qmp_socket_path: Path
+    vcpu_count: int
+    mem_size_mb: int
+    interface_name: Optional[str]
+    host_volumes: List[QemuVMHostVolume]
+    ovmf_path: Path
+    sev_session_file: Path
+    sev_dh_cert_file: Path
+    sev_policy: int
 
 
 class HypervisorType(str, Enum):
@@ -38,7 +61,7 @@ class Configuration(BaseModel):
     vm_id: int
     vm_hash: str
     settings: Settings
-    vm_configuration: Union[QemuVMConfiguration, VMConfiguration]
+    vm_configuration: Union[QemuConfidentialVMConfiguration, QemuVMConfiguration, VMConfiguration]
     hypervisor: HypervisorType = HypervisorType.firecracker
 
 
diff --git a/src/aleph/vm/controllers/qemu/client.py b/src/aleph/vm/controllers/qemu/client.py
new file mode 100644
index 000000000..936f65b5b
--- /dev/null
+++ b/src/aleph/vm/controllers/qemu/client.py
@@ -0,0 +1,75 @@
+import qmp
+from pydantic import BaseModel
+
+
+class VmSevInfo(BaseModel):
+    enabled: bool
+    api_major: int
+    api_minor: int
+    build_id: int
+    policy: int
+    state: str
+    handle: int
+
+
+class QemuVmClient:
+    def __init__(self, vm):
+        self.vm = vm
+        if not (vm.qmp_socket_path and vm.qmp_socket_path.exists()):
+            raise Exception("VM is not running")
+        client = qmp.QEMUMonitorProtocol(str(vm.qmp_socket_path))
+        client.connect()
+
+        # qmp_client = qmp.QEMUMonitorProtocol(address=("localhost", vm.qmp_port))
+        self.qmp_client = client
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def close(self) -> None:
+        self.qmp_client.close()
+
+    def query_sev_info(self) -> VmSevInfo:
+        caps = self.qmp_client.command("query-sev")
+        return VmSevInfo(
+            enabled=caps["enabled"],
+            api_major=caps["api-major"],
+            api_minor=caps["api-minor"],
+            handle=caps["handle"],
+            state=caps["state"],
+            build_id=caps["build-id"],
+            policy=caps["policy"],
+        )
+
+    def query_launch_measure(self) -> str:
+        measure = self.qmp_client.command("query-sev-launch-measure")
+        return measure["data"]
+
+    def inject_secret(self, packet_header: str, secret: str) -> None:
+        """
+        Injects the secret in the SEV secret area.
+
+        :param packet_header: The packet header, as a base64 string.
+        :param secret: The encoded secret, as a base64 string.
+        """
+
+        self.qmp_client.command(
+            "sev-inject-launch-secret",
+            **{"packet-header": packet_header, "secret": secret},
+        )
+
+    def continue_execution(self) -> None:
+        """
+        Resumes the execution of the VM.
+        """
+        self.qmp_client.command("cont")
+
+    def query_status(self) -> None:
+        """
+        Get running status.
+        """
+        # {'status': 'prelaunch', 'singlestep': False, 'running': False}
+        return self.qmp_client.command("query-status")
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index 81dea8ff3..e3c2e5435 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -1,3 +1,4 @@
+import asyncio
 import json
 import logging
 import shutil
@@ -17,6 +18,7 @@
     Configuration,
     HypervisorType,
     QemuVMConfiguration,
+    QemuVMHostVolume,
     save_controller_configuration,
 )
 from aleph.vm.controllers.firecracker.executable import (
@@ -34,11 +36,17 @@
 
 
 class AlephQemuResources(AlephFirecrackerResources):
-    async def download_all(self) -> None:
+    async def download_runtime(self) -> None:
         volume = self.message_content.rootfs
         parent_image_path = await get_rootfs_base_path(volume.parent.ref)
         self.rootfs_path = await self.make_writable_volume(parent_image_path, volume)
 
+    async def download_all(self):
+        await asyncio.gather(
+            self.download_runtime(),
+            self.download_volumes(),
+        )
+
     async def make_writable_volume(self, parent_image_path, volume: Union[PersistentVolume, RootfsVolume]):
         """Create a new qcow2 image file based on the passed one, that we give to the VM to write onto"""
         qemu_img_path: Optional[str] = shutil.which("qemu-img")
@@ -95,7 +103,6 @@ class AlephQemuInstance(Generic[ConfigurationType], CloudInitMixin, AlephVmContr
     is_instance: bool
     qemu_process: Optional[Process]
     support_snapshot = False
-    qmp_socket_path = None
     persistent = True
     controller_configuration: Configuration
 
@@ -115,14 +122,13 @@ def __init__(
         tap_interface: Optional[TapInterface] = None,
     ):
         self.vm_id = vm_id
+        self.vm_hash = vm_hash
         self.resources = resources
         self.enable_networking = enable_networking and settings.ALLOW_VM_NETWORKING
         self.hardware_resources = hardware_resources
         self.tap_interface = tap_interface
         self.qemu_process = None
 
-        self.vm_hash = vm_hash
-
     # TODO : wait for andress soltion for pid handling
     def to_dict(self):
         """Dict representation of the virtual machine. Used to record resource usage and for JSON serialization."""
@@ -161,7 +167,7 @@ async def configure(self):
 
         logger.debug(f"Making  Qemu configuration: {self} ")
         monitor_socket_path = settings.EXECUTION_ROOT / (str(self.vm_id) + "-monitor.socket")
-        self.qmp_socket_path = qmp_socket_path = settings.EXECUTION_ROOT / (str(self.vm_id) + "-qmp.socket")
+
         cloud_init_drive = await self._create_cloud_init_drive()
 
         image_path = str(self.resources.rootfs_path)
@@ -179,10 +185,18 @@ async def configure(self):
             cloud_init_drive_path=cloud_init_drive_path,
             image_path=image_path,
             monitor_socket_path=monitor_socket_path,
-            qmp_socket_path=qmp_socket_path,
+            qmp_socket_path=self.qmp_socket_path,
             vcpu_count=vcpu_count,
             mem_size_mb=mem_size_mb,
             interface_name=interface_name,
+            host_volumes=[
+                QemuVMHostVolume(
+                    mount=volume.mount,
+                    path_on_host=volume.path_on_host,
+                    read_only=volume.read_only,
+                )
+                for volume in self.resources.volumes
+            ],
         )
 
         configuration = Configuration(
@@ -192,7 +206,7 @@ async def configure(self):
             vm_configuration=vm_configuration,
             hypervisor=HypervisorType.qemu,
         )
-
+        logger.debug(configuration)
         save_controller_configuration(self.vm_hash, configuration)
 
     def save_controller_configuration(self):
@@ -202,6 +216,10 @@ def save_controller_configuration(self):
         path.chmod(0o644)
         return path
 
+    @property
+    def qmp_socket_path(self) -> Path:
+        return settings.EXECUTION_ROOT / f"{self.vm_id}-qmp.socket"
+
     async def start(self):
         # Start via systemd not here
         raise NotImplementedError()
@@ -214,7 +232,6 @@ async def wait_for_init(self) -> None:
         if not ip:
             msg = "Host IP not available"
             raise ValueError(msg)
-
         ip = ip.split("/", 1)[0]
 
         attempts = 30
diff --git a/src/aleph/vm/controllers/qemu_confidential/__init__.py b/src/aleph/vm/controllers/qemu_confidential/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/aleph/vm/controllers/qemu_confidential/instance.py b/src/aleph/vm/controllers/qemu_confidential/instance.py
new file mode 100644
index 000000000..2b22044ec
--- /dev/null
+++ b/src/aleph/vm/controllers/qemu_confidential/instance.py
@@ -0,0 +1,145 @@
+import asyncio
+import logging
+import shutil
+from asyncio.subprocess import Process
+from pathlib import Path
+from typing import Callable, Optional
+
+from aleph_message.models import ItemHash
+from aleph_message.models.execution.environment import AMDSEVPolicy, MachineResources
+
+from aleph.vm.conf import settings
+from aleph.vm.controllers.configuration import (
+    Configuration,
+    HypervisorType,
+    QemuConfidentialVMConfiguration,
+    QemuVMHostVolume,
+    save_controller_configuration,
+)
+from aleph.vm.controllers.qemu import AlephQemuInstance
+from aleph.vm.controllers.qemu.instance import (
+    AlephQemuResources,
+    ConfigurationType,
+    logger,
+)
+from aleph.vm.network.interfaces import TapInterface
+from aleph.vm.storage import get_existing_file
+
+logger = logging.getLogger(__name__)
+
+
+class AlephQemuConfidentialResources(AlephQemuResources):
+    firmware_path: Path
+
+    async def download_firmware(self):
+        firmware = self.message_content.environment.trusted_execution.firmware
+        self.firmware_path = await get_existing_file(firmware)
+
+    async def download_all(self):
+        await asyncio.gather(
+            self.download_runtime(),
+            self.download_firmware(),
+            self.download_volumes(),
+        )
+
+
+class AlephQemuConfidentialInstance(AlephQemuInstance):
+    vm_id: int
+    vm_hash: ItemHash
+    resources: AlephQemuConfidentialResources
+    enable_console: bool
+    enable_networking: bool
+    hardware_resources: MachineResources
+    tap_interface: Optional[TapInterface] = None
+    vm_configuration: Optional[ConfigurationType]
+    is_instance: bool
+    qemu_process: Optional[Process]
+    support_snapshot = False
+    persistent = True
+    _queue_cancellers: dict[asyncio.Queue, Callable] = {}
+    controller_configuration: Configuration
+    confidential_policy: int
+
+    def __repr__(self):
+        return f""
+
+    def __str__(self):
+        return f"vm-{self.vm_id}"
+
+    def __init__(
+        self,
+        vm_id: int,
+        vm_hash: ItemHash,
+        resources: AlephQemuConfidentialResources,
+        enable_networking: bool = False,
+        confidential_policy: int = AMDSEVPolicy.NO_DBG,
+        hardware_resources: MachineResources = MachineResources(),
+        tap_interface: Optional[TapInterface] = None,
+    ):
+        super().__init__(vm_id, vm_hash, resources, enable_networking, hardware_resources, tap_interface)
+        self.confidential_policy = confidential_policy
+
+    async def setup(self):
+        pass
+
+    async def configure(self):
+        """Configure the VM by saving controller service configuration"""
+
+        logger.debug(f"Making  Qemu configuration: {self} ")
+        monitor_socket_path = settings.EXECUTION_ROOT / (str(self.vm_id) + "-monitor.socket")
+
+        cloud_init_drive = await self._create_cloud_init_drive()
+
+        image_path = str(self.resources.rootfs_path)
+        firmware_path = str(self.resources.firmware_path)
+        vcpu_count = self.hardware_resources.vcpus
+        mem_size_mib = self.hardware_resources.memory
+        mem_size_mb = str(int(mem_size_mib / 1024 / 1024 * 1000 * 1000))
+
+        vm_session_path = settings.CONFIDENTIAL_SESSION_DIRECTORY / self.vm_hash
+        session_file_path = vm_session_path / "vm_session.b64"
+        godh_file_path = vm_session_path / "vm_godh.b64"
+
+        qemu_bin_path = shutil.which("qemu-system-x86_64")
+        interface_name = None
+        if self.tap_interface:
+            interface_name = self.tap_interface.device_name
+        cloud_init_drive_path = str(cloud_init_drive.path_on_host) if cloud_init_drive else None
+        vm_configuration = QemuConfidentialVMConfiguration(
+            qemu_bin_path=qemu_bin_path,
+            cloud_init_drive_path=cloud_init_drive_path,
+            image_path=image_path,
+            monitor_socket_path=monitor_socket_path,
+            qmp_socket_path=self.qmp_socket_path,
+            vcpu_count=vcpu_count,
+            mem_size_mb=mem_size_mb,
+            interface_name=interface_name,
+            ovmf_path=firmware_path,
+            sev_session_file=session_file_path,
+            sev_dh_cert_file=godh_file_path,
+            sev_policy=self.confidential_policy,
+            host_volumes=[
+                QemuVMHostVolume(
+                    mount=volume.mount,
+                    path_on_host=volume.path_on_host,
+                    read_only=volume.read_only,
+                )
+                for volume in self.resources.volumes
+            ],
+        )
+
+        configuration = Configuration(
+            vm_id=self.vm_id,
+            vm_hash=self.vm_hash,
+            settings=settings,
+            vm_configuration=vm_configuration,
+            hypervisor=HypervisorType.qemu,
+        )
+        logger.debug(configuration)
+
+        save_controller_configuration(self.vm_hash, configuration)
+
+    async def wait_for_init(self) -> None:
+        """Wait for the init process of the instance to be ready."""
+        # FIXME: Cannot ping since network is not set up yet.
+        return
diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index 0d49403c8..384c31b05 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -1,7 +1,6 @@
 import asyncio
-import sys
-from asyncio import Task
 from asyncio.subprocess import Process
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Optional, TextIO
 
@@ -12,6 +11,12 @@
 from aleph.vm.controllers.qemu.instance import logger
 
 
+@dataclass
+class HostVolume:
+    path_on_host: Path
+    read_only: bool
+
+
 class QemuVM:
     qemu_bin_path: str
     cloud_init_drive_path: Optional[str]
@@ -21,7 +26,8 @@ class QemuVM:
     vcpu_count: int
     mem_size_mb: int
     interface_name: str
-    qemu_process = None
+    qemu_process: Optional[Process] = None
+    host_volumes: list[HostVolume]
 
     def __repr__(self) -> str:
         if self.qemu_process:
@@ -40,6 +46,14 @@ def __init__(self, vm_hash, config: QemuVMConfiguration):
         self.interface_name = config.interface_name
         self.vm_hash = vm_hash
 
+        self.host_volumes = [
+            HostVolume(
+                path_on_host=volume.path_on_host,
+                read_only=volume.read_only,
+            )
+            for volume in config.host_volumes
+        ]
+
     @property
     def _journal_stdout_name(self) -> str:
         return f"vm-{self.vm_hash}-stdout"
@@ -90,6 +104,11 @@ async def start(
             # "-serial", "telnet:localhost:4321,server,nowait",
             # "-snapshot",  # Do not save anything to disk
         ]
+        for volume in self.host_volumes:
+            args += [
+                "-drive",
+                f"file={volume.path_on_host},format=raw,readonly={'on' if volume.read_only else 'off'},media=disk,if=virtio",
+            ]
         if self.interface_name:
             # script=no, downscript=no tell qemu not to try to set up the network itself
             args += ["-net", "nic,model=virtio", "-net", f"tap,ifname={self.interface_name},script=no,downscript=no"]
@@ -97,6 +116,7 @@ async def start(
         if self.cloud_init_drive_path:
             args += ["-cdrom", f"{self.cloud_init_drive_path}"]
         print(*args)
+
         self.qemu_process = proc = await asyncio.create_subprocess_exec(
             *args,
             stdin=asyncio.subprocess.DEVNULL,
diff --git a/src/aleph/vm/hypervisors/qemu_confidential/__init__.py b/src/aleph/vm/hypervisors/qemu_confidential/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
new file mode 100644
index 000000000..6b76f62f3
--- /dev/null
+++ b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
@@ -0,0 +1,135 @@
+import asyncio
+from asyncio.subprocess import Process
+from pathlib import Path
+from typing import TextIO
+
+from aleph_message.models.execution.environment import AMDSEVPolicy
+from cpuid.features import secure_encryption_info
+from systemd import journal
+
+from aleph.vm.controllers.configuration import QemuConfidentialVMConfiguration
+from aleph.vm.controllers.qemu.instance import logger
+from aleph.vm.hypervisors.qemu.qemuvm import QemuVM
+
+
+class QemuConfidentialVM(QemuVM):
+
+    sev_policy: str = hex(AMDSEVPolicy.NO_DBG)
+    sev_dh_cert_file: Path  # "vm_godh.b64"
+    sev_session_file: Path  # "vm_session.b64"
+
+    def __repr__(self) -> str:
+        if self.qemu_process:
+            return f""
+        else:
+            return ""
+
+    def __init__(self, vm_hash, config: QemuConfidentialVMConfiguration):
+        super().__init__(vm_hash, config)
+        self.qemu_bin_path = config.qemu_bin_path
+        self.cloud_init_drive_path = config.cloud_init_drive_path
+        self.image_path = config.image_path
+        self.monitor_socket_path = config.monitor_socket_path
+        self.qmp_socket_path = config.qmp_socket_path
+        self.vcpu_count = config.vcpu_count
+        self.mem_size_mb = config.mem_size_mb
+        self.interface_name = config.interface_name
+        self.log_queues: list[asyncio.Queue] = []
+        self.ovmf_path: Path = config.ovmf_path
+        self.sev_session_file = config.sev_session_file
+        self.sev_dh_cert_file = config.sev_dh_cert_file
+        self.sev_policy = hex(config.sev_policy)
+
+    def prepare_start(self):
+        pass
+
+    async def start(
+        self,
+    ) -> Process:
+        # Based on the command
+        #  qemu-system-x86_64 -enable-kvm -m 2048 -net nic,model=virtio
+        # -net tap,ifname=tap0,script=no,downscript=no -drive file=alpine.qcow2,media=disk,if=virtio -nographic
+        # hardware_resources.published ports -> not implemented at the moment
+        # hardware_resources.seconds -> only for microvm
+        journal_stdout: TextIO = journal.stream(self._journal_stdout_name)
+        journal_stderr: TextIO = journal.stream(self._journal_stderr_name)
+
+        # TODO : ensure this is ok at launch
+        sev_info = secure_encryption_info()
+        if sev_info is None:
+            raise ValueError("Not running on an AMD SEV platform?")
+        godh = self.sev_dh_cert_file
+        launch_blob = self.sev_session_file
+
+        if not (godh.is_file() and launch_blob.is_file()):
+            raise FileNotFoundError("Missing guest owner certificates, cannot start the VM.`")
+        args = [
+            self.qemu_bin_path,
+            "-enable-kvm",
+            "-nodefaults",
+            "-m",
+            str(self.mem_size_mb),
+            "-smp",
+            str(self.vcpu_count),
+            "-drive",
+            f"if=pflash,format=raw,unit=0,file={self.ovmf_path},readonly=on",
+            "-drive",
+            f"file={self.image_path},media=disk,if=virtio,format=qcow2",
+            # To debug you can pass gtk or curses instead
+            "-display",
+            "none",
+            "--no-reboot",  # Rebooting from inside the VM shuts down the machine
+            # Listen for commands on this socket
+            "-monitor",
+            f"unix:{self.monitor_socket_path},server,nowait",
+            # Listen for commands on this socket (QMP protocol in json). Supervisor use it to send shutdown or start
+            # command
+            "-qmp",
+            f"unix:{self.qmp_socket_path},server,nowait",
+            # Tell to put the output to std fd, so we can include them in the log
+            "-nographic",
+            "-serial",
+            "stdio",
+            "--no-reboot",  # Rebooting from inside the VM shuts down the machine
+            "-S",
+            # Confidential options
+            "-object",
+            f"sev-guest,id=sev0,policy={self.sev_policy},cbitpos={sev_info.c_bit_position},"
+            f"reduced-phys-bits={sev_info.phys_addr_reduction},"
+            f"dh-cert-file={godh},session-file={launch_blob}",
+            "-machine",
+            "confidential-guest-support=sev0",
+            # Linux kernel 6.9 added a control on the RDRAND function to ensure that the random numbers generation
+            # works well, on Qemu emulation for confidential computing the CPU model us faked and this makes control
+            # raise an error and prevent boot. Passing the argument --cpu host instruct the VM to use the same CPU
+            # model than the host thus the VM's kernel knows which method is used to get random numbers (Intel and
+            # AMD have different methods) and properly boot.
+            "-cpu",
+            "host",
+            # Uncomment following for debug
+            # "-serial", "telnet:localhost:4321,server,nowait",
+            # "-snapshot",  # Do not save anything to disk
+        ]
+        for volume in self.host_volumes:
+            args += [
+                "-drive",
+                f"file={volume.path_on_host},format=raw,readonly={'on' if volume.read_only else 'off'},media=disk,if=virtio",
+            ]
+        if self.interface_name:
+            # script=no, downscript=no tell qemu not to try to set up the network itself
+            args += ["-net", "nic,model=virtio", "-net", f"tap,ifname={self.interface_name},script=no,downscript=no"]
+
+        if self.cloud_init_drive_path:
+            args += ["-cdrom", f"{self.cloud_init_drive_path}"]
+        print(*args)
+        self.qemu_process = proc = await asyncio.create_subprocess_exec(
+            *args,
+            stdin=asyncio.subprocess.DEVNULL,
+            stdout=journal_stdout,
+            stderr=journal_stderr,
+        )
+
+        print(
+            f"Started QemuVm {self}, {proc}. Log available with: journalctl -t  {self._journal_stdout_name} -t {self._journal_stderr_name}"
+        )
+        return proc
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 5a44c132a..6f87b2363 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -26,6 +26,10 @@
 from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
 from aleph.vm.controllers.interface import AlephVmControllerInterface
 from aleph.vm.controllers.qemu.instance import AlephQemuInstance, AlephQemuResources
+from aleph.vm.controllers.qemu_confidential.instance import (
+    AlephQemuConfidentialInstance,
+    AlephQemuConfidentialResources,
+)
 from aleph.vm.network.interfaces import TapInterface
 from aleph.vm.orchestrator.metrics import (
     ExecutionRecord,
@@ -104,6 +108,10 @@ def is_program(self) -> bool:
     def is_instance(self) -> bool:
         return isinstance(self.message, InstanceContent)
 
+    @property
+    def is_confidential(self) -> bool:
+        return True if self.message.environment.trusted_execution else False
+
     @property
     def hypervisor(self) -> HypervisorType:
         if self.is_program:
@@ -178,14 +186,19 @@ async def prepare(self) -> None:
                 return
 
             self.times.preparing_at = datetime.now(tz=timezone.utc)
-            resources: Union[AlephProgramResources, AlephInstanceResources, AlephQemuResources]
+            resources: Union[
+                AlephProgramResources, AlephInstanceResources, AlephQemuResources, AlephQemuConfidentialInstance
+            ]
             if self.is_program:
                 resources = AlephProgramResources(self.message, namespace=self.vm_hash)
             elif self.is_instance:
                 if self.hypervisor == HypervisorType.firecracker:
                     resources = AlephInstanceResources(self.message, namespace=self.vm_hash)
                 elif self.hypervisor == HypervisorType.qemu:
-                    resources = AlephQemuResources(self.message, namespace=self.vm_hash)
+                    if self.is_confidential:
+                        resources = AlephQemuConfidentialResources(self.message, namespace=self.vm_hash)
+                    else:
+                        resources = AlephQemuResources(self.message, namespace=self.vm_hash)
                 else:
                     raise ValueError(f"Unknown hypervisor type {self.hypervisor}")
             else:
@@ -231,15 +244,26 @@ def create(
                     prepare_jailer=prepare,
                 )
             elif self.hypervisor == HypervisorType.qemu:
-                assert isinstance(self.resources, AlephQemuResources)
-                self.vm = vm = AlephQemuInstance(
-                    vm_id=vm_id,
-                    vm_hash=self.vm_hash,
-                    resources=self.resources,
-                    enable_networking=self.message.environment.internet,
-                    hardware_resources=self.message.resources,
-                    tap_interface=tap_interface,
-                )
+                if self.is_confidential:
+                    assert isinstance(self.resources, AlephQemuConfidentialResources)
+                    self.vm = vm = AlephQemuConfidentialInstance(
+                        vm_id=vm_id,
+                        vm_hash=self.vm_hash,
+                        resources=self.resources,
+                        enable_networking=self.message.environment.internet,
+                        hardware_resources=self.message.resources,
+                        tap_interface=tap_interface,
+                    )
+                else:
+                    assert isinstance(self.resources, AlephQemuResources)
+                    self.vm = vm = AlephQemuInstance(
+                        vm_id=vm_id,
+                        vm_hash=self.vm_hash,
+                        resources=self.resources,
+                        enable_networking=self.message.environment.internet,
+                        hardware_resources=self.message.resources,
+                        tap_interface=tap_interface,
+                    )
             else:
                 raise Exception("Unknown VM")
         else:
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index 8dec7e963..da29084dd 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -57,31 +57,12 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool
 
     logger.debug(f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}")
 
-    try:
-        execution = await pool.create_a_vm(
-            vm_hash=vm_hash,
-            message=message.content,
-            original=original_message.content,
-            persistent=persistent,
-        )
-    except ResourceDownloadError as error:
-        logger.exception(error)
-        pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPBadRequest(reason="Code, runtime or data not available") from error
-    except FileTooLargeError as error:
-        raise HTTPInternalServerError(reason=error.args[0]) from error
-    except VmSetupError as error:
-        logger.exception(error)
-        pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(reason="Error during vm initialisation") from error
-    except MicroVMFailedInitError as error:
-        logger.exception(error)
-        pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(reason="Error during runtime initialisation") from error
-    except HostNotFoundError as error:
-        logger.exception(error)
-        pool.forget_vm(vm_hash=vm_hash)
-        raise HTTPInternalServerError(reason="Host did not respond to ping") from error
+    execution = await pool.create_a_vm(
+        vm_hash=vm_hash,
+        message=message.content,
+        original=original_message.content,
+        persistent=persistent,
+    )
 
     return execution
 
@@ -273,6 +254,8 @@ async def start_persistent_vm(vm_hash: ItemHash, pubsub: Optional[PubSub], pool:
     if not execution:
         logger.info(f"Starting persistent virtual machine with id: {vm_hash}")
         execution = await create_vm_execution(vm_hash=vm_hash, pool=pool, persistent=True)
+    else:
+        logger.info(f"{vm_hash} is already running")
 
     await execution.becomes_ready()
 
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index a3905cae2..09cb79ede 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -47,6 +47,9 @@
     update_allocations,
 )
 from .views.operator import (
+    operate_confidential_initialize,
+    operate_confidential_inject_secret,
+    operate_confidential_measurement,
     operate_erase,
     operate_expire,
     operate_logs,
@@ -107,6 +110,9 @@ def setup_webapp():
         web.post("/control/machine/{ref}/stop", operate_stop),
         web.post("/control/machine/{ref}/erase", operate_erase),
         web.post("/control/machine/{ref}/reboot", operate_reboot),
+        web.post("/control/machine/{ref}/confidential/initialize", operate_confidential_initialize),
+        web.get("/control/machine/{ref}/confidential/measurement", operate_confidential_measurement),
+        web.post("/control/machine/{ref}/confidential/inject_secret", operate_confidential_inject_secret),
         # /status APIs are used to check that the VM Orchestrator is running properly
         web.get("/status/check/fastapi", status_check_fastapi),
         web.get("/status/check/fastapi/legacy", status_check_fastapi_legacy),
diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index a50dde45e..1043ce994 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -46,7 +46,6 @@ class SignedPubKeyPayload(BaseModel):
     # {'pubkey': {'alg': 'ES256', 'crv': 'P-256', 'ext': True, 'key_ops': ['verify'], 'kty': 'EC',
     #  'x': '4blJBYpltvQLFgRvLE-2H7dsMr5O0ImHkgOnjUbG2AU', 'y': '5VHnq_hUSogZBbVgsXMs0CjrVfMy4Pa3Uv2BEBqfrN4'}
     # alg: Literal["ECDSA"]
-    domain: str
     address: str
     expires: str
 
@@ -100,6 +99,7 @@ def content(self) -> SignedPubKeyPayload:
 class SignedOperationPayload(BaseModel):
     time: datetime.datetime
     method: Union[Literal["POST"], Literal["GET"]]
+    domain: str
     path: str
     # body_sha256: str  # disabled since there is no body
 
@@ -201,8 +201,8 @@ async def authenticate_jwk(request: web.Request) -> str:
     """Authenticate a request using the X-SignedPubKey and X-SignedOperation headers."""
     signed_pubkey = get_signed_pubkey(request)
     signed_operation = get_signed_operation(request)
-    if signed_pubkey.content.domain != settings.DOMAIN_NAME:
-        logger.debug(f"Invalid domain '{signed_pubkey.content.domain}' != '{settings.DOMAIN_NAME}'")
+    if signed_operation.content.domain != settings.DOMAIN_NAME:
+        logger.debug(f"Invalid domain '{signed_operation.content.domain}' != '{settings.DOMAIN_NAME}'")
         raise web.HTTPUnauthorized(reason="Invalid domain")
     if signed_operation.content.path != request.path:
         logger.debug(f"Invalid path '{signed_operation.content.path}' != '{request.path}'")
@@ -217,8 +217,8 @@ async def authenticate_websocket_message(message) -> str:
     """Authenticate a websocket message since JS cannot configure headers on WebSockets."""
     signed_pubkey = SignedPubKeyHeader.parse_obj(message["X-SignedPubKey"])
     signed_operation = SignedOperation.parse_obj(message["X-SignedOperation"])
-    if signed_pubkey.content.domain != settings.DOMAIN_NAME:
-        logger.debug(f"Invalid domain '{signed_pubkey.content.domain}' != '{settings.DOMAIN_NAME}'")
+    if signed_operation.content.domain != settings.DOMAIN_NAME:
+        logger.debug(f"Invalid domain '{signed_operation.content.domain}' != '{settings.DOMAIN_NAME}'")
         raise web.HTTPUnauthorized(reason="Invalid domain")
     return verify_signed_operation(signed_operation, signed_pubkey)
 
@@ -237,6 +237,7 @@ async def wrapper(request):
             logging.exception(e)
             raise
 
+        # authenticated_sender is the authenticted wallet address of the requester (as a string)
         response = await handler(request, authenticated_sender)
         return response
 
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index cd8fbae14..346b22d17 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -1,21 +1,26 @@
+import json
 import logging
 from datetime import timedelta
 
 import aiohttp.web_exceptions
+import pydantic
 from aiohttp import web
 from aiohttp.web_urldispatcher import UrlMappingMatchInfo
 from aleph_message.exceptions import UnknownHashError
 from aleph_message.models import ItemHash
 from aleph_message.models.execution import BaseExecutableContent
+from pydantic import BaseModel
 
+from aleph.vm.conf import settings
+from aleph.vm.controllers.qemu.client import QemuVmClient
 from aleph.vm.models import VmExecution
-from aleph.vm.orchestrator.run import create_vm_execution
+from aleph.vm.orchestrator.run import create_vm_execution_or_raise_http_error
 from aleph.vm.orchestrator.views.authentication import (
     authenticate_websocket_message,
     require_jwk_authentication,
 )
 from aleph.vm.pool import VmPool
-from aleph.vm.utils import cors_allow_all
+from aleph.vm.utils import cors_allow_all, dumps_for_json
 
 logger = logging.getLogger(__name__)
 
@@ -160,6 +165,50 @@ async def operate_expire(request: web.Request, authenticated_sender: str) -> web
     return web.Response(status=200, body=f"Expiring VM with ref {vm_hash} in {timeout} seconds")
 
 
+@cors_allow_all
+@require_jwk_authentication
+async def operate_confidential_initialize(request: web.Request, authenticated_sender: str) -> web.Response:
+    """Start the confidential virtual machine if possible."""
+    # TODO: Add user authentication
+    vm_hash = get_itemhash_or_400(request.match_info)
+
+    pool: VmPool = request.app["vm_pool"]
+    logger.debug(f"Iterating through running executions... {pool.executions}")
+    execution = get_execution_or_404(vm_hash, pool=pool)
+
+    if not is_sender_authorized(authenticated_sender, execution.message):
+        return web.Response(status=403, body="Unauthorized sender")
+
+    if execution.is_running:
+        return web.Response(status=403, body=f"VM with ref {vm_hash} already running")
+
+    if not execution.is_confidential:
+        return web.Response(status=403, body=f"Operation not allowed for VM {vm_hash} because it isn't confidential")
+
+    post = await request.post()
+
+    vm_session_path = settings.CONFIDENTIAL_SESSION_DIRECTORY / vm_hash
+    vm_session_path.mkdir(exist_ok=True)
+
+    session_file_content = post.get("session")
+    if not session_file_content:
+        return web.Response(status=403, body=f"Session file required for VM with ref {vm_hash}")
+
+    session_file_path = vm_session_path / "vm_session.b64"
+    session_file_path.write_bytes(session_file_content.file.read())
+
+    godh_file_content = post.get("godh")
+    if not godh_file_content:
+        return web.Response(status=403, body=f"GODH file required for VM with ref {vm_hash}")
+
+    godh_file_path = vm_session_path / "vm_godh.b64"
+    godh_file_path.write_bytes(godh_file_content.file.read())
+
+    pool.systemd_manager.enable_and_start(execution.controller_service)
+
+    return web.Response(status=200, body=f"Started VM with ref {vm_hash}")
+
+
 @cors_allow_all
 @require_jwk_authentication
 async def operate_stop(request: web.Request, authenticated_sender: str) -> web.Response:
@@ -206,12 +255,84 @@ async def operate_reboot(request: web.Request, authenticated_sender: str) -> web
             await pool.stop_vm(vm_hash)
             pool.forget_vm(vm_hash)
 
-            await create_vm_execution(vm_hash=vm_hash, pool=pool)
+            await create_vm_execution_or_raise_http_error(vm_hash=vm_hash, pool=pool)
         return web.Response(status=200, body=f"Rebooted VM with ref {vm_hash}")
     else:
         return web.Response(status=200, body=f"Starting VM (was not running) with ref {vm_hash}")
 
 
+@cors_allow_all
+@require_jwk_authentication
+async def operate_confidential_measurement(request: web.Request, authenticated_sender) -> web.Response:
+    """
+    Fetch the sev measurement for the VM
+    """
+    vm_hash = get_itemhash_or_400(request.match_info)
+    pool: VmPool = request.app["vm_pool"]
+    execution = get_execution_or_404(vm_hash, pool=pool)
+
+    if not is_sender_authorized(authenticated_sender, execution.message):
+        return web.Response(status=403, body="Unauthorized sender")
+
+    if not execution.is_running:
+        raise web.HTTPForbidden(body="Operation not running")
+    vm_client = QemuVmClient(execution.vm)
+    vm_sev_info = vm_client.query_sev_info()
+    launch_measure = vm_client.query_launch_measure()
+
+    return web.json_response(
+        data={"sev_info": vm_sev_info, "launch_measure": launch_measure},
+        status=200,
+        dumps=dumps_for_json,
+    )
+
+
+class InjectSecretParams(BaseModel):
+    """
+    packet_header: as base64 string
+    secret : encrypted secret table as base64 string
+    """
+
+    packet_header: str
+    secret: str
+
+
+@cors_allow_all
+@require_jwk_authentication
+async def operate_confidential_inject_secret(request: web.Request, authenticated_sender) -> web.Response:
+    """
+    Send secret to the VM and start it
+    """
+    try:
+        data = await request.json()
+        params = InjectSecretParams.parse_obj(data)
+    except json.JSONDecodeError:
+        return web.HTTPBadRequest(reason="Body is not valid JSON")
+    except pydantic.ValidationError as error:
+        return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code)
+
+    vm_hash = get_itemhash_or_400(request.match_info)
+    pool: VmPool = request.app["vm_pool"]
+    execution = get_execution_or_404(vm_hash, pool=pool)
+    if not is_sender_authorized(authenticated_sender, execution.message):
+        return web.Response(status=403, body="Unauthorized sender")
+
+    # if not execution.is_running:
+    #     raise web.HTTPForbidden(body="Operation not running")
+    vm_client = QemuVmClient(execution.vm)
+    vm_client.inject_secret(params.packet_header, params.secret)
+    vm_client.continue_execution()
+
+    status = vm_client.query_status()
+    print(status["status"] != "running")
+
+    return web.json_response(
+        data={"status": status},
+        status=200,
+        dumps=dumps_for_json,
+    )
+
+
 @cors_allow_all
 @require_jwk_authentication
 async def operate_erase(request: web.Request, authenticated_sender: str) -> web.Response:
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 3e5c5f3ec..01e5afd72 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -123,7 +123,9 @@ async def create_a_vm(
                 await execution.start()
 
                 # Start VM and snapshots automatically
-                if execution.persistent:
+                # If the execution is confidential, don't start it because we need to wait for the session certificate
+                # files, use the endpoint /control/machine/{ref}/confidential/initialize to get session files and start the VM
+                if execution.persistent and not execution.is_confidential:
                     self.systemd_manager.enable_and_start(execution.controller_service)
                     await execution.wait_for_init()
                     if execution.is_program and execution.vm:
diff --git a/src/aleph/vm/sevclient.py b/src/aleph/vm/sevclient.py
index a5b4ed154..01f25acf9 100644
--- a/src/aleph/vm/sevclient.py
+++ b/src/aleph/vm/sevclient.py
@@ -19,7 +19,7 @@ def __init__(self, sev_dir: Path, sev_ctl_executable: Path):
     async def sev_ctl_cmd(self, *args) -> bytes:
         """Run a command of the 'sevctl' tool."""
         return await run_in_subprocess(
-            [self.sev_ctl_executable, *args],
+            [str(self.sev_ctl_executable), *args],
             check=True,
         )
 
diff --git a/src/aleph/vm/utils/__init__.py b/src/aleph/vm/utils/__init__.py
index 35bbf2ff7..d96b519f1 100644
--- a/src/aleph/vm/utils/__init__.py
+++ b/src/aleph/vm/utils/__init__.py
@@ -100,6 +100,7 @@ def create_task_log_exceptions(coro: Coroutine, *, name=None):
 
 async def run_in_subprocess(command: list[str], check: bool = True, stdin_input: Optional[bytes] = None) -> bytes:
     """Run the specified command in a subprocess, returns the stdout of the process."""
+    command = [str(arg) for arg in command]
     logger.debug(f"command: {' '.join(command)}")
 
     process = await asyncio.create_subprocess_exec(
diff --git a/tests/supervisor/test_authentication.py b/tests/supervisor/test_authentication.py
index f4269a4ad..77ba154d7 100644
--- a/tests/supervisor/test_authentication.py
+++ b/tests/supervisor/test_authentication.py
@@ -74,7 +74,6 @@ async def test_require_jwk_authentication_expired(aiohttp_client):
     pubkey = {
         "pubkey": json.loads(key.export_public()),
         "alg": "ECDSA",
-        "domain": "localhost",
         "address": signer_account.address,
         "expires": "2023-05-02T10:44:42.754994Z",
     }
@@ -124,7 +123,7 @@ async def view(request, authenticated_sender):
             )
         )
     }
-    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/"}
+    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/", "domain": "localhost"}
     headers["X-SignedOperation"] = json.dumps(
         {
             "payload": bytes.hex(json.dumps(payload).encode("utf-8")),
@@ -158,7 +157,6 @@ async def view(request, authenticated_sender):
     pubkey = {
         "pubkey": json.loads(key.export_public()),
         "alg": "ECDSA",
-        "domain": "localhost",
         "address": signer_account.address,
         "expires": "2023-05-02T10:44:42.754994Z",
     }
@@ -167,26 +165,22 @@ async def view(request, authenticated_sender):
     signed_message: SignedMessage = signer_account.sign_message(signable_message)
     pubkey_signature = to_0x_hex(signed_message.signature)
 
-    # Modify the payload to render the signature invalid
-    pubkey["domain"] = "baddomain"
-    invalid_pubkey_payload = json.dumps(pubkey).encode("utf-8").hex()
-
     app.router.add_get("", view)
     client = await aiohttp_client(app)
     headers = {
         "X-SignedPubKey": (
             json.dumps(
                 {
-                    "payload": invalid_pubkey_payload,
+                    "payload": pubkey_payload,
                     "signature": pubkey_signature,
                 }
             )
         )
     }
-    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/"}
+    invalid_operation_payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/", "domain": "baddomain"}
     headers["X-SignedOperation"] = json.dumps(
         {
-            "payload": bytes.hex(json.dumps(payload).encode("utf-8")),
+            "payload": bytes.hex(json.dumps(invalid_operation_payload).encode("utf-8")),
             "signature": "96ffdbbd1704d5f6bfe4698235a0de0d2f58668deaa4371422bee26664f313f51fd483c78c34c6b317fc209779f9ddd9c45accf558e3bf881b49ad970ebf0ade",
         }
     )
@@ -195,7 +189,7 @@ async def view(request, authenticated_sender):
     assert resp.status == 401, await resp.text()
 
     r = await resp.json()
-    assert {"error": "Invalid signature"} == r
+    assert {"error": "Invalid domain"} == r
 
 
 @pytest.mark.asyncio
@@ -226,7 +220,7 @@ async def test_require_jwk_authentication_good_key(aiohttp_client, patch_datetim
     """An HTTP request to a view decorated by `@require_jwk_authentication`
     auth correctly a temporary key signed by a wallet and an operation signed by that key"""
     app = web.Application()
-    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/"}
+    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/", "domain": "localhost"}
     signer_account, headers = await generate_signer_and_signed_headers_for_operation(patch_datetime_now, payload)
 
     @require_jwk_authentication
diff --git a/tests/supervisor/test_qemu_instance.py b/tests/supervisor/test_qemu_instance.py
index 0d7f08fcd..9834e253c 100644
--- a/tests/supervisor/test_qemu_instance.py
+++ b/tests/supervisor/test_qemu_instance.py
@@ -53,6 +53,7 @@ async def test_create_qemu_instance():
     settings.USE_FAKE_INSTANCE_BASE = True
     settings.FAKE_INSTANCE_MESSAGE = settings.FAKE_INSTANCE_QEMU_MESSAGE
     settings.FAKE_INSTANCE_BASE = settings.FAKE_QEMU_INSTANCE_BASE
+    settings.ENABLE_CONFIDENTIAL_COMPUTING = False
     settings.ALLOW_VM_NETWORKING = False
     settings.USE_JAILER = False
 
@@ -107,6 +108,7 @@ async def test_create_qemu_instance_online():
     settings.USE_FAKE_INSTANCE_BASE = True
     settings.FAKE_INSTANCE_MESSAGE = settings.FAKE_INSTANCE_QEMU_MESSAGE
     settings.FAKE_INSTANCE_BASE = settings.FAKE_QEMU_INSTANCE_BASE
+    settings.ENABLE_CONFIDENTIAL_COMPUTING = False
     settings.ALLOW_VM_NETWORKING = True
     settings.USE_JAILER = False
 
diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
index 4e1f2746f..fff8b5492 100644
--- a/tests/supervisor/test_views.py
+++ b/tests/supervisor/test_views.py
@@ -168,8 +168,8 @@ async def test_about_certificates(aiohttp_client):
                 client = await aiohttp_client(app)
                 response: web.Response = await client.get("/about/certificates")
                 assert response.status == 200
-                is_file_mock.assert_has_calls([call(), call()])
+                is_file_mock.assert_has_calls([call()])
                 certificates_expected_dir = sev_client.certificates_archive
                 export_mock.assert_called_once_with(
-                    [PosixPath("/opt/sevctl"), "export", str(certificates_expected_dir)], check=True
+                    ["/opt/sevctl", "export", str(certificates_expected_dir)], check=True
                 )
diff --git a/tests/supervisor/views/test_operator.py b/tests/supervisor/views/test_operator.py
index 72a42ae09..729681194 100644
--- a/tests/supervisor/views/test_operator.py
+++ b/tests/supervisor/views/test_operator.py
@@ -1,14 +1,20 @@
 import asyncio
 import datetime
 import json
+import tempfile
 from asyncio import Queue
+from unittest import mock
+from unittest.mock import MagicMock
 
 import aiohttp
 import pytest
 from aiohttp.test_utils import TestClient
+from aleph_message.models import ItemHash
 
+from aleph.vm.conf import settings
 from aleph.vm.orchestrator.supervisor import setup_webapp
 from aleph.vm.pool import VmPool
+from aleph.vm.storage import get_message
 from aleph.vm.utils.logs import EntryDict
 from aleph.vm.utils.test_helpers import (
     generate_signer_and_signed_headers_for_operation,
@@ -16,6 +22,129 @@
 )
 
 
+@pytest.mark.asyncio
+async def test_operator_confidential_initialize_not_authorized(aiohttp_client):
+    """Test that the confidential initialize endpoint rejects if the sender is not the good one. Auth needed"""
+
+    settings.ENABLE_QEMU_SUPPORT = True
+    settings.ENABLE_CONFIDENTIAL_COMPUTING = True
+    settings.setup()
+
+    class FakeExecution:
+        message = None
+        is_running: bool = True
+        is_confidential: bool = False
+
+    class FakeVmPool:
+        executions: dict[ItemHash, FakeExecution] = {}
+
+        def __init__(self):
+            self.executions[settings.FAKE_INSTANCE_ID] = FakeExecution()
+
+    with mock.patch(
+        "aleph.vm.orchestrator.views.authentication.authenticate_jwk",
+        return_value="",
+    ):
+        with mock.patch(
+            "aleph.vm.orchestrator.views.operator.is_sender_authorized",
+            return_value=False,
+        ) as is_sender_authorized_mock:
+            app = setup_webapp()
+            app["vm_pool"] = FakeVmPool()
+            client = await aiohttp_client(app)
+            response = await client.post(
+                f"/control/machine/{settings.FAKE_INSTANCE_ID}/confidential/initialize",
+            )
+            assert response.status == 403
+            assert await response.text() == "Unauthorized sender"
+            is_sender_authorized_mock.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_operator_confidential_initialize_already_running(aiohttp_client):
+    """Test that the confidential initialize endpoint rejects if the VM is already running. Auth needed"""
+
+    settings.ENABLE_QEMU_SUPPORT = True
+    settings.ENABLE_CONFIDENTIAL_COMPUTING = True
+    settings.setup()
+
+    vm_hash = ItemHash(settings.FAKE_INSTANCE_ID)
+    instance_message = await get_message(ref=vm_hash)
+
+    class FakeExecution:
+        message = instance_message.content
+        is_running: bool = True
+        is_confidential: bool = False
+
+    class FakeVmPool:
+        executions: dict[ItemHash, FakeExecution] = {}
+
+        def __init__(self):
+            self.executions[vm_hash] = FakeExecution()
+
+    with mock.patch(
+        "aleph.vm.orchestrator.views.authentication.authenticate_jwk",
+        return_value=instance_message.sender,
+    ):
+        app = setup_webapp()
+        app["vm_pool"] = FakeVmPool()
+        client = await aiohttp_client(app)
+        response = await client.post(
+            f"/control/machine/{vm_hash}/confidential/initialize",
+            json={"persistent_vms": []},
+        )
+        assert response.status == 403
+        assert await response.text() == f"VM with ref {vm_hash} already running"
+
+
+@pytest.mark.asyncio
+async def test_operator_confidential_initialize(aiohttp_client):
+    """Test that the certificates system endpoint responds. No auth needed"""
+
+    settings.ENABLE_QEMU_SUPPORT = True
+    settings.ENABLE_CONFIDENTIAL_COMPUTING = True
+    settings.setup()
+
+    vm_hash = ItemHash(settings.FAKE_INSTANCE_ID)
+    instance_message = await get_message(ref=vm_hash)
+
+    class FakeExecution:
+        message = instance_message.content
+        is_running: bool = False
+        is_confidential: bool = True
+        controller_service: str = ""
+
+    class MockSystemDManager:
+        enable_and_start = MagicMock(return_value=True)
+
+    class FakeVmPool:
+        executions: dict[ItemHash, FakeExecution] = {}
+
+        def __init__(self):
+            self.executions[vm_hash] = FakeExecution()
+            self.systemd_manager = MockSystemDManager()
+
+    with tempfile.NamedTemporaryFile() as temp_file:
+        form_data = aiohttp.FormData()
+        form_data.add_field("session", open(temp_file.name, "rb"), filename="session.b64")
+        form_data.add_field("godh", open(temp_file.name, "rb"), filename="godh.b64")
+
+        with mock.patch(
+            "aleph.vm.orchestrator.views.authentication.authenticate_jwk",
+            return_value=instance_message.sender,
+        ):
+            app = setup_webapp()
+            app["vm_pool"] = FakeVmPool()
+            client = await aiohttp_client(app)
+            response = await client.post(
+                f"/control/machine/{vm_hash}/confidential/initialize",
+                data=form_data,
+            )
+            assert response.status == 200
+            assert await response.text() == f"Started VM with ref {vm_hash}"
+            app["vm_pool"].systemd_manager.enable_and_start.assert_called_once()
+
+
 @pytest.mark.asyncio
 async def test_reboot_ok(aiohttp_client, mocker):
     mock_address = "mock_address"
@@ -227,7 +356,7 @@ async def test_websocket_logs_invalid_auth(aiohttp_client, mocker):
 @pytest.mark.asyncio
 async def test_websocket_logs_good_auth(aiohttp_client, mocker, patch_datetime_now):
     "Test valid authentification for websocket logs endpoint"
-    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/"}
+    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/", "domain": "localhost"}
     signer_account, headers = await generate_signer_and_signed_headers_for_operation(patch_datetime_now, payload)
 
     mock_address = signer_account.address

From d643acb1dd3b2ee8b942ae49c04c75f484e6d75a Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 5 Jul 2024 16:47:39 +0200
Subject: [PATCH 809/990] Problem: sevctl command was not tested on the proplet
 (#651)

* Problem: sevctl command was not tested on the proplet

* Update .github/workflows/test-on-droplets-matrix.yml

Co-authored-by: Hugo Herter 

---------

Co-authored-by: Hugo Herter 
---
 .github/workflows/test-on-droplets-matrix.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index 33ff38005..adc04e0aa 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -169,6 +169,12 @@ jobs:
             "http://${DROPLET_IPV4}:4020/about/usage/system"
 
 
+      - name: Run the sevctl command to ensure it's properly packaged and working
+        run: |
+          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
+          ssh root@${DROPLET_IPV4} "/opt/sevctl --version"
+
+
       - name: Export aleph logs
         if: always()
         run: |

From 35dd30a50d31b3ba957cf809cb72a2f964dfd73b Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Fri, 5 Jul 2024 20:55:41 +0200
Subject: [PATCH 810/990] Add missing `cpuid` dependency (#656)

Problem: The supervisor cannot start a new instance because there are a missing `cpuid` dependency.

Solution: Add missing `cpuid` dependency to `Makefile` to add it on the Debian and Ubuntu packages.
---
 packaging/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 8b497da13..71a96d579 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.7' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.7' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo  target/bin/sevctl

From a27a488b6d8099c1a60f3f777f691786a0070bfc Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Wed, 17 Jul 2024 16:54:53 +0200
Subject: [PATCH 811/990] Solve Websocket error handling (#657)

* Problem: If the frontend or a user send and incorrect auth payload, the endpoint just stop sharing data without return anything or close the connection.

Solution: Handle error issues on the endpoint to always return status field and the reason why it's failing, and closing the connection.

* Fix: Solve test issue after the failed response.
---
 src/aleph/vm/orchestrator/views/operator.py | 22 +++++++++++++--------
 tests/supervisor/views/test_operator.py     |  3 +++
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 346b22d17..930053a33 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -125,17 +125,23 @@ async def authenticate_websocket_for_vm_or_403(execution: VmExecution, vm_hash:
         first_message = await ws.receive_json()
     except TypeError as error:
         logging.exception(error)
+        await ws.send_json({"status": "failed", "reason": str(error)})
         raise web.HTTPForbidden(body="Invalid auth package")
     credentials = first_message["auth"]
-    authenticated_sender = await authenticate_websocket_message(credentials)
 
-    if is_sender_authorized(authenticated_sender, execution.message):
-        logger.debug(f"Accepted request to access logs by {authenticated_sender} on {vm_hash}")
-        return True
-
-    logger.debug(f"Denied request to access logs by {authenticated_sender} on {vm_hash}")
-    await ws.send_json({"status": "failed", "reason": "unauthorized sender"})
-    raise web.HTTPForbidden(body="Unauthorized sender")
+    try:
+        authenticated_sender = await authenticate_websocket_message(credentials)
+
+        if is_sender_authorized(authenticated_sender, execution.message):
+            logger.debug(f"Accepted request to access logs by {authenticated_sender} on {vm_hash}")
+            return True
+
+            logger.debug(f"Denied request to access logs by {authenticated_sender} on {vm_hash}")
+            await ws.send_json({"status": "failed", "reason": "unauthorized sender"})
+            raise web.HTTPForbidden(body="Unauthorized sender")
+    except Exception as error:
+        await ws.send_json({"status": "failed", "reason": str(error)})
+        raise web.HTTPForbidden(body="Unauthorized sender")
 
 
 @cors_allow_all
diff --git a/tests/supervisor/views/test_operator.py b/tests/supervisor/views/test_operator.py
index 729681194..0b0c4cf13 100644
--- a/tests/supervisor/views/test_operator.py
+++ b/tests/supervisor/views/test_operator.py
@@ -349,6 +349,9 @@ async def test_websocket_logs_invalid_auth(aiohttp_client, mocker):
     await websocket.send_json({"auth": "invalid auth package"})
     response = await websocket.receive()
     # Subject to change in the future, for now the connexion si broken and closed
+    assert response.type == aiohttp.WSMsgType.TEXT
+    assert response.data == '{"status": "failed", "reason": "string indices must be integers"}'
+    response = await websocket.receive()
     assert response.type == aiohttp.WSMsgType.CLOSE
     assert websocket.closed
 

From f9cb9b7fa9d6d3294fef2aea031195eda5761520 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Tue, 23 Jul 2024 11:38:01 +0200
Subject: [PATCH 812/990] Small fixes noticed on new installations. (#659)

* Problem: If the server don't have access to Internet, it just crashes. Also, if an instance is removed, it fails removing the base block device.

Solution: Handle the crash and log as an error about the Internet connection. Check if the mounted block devices exists before removing it.

* Fix: Solve failing test.

* Fix: Changed the behaviour raising an issue if the response don't have headers.
---
 src/aleph/vm/hypervisors/firecracker/microvm.py | 10 +++++++---
 src/aleph/vm/orchestrator/status.py             |  3 +++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index bae76b95f..14230f6ed 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -472,12 +472,16 @@ async def teardown(self):
         if self.stderr_task:
             self.stderr_task.cancel()
 
+        # Clean mounted block devices
         if self.mounted_rootfs:
             logger.debug("Waiting for one second for the VM to shutdown")
             await asyncio.sleep(1)
-            root_fs = self.mounted_rootfs.name
-            system(f"dmsetup remove {root_fs}")
-            system(f"dmsetup remove {root_fs}_base")
+            if self.mounted_rootfs.is_block_device():
+                root_fs = self.mounted_rootfs.name
+                system(f"dmsetup remove {root_fs}")
+            base_device = Path(self.mounted_rootfs.name.replace("_rootfs", "_base"))
+            if base_device.is_block_device():
+                system(f"dmsetup remove {base_device}")
             if self.use_jailer and Path(self.jailer_path).is_dir():
                 shutil.rmtree(self.jailer_path)
 
diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py
index 90cf15d2f..f09127f03 100644
--- a/src/aleph/vm/orchestrator/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -120,6 +120,9 @@ async def check_internet(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
         response: dict = await get_json_from_vm(session, vm_id, "/internet")
 
+        if "headers" not in response:
+            raise ValueError("The server cannot connect to Internet")
+
         # The HTTP Header "Server" must always be present in the result.
         if "Server" not in response["headers"]:
             raise ValueError("Server header not found in the result.")

From e47abd629152bdce8b3565fdd36c9c15470e6cd6 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Tue, 23 Jul 2024 11:38:12 +0200
Subject: [PATCH 813/990] Solve Firecracker reboot issues (#658)

Problem: If the frontend or a user send the request to reboot a Firecracker instance, it stops but doesn't start again.

Solution: Change method to restart it just stopping the process, cleaning the firecracker run files and starting it again.
---
 src/aleph/vm/controllers/__main__.py            | 2 +-
 src/aleph/vm/hypervisors/firecracker/microvm.py | 1 +
 src/aleph/vm/hypervisors/qemu/qemuvm.py         | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index 2ba923abd..90475086f 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -91,7 +91,7 @@ async def handle_persistent_vm(config: Configuration, execution: Union[MicroVM,
 
     def callback():
         """Callback for the signal handler to stop the VM and cleanup properly on SIGTERM."""
-        loop.create_task(execution.teardown())
+        loop.create_task(execution.stop())
 
     loop.add_signal_handler(signal.SIGTERM, callback)
 
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 14230f6ed..e5a7c94dc 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -181,6 +181,7 @@ def prepare_start(self):
         system(f"rm -fr {self.jailer_path}/dev/net/")
         system(f"rm -fr {self.jailer_path}/dev/kvm")
         system(f"rm -fr {self.jailer_path}/dev/urandom")
+        system(f"rm -fr {self.jailer_path}/dev/userfaultfd")
         system(f"rm -fr {self.jailer_path}/run/")
 
         if os.path.exists(path=self.vsock_path):
diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index 384c31b05..53518eb0c 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -146,6 +146,6 @@ def send_shutdown_message(self):
             print("shutdown message sent")
             client.close()
 
-    async def teardown(self):
+    async def stop(self):
         """Stop the VM."""
         self.send_shutdown_message()

From 15e70e730b8e9609209462fe8f86d896594befbc Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 26 Jul 2024 14:15:24 +0200
Subject: [PATCH 814/990] Allocate endpoint allow starting  confidential with
 hold payment method (#660)

* WIP : allocate command allow hold on confidential

* Problem: User could not start confidential instance directly

* isort
---
 ...fidential_instance_message_from_aleph.json |  2 +-
 src/aleph/vm/orchestrator/tasks.py            | 34 ++++----
 src/aleph/vm/orchestrator/views/__init__.py   | 81 ++++++++++++-------
 3 files changed, 71 insertions(+), 46 deletions(-)

diff --git a/examples/confidential_instance_message_from_aleph.json b/examples/confidential_instance_message_from_aleph.json
index 6b130c65c..30cf7ee7a 100644
--- a/examples/confidential_instance_message_from_aleph.json
+++ b/examples/confidential_instance_message_from_aleph.json
@@ -6,7 +6,7 @@
   "channel": "Fun-dApps",
   "confirmed": true,
   "content": {
-    "address": "0x2b0eE984F821C710104e575953634b3f3d364Ec4",
+    "address": "0xE0178501683a4C321cAE8263839F349e0f07dECd",
     "allow_amend": false,
     "variables": {
       "VM_CUSTOM_NUMBER": "32"
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 4e85f0d91..fc627e29a 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -23,7 +23,12 @@
 from aleph.vm.utils import create_task_log_exceptions
 
 from .messages import load_updated_message
-from .payment import compute_required_flow, get_stream
+from .payment import (
+    compute_required_balance,
+    compute_required_flow,
+    fetch_balance_of_address,
+    get_stream,
+)
 from .pubsub import PubSub
 from .reactor import Reactor
 
@@ -144,19 +149,20 @@ async def monitor_payments(app: web.Application):
         await asyncio.sleep(settings.PAYMENT_MONITOR_INTERVAL)
 
         # Check if the balance held in the wallet is sufficient holder tier resources (Not do it yet)
-        # for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.hold).items():
-        #     for chain, executions in chains.items():
-        #         balance = await fetch_balance_of_address(sender)
-        #
-        #         # Stop executions until the required balance is reached
-        #         required_balance = await compute_required_balance(executions)
-        #         logger.debug(f"Required balance for Sender {sender} executions: {required_balance}")
-        #         # Stop executions until the required balance is reached
-        #         while balance < (required_balance + settings.PAYMENT_BUFFER):
-        #             last_execution = executions.pop(-1)
-        #             logger.debug(f"Stopping {last_execution} due to insufficient balance")
-        #             await pool.stop_vm(last_execution.vm_hash)
-        #             required_balance = await compute_required_balance(executions)
+        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.hold).items():
+            for chain, executions in chains.items():
+                executions = [execution for execution in executions if execution.is_confidential]
+                balance = await fetch_balance_of_address(sender)
+
+                # Stop executions until the required balance is reached
+                required_balance = await compute_required_balance(executions)
+                logger.debug(f"Required balance for Sender {sender} executions: {required_balance}")
+                # Stop executions until the required balance is reached
+                while balance < (required_balance + settings.PAYMENT_BUFFER):
+                    last_execution = executions.pop(-1)
+                    logger.debug(f"Stopping {last_execution} due to insufficient balance")
+                    await pool.stop_vm(last_execution.vm_hash)
+                    required_balance = await compute_required_balance(executions)
 
         # Check if the balance held in the wallet is sufficient stream tier resources
         for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 31198676c..614314461 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -13,7 +13,7 @@
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPBadRequest, HTTPNotFound
 from aleph_message.exceptions import UnknownHashError
-from aleph_message.models import ItemHash, MessageType
+from aleph_message.models import ItemHash, MessageType, PaymentType
 from pydantic import ValidationError
 
 from aleph.vm.conf import settings
@@ -23,7 +23,7 @@
 )
 from aleph.vm.controllers.firecracker.program import FileTooLargeError
 from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInitError
-from aleph.vm.orchestrator import status
+from aleph.vm.orchestrator import payment, status
 from aleph.vm.orchestrator.messages import try_get_message
 from aleph.vm.orchestrator.metrics import get_execution_records
 from aleph.vm.orchestrator.payment import (
@@ -464,35 +464,53 @@ async def notify_allocation(request: web.Request):
     if message.type != MessageType.instance:
         return web.HTTPBadRequest(reason="Message is not an instance")
 
-    if not message.content.payment:
-        return web.HTTPBadRequest(reason="Message does not have payment information")
-
-    if message.content.payment.receiver != settings.PAYMENT_RECEIVER_ADDRESS:
-        return web.HTTPBadRequest(reason="Message is not for this instance")
-
-    # Check that there is a payment stream for this instance
-    try:
-        active_flow: Decimal = await get_stream(
-            sender=message.sender, receiver=message.content.payment.receiver, chain=message.content.payment.chain
-        )
-    except InvalidAddressError as error:
-        logger.warning(f"Invalid address {error}", exc_info=True)
-        return web.HTTPBadRequest(reason=f"Invalid address {error}")
-
-    if not active_flow:
-        raise web.HTTPPaymentRequired(reason="Empty payment stream for this instance")
-
-    required_flow: Decimal = await fetch_execution_flow_price(item_hash)
-
-    if active_flow < required_flow:
-        active_flow_per_month = active_flow * 60 * 60 * 24 * (Decimal("30.41666666666923904761904784"))
-        required_flow_per_month = required_flow * 60 * 60 * 24 * Decimal("30.41666666666923904761904784")
-        return web.HTTPPaymentRequired(
-            reason="Insufficient payment stream",
-            text="Insufficient payment stream for this instance\n\n"
-            f"Required: {required_flow_per_month} / month (flow = {required_flow})\n"
-            f"Present: {active_flow_per_month} / month (flow = {active_flow})",
-        )
+    payment_type = message.content.payment and message.content.payment.type or PaymentType.hold
+
+    is_confidential = message.content.environment.trusted_execution is not None
+
+    if payment_type == PaymentType.hold and is_confidential:
+        # At the moment we will allow hold for PAYG
+        logger.debug("Confidential instance not using PAYG")
+        user_balance = await payment.fetch_balance_of_address(message.sender)
+        hold_price = await payment.fetch_execution_hold_price(item_hash)
+        logger.debug(f"Address {message.sender} Balance: {user_balance}, Price: {hold_price}")
+        if hold_price > user_balance:
+            return web.HTTPPaymentRequired(
+                reason="Insufficient balance",
+                text="Insufficient balance for this instance\n\n"
+                f"Required: {hold_price} token \n"
+                f"Current user balance: {user_balance}",
+            )
+    elif payment_type == PaymentType.superfluid:
+        # Payment via PAYG
+        if message.content.payment.receiver != settings.PAYMENT_RECEIVER_ADDRESS:
+            return web.HTTPBadRequest(reason="Message is not for this instance")
+
+        # Check that there is a payment stream for this instance
+        try:
+            active_flow: Decimal = await get_stream(
+                sender=message.sender, receiver=message.content.payment.receiver, chain=message.content.payment.chain
+            )
+        except InvalidAddressError as error:
+            logger.warning(f"Invalid address {error}", exc_info=True)
+            return web.HTTPBadRequest(reason=f"Invalid address {error}")
+
+        if not active_flow:
+            raise web.HTTPPaymentRequired(reason="Empty payment stream for this instance")
+
+        required_flow: Decimal = await fetch_execution_flow_price(item_hash)
+
+        if active_flow < required_flow:
+            active_flow_per_month = active_flow * 60 * 60 * 24 * (Decimal("30.41666666666923904761904784"))
+            required_flow_per_month = required_flow * 60 * 60 * 24 * Decimal("30.41666666666923904761904784")
+            return web.HTTPPaymentRequired(
+                reason="Insufficient payment stream",
+                text="Insufficient payment stream for this instance\n\n"
+                f"Required: {required_flow_per_month} / month (flow = {required_flow})\n"
+                f"Present: {active_flow_per_month} / month (flow = {active_flow})",
+            )
+    else:
+        return web.HTTPBadRequest(reason="Invalid payment method")
 
     # Exceptions that can be raised when starting a VM:
     vm_creation_exceptions = (
@@ -506,6 +524,7 @@ async def notify_allocation(request: web.Request):
 
     scheduling_errors: dict[ItemHash, Exception] = {}
     try:
+        logger.info(f"Starting persistent vm {item_hash} from notify_allocation")
         await start_persistent_vm(item_hash, pubsub, pool)
         successful = True
     except vm_creation_exceptions as error:

From bd3a7354ae59b9d9a339d3ffb57050c37b5631ca Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 29 Jul 2024 12:14:39 +0200
Subject: [PATCH 815/990] =?UTF-8?q?Fix:=20error=20when=20user=20balance=20?=
 =?UTF-8?q?is=20zero=20and=20no=20remaining=20executions=20to=20r=E2=80=A6?=
 =?UTF-8?q?=20(#661)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fix: error when user balance is zero and no remaining executions to remove
---
 src/aleph/vm/orchestrator/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index fc627e29a..006de6e6b 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -158,7 +158,7 @@ async def monitor_payments(app: web.Application):
                 required_balance = await compute_required_balance(executions)
                 logger.debug(f"Required balance for Sender {sender} executions: {required_balance}")
                 # Stop executions until the required balance is reached
-                while balance < (required_balance + settings.PAYMENT_BUFFER):
+                while executions and balance < (required_balance + settings.PAYMENT_BUFFER):
                     last_execution = executions.pop(-1)
                     logger.debug(f"Stopping {last_execution} due to insufficient balance")
                     await pool.stop_vm(last_execution.vm_hash)

From b21c27eed6e6b809043d3df4cd238617cfd64d85 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 29 Jul 2024 12:46:29 +0200
Subject: [PATCH 816/990] Start documentation on confidential (#655)

* Documentat on confidential

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Update doc/confidential.md

Co-authored-by: Hugo Herter 

* Review comments

* Update confidential image creation doc

---------

Co-authored-by: Hugo Herter 
---
 doc/confidential.md                           | 266 ++++++++++++++++++
 doc/images/boot_process.drawio.png            | Bin 0 -> 21818 bytes
 examples/example_confidential_image/README.md |   7 +-
 .../setup_debian_rootfs.sh                    |   8 +-
 4 files changed, 275 insertions(+), 6 deletions(-)
 create mode 100644 doc/confidential.md
 create mode 100644 doc/images/boot_process.drawio.png

diff --git a/doc/confidential.md b/doc/confidential.md
new file mode 100644
index 000000000..4e208d7c9
--- /dev/null
+++ b/doc/confidential.md
@@ -0,0 +1,266 @@
+# Confidential computing
+
+Aleph-vm offers to launch confidential VM with AMD SEV. This is also known as TEE, Trusted Execution Environment.
+
+This is only supported for instances using Qemu as their hypervisor.
+
+## Life cycle
+First, a user creates a VM message and sends it with notify_allocate. This notifies the orchestrator about the creation of the new VM.  
+The user fetches the platform certificate, validates its chain again AMD root certificate.
+The user must then upload so-called Guest Owner certificates (created with sevctl) to create an encrypted channel between the user and the Security Processor.
+
+Once  uploaded, the VM is started in Qemu in stopped mode: Qemu will allocate the RAM for the VM, load the firmware inside it and then let the AMD Security Processor encrypt the memory. Once this is done, the SEV endpoints allow to retrieve a measure of the memory of the VM and to decide whether to inject a user secret in the VM. Upon secret injection, the VM is launched, i.e. the VM CPU is started and goes through the boot sequence of the VM.
+
+The end result is a virtual machine that is accessible through SSH and is completely encrypted in RAM, making it inaccessible from the point of view of the hypervisor.
+
+```mermaid
+flowchart TD
+    A[Start] -->|Allocate VM on CRN| B(CRN: Check payment, download image, volume)
+    B --> |Download certificate from CRN| C(User: Validate Certificate again CHAIN)
+    C --> |Create session certificates| D[Certificates file created]
+    D --> |Send certificate to CRN to init sessions | E[CRN: Launch VM with firmware with encrypted communication channel]
+    E --> |Fetch measurement from VM| F[User: Calculate it's own measurement and verify them again the CRN's]
+    F --> | if ok: Send secret in encrypted channel | G[CRN: Start and unlock VM]
+```
+    
+
+# CRN Side
+
+## Hardware requirement
+4th Generation AMD EPYC™ Processors with SEV support.
+
+This includes the [9004 Series Processors and 8004 Series Processors](https://www.amd.com/en/products/processors/server/epyc/4th-generation-9004-and-8004-series.html#tabs-4380fde236-item-2130f0d757-tab).
+
+Note that the [4004 Series Processors do not provide SEV](https://www.amd.com/en/products/processors/server/epyc/infinity-guard.html) and are therefore not supported.
+
+> ℹ️ The 4th Generation requirement stems from security vulnerabilities discovered in SEV on Zen3 and earlier architectures.
+
+## Requirements for the CRN
+* Support must be [enabled in the computer BIOS](https://www.amd.com/content/dam/amd/en/documents/epyc-technical-docs/tuning-guides/58207-using-sev-with-amd-epyc-processors.pdf) (see Section 2.1). 
+* The kernel and platform must support SEV. (e.g Ubuntu 24.04 support it by default) 
+* [sevctl](https://github.com/virtee/sevctl) must be installed. A copy is included in the aleph-vm Debian package and installed as `/opt/sevctl`.
+* QEMU must be installed.
+
+Check with the `sevctl ok` command that the system is supporting AMD SEV properly,  at least:
+
+```[ PASS ]   - Secure Encrypted Virtualization (SEV)```
+
+
+
+See AMD DOC for more info on enabling SEV for your system
+https://www.amd.com/fr/developer/sev.html
+
+
+## Enabling the confidential computing feature in aleph-vm
+
+Enable SEV in the configuration of `aleph-vm`, by default in `/etc/aleph-vm/supervisor.env`:
+```
+ALEPH_VM_ENABLE_QEMU_SUPPORT=1
+ALEPH_VM_ENABLE_CONFIDENTIAL_COMPUTING=1
+
+```
+
+After launching the server you can check the endpoint
+http://localhost:4020/status/config and verify that ENABLE_CONFIDENTIAL_COMPUTING is true
+
+
+# User side
+The user wanting to launch the VM, referred as the Guest Owner.
+
+The aleph-sdk-python and the aleph-client provide way to launch , validate and start the VM.
+
+## Create an encrypted VM image
+
+The user must create a virtual machine disk image that has been encrypted using a password of their choice.
+Follow the instruction here: https://github.com/aleph-im/aleph-vm/blob/dev-confidential/examples/example_confidential_image/README.md
+
+## OVMF Launcher Firmware
+The OMVF file, a UEFI firmware for virtual machines, handle launching the confidential VM.
+It receives the secret (decryption key) in a secure manner and pass it to the VM bootloader (see Boot process section).
+
+Aleph.im provide a default one, destined to work with confidential image created following the procedure described above.
+
+
+In the usual case a user would just create an encrypted VM image but they might also provide a customised firmware in the `firmare` field  of `trusted_execution`.    
+
+See [the instructions on how the Firmware is built](runtimes/ovmf/README.md)
+
+The hash from the Firmware is needed to validate if it's the one launched the CRN.
+
+
+# Implementation details
+## Aleph-message
+on Instance type message, we check if the `content.environment.trusted_execution` is set  
+
+```
+    "trusted_execution": {
+        "policy": 1,
+        "firmware": "e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317"
+    }
+```
+
+* Firmware is an [IPFS CID](https://docs.ipfs.tech/concepts/content-addressing/) reference to the OVMF firmware file (see OVMF firmware section)
+* policy is an AMD SEV Policy (for now we only expose if AMD SEV and SEV-ES are supported)
+
+
+## Boot process
+The following diagram describes the different pieces of the VM boot process.
+
+![Boot process](./images/boot_process.drawio.png)
+
+* OVMF: UEFI firmware (see section above), finds the bootloader and launches it
+* GRUB, the boot loader, decrypts the VM image and jumps to it.
+* GRUB configuration files: the unencrypted script looks for the user disk decryption password injected during
+  the SEV boot process, then jumps to a complete Grub configuration file provided by the user inside the VM
+  image.
+* Kernel + initrd + root filesystem: The OS of the VM.
+
+OVMF and Grub must be unencrypted. This means that the VM supervisor can alter these pieces at will.
+It is therefore crucial that these pieces are part of the launch measurement retrieved during the SEV
+sequence.
+
+The  process documented in `runtimes/ovmf/README.md` can be used to generate a firmware image that combines OVMF and Grub
+into one binary.
+
+
+## Detailed sequence with endpoints 
+```mermaid
+sequenceDiagram
+    participant Qemu
+    participant CRN
+    actor User
+    CRN->>User: Fetch platform certificate (GET /about/certificates/)
+    Note right of User: Generate via sevctl using the platfom certificate:
               TIK, TEK, GODH, Session
+    User->>CRN:Upload certificates POST /control/machine/{ref}/confidential/inialize
+    Note over CRN,User:session.b64, godh.b64
+    CRN->>Qemu: Run qemu process (pass session, godh, image, ovmf)
+    Note left of Qemu: Emulator is in stopped state
+    User->>CRN: Fetch measurement   (GET /control/machine/{ref}/confidential/measurement)
+    Qemu->>CRN: Retrieve launch measurement (via qmp)
+    CRN->>User:   Measurements (SEV version, policy, firmware hash, signature) 
+    Note right of User: Verify measuremetn signature
+    Note right of User: Encrypt secret using TEK key
+    User->>CRN: Pass encoded secrets (POST /control/machine/{ref}/confidential/inject_secret)
+    CRN->>Qemu: Inject secret (via qmp)
+    CRN->>Qemu: Start VM (via qmp)
+    Note left of Qemu: Emulator is in started state, VM Boot
+    User->>Qemu: SSH or other interaction
+```
+
+# Development and debugging
+
+See QEMU.md in general for QEMU related developement
+
+ ## Note on systemd in dev
+ If you use a local copy  of aleph-vm, for example a version you are developping on, by default  systemd will still use the system version  of  the  aleph controller. It is necessary to modify 
+ `/etc/systemd/system/aleph-vm-controller@.service` to point  to your version.
+ 
+ For example here is what I use
+ ```
+ [Unit]
+Description=Aleph VM %i Controller Olivier
+After=network.target
+
+[Service]
+Type=simple
+RestartSec=5s
+PrivateTmp=yes
+NoNewPrivileges=true
+WorkingDirectory=/home/olivier/pycharm-aleph-vm/src
+Environment=PYTHONPATH=/home/olivier/pycharm-aleph-vm/src:$PYTHONPATH
+ExecStart=/home/olivier/.virtualenvs/aleph-vm/bin/python3 -m aleph.vm.controllers --config=/var/lib/aleph/vm/%i-controller.json
+Restart=no
+
+[Install]
+WantedBy=multi-user.target
+```
+
+After modification use the following command to have the modification taken into account
+```shell
+sudo systemctl daemon-reload
+```
+
+# Testing
+
+After initializing the VM you can check it's status with:
+`sudo systemctl status aleph-vm-controller@decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca.service`
+
+and see the logs with
+` sudo journalctl -u aleph-vm-controller@decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca.service`
+
+**Important**
+ 
+If you modify your base image between tests, you will need to delete the image file on disk (which is a delta of the base image)
+For example using :
+`sudo rm /var/lib/aleph/vm/volumes/persistent/decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca/rootfs.qcow2`
+
+Ensure the VM controller is stopped before!
+`sudo systemctl stop aleph-vm-controller@decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca.service`
+
+ Between your test you can also stop the execution using
+ ```http
+ ### Stop all VMs
+POST http://localhost:4020/control/allocations
+Content-Type: application/json
+X-Auth-Signature: test
+Accept: application/json
+
+
+{
+  "persistent_vms": [],
+  "instances": [
+  ]
+}
+
+```
+
+## Sevctl
+Most operations done by `sevctl` are implemented in [aleph-sdk-python](https://github.com/aleph-im/aleph-sdk-python), either by calling it, calling the relevant endpoint
+or by reimplementing the functionality in python. Here is a primer in case you need to call it manually.
+
+### Install `sevctl`
+If you are not taking the version from the debian package, you can install sevctl manually with cargo
+
+Requirements:
+ * `cargo`
+
+On Ubuntu/ Debian install it via `apt install cargo` (as root)
+
+To build and install sevctl
+```cargo install  sevctl```
+
+Ensure $HOME/.cargo/bin is in your PATH to launch it manually.
+
+To configure which bin aleph-vm use, set the environment variable
+```
+ALEPH_VM_SEV_CTL_PATH=/home/olivier/.cargo/bin/sevctl
+```
+
+Alternatively, `sevctl` can be build from `git` : ```cargo install  --git  https://github.com/virtee/sevctl```
+
+
+## Example Commands
+## Generate session key
+You can generate the sessions keys using sevctl
+1. Export the platform key
+ `sudo sevctl export  platform.pem`
+2. Create the sessions files
+   `sevctl session platform.pem 0x1 dwdw`
+   
+This will create the files `vm_godh.b64`,  `vm_session.b64`,  `vm_tek.bin`,  `vm_tik.bin` in your current directory 
+
+### Calculate measurement
+
+```shell
+RUST_LOG=trace sevctl measurement build
+  --api-major 01 --api-minor 55 --build-id 24 --policy 1
+  --tik ~/pycharm-aleph-sdk-python/decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca_tik.bin
+  --firmware /usr/share/ovmf/OVMF.fd
+  --nonce URQNqJAqh/2ep4drjx/XvA
+  ```
+
+### Debug
+To enable debugging log,  set the environment variable 
+```env
+RUST_LOG=trace
+```
\ No newline at end of file
diff --git a/doc/images/boot_process.drawio.png b/doc/images/boot_process.drawio.png
new file mode 100644
index 0000000000000000000000000000000000000000..a0e20f74ec49f7328a639a6709b1dc2b0f5bec80
GIT binary patch
literal 21818
zcmbTe1yq&I_CF4YazF(RAxO%h1?iNILw7eGy1QFF(j}5oN=pmUDIp>V1})vvAzi;Y
z`o8zx_ulVX|F!;WJ-D9pJoC)#nZ0MvXZB~05o#*3w{ggD(9qCs%gae?prK)OprN4?
zLvDa4Yww?aM?<^*%>%CI;pk&!>tKmS&n@-mj-HFd#@XG2o?Dupiwoi6!e(KMuy#W@
zy0bZ1dVojZy`!^*t(C2%#h)@<99&!vIQSoMa%ypM(Q`}ia)5uh`Pn%61PuO^M_5}r
z{Z%0^8waR>Nsp70`OiB`4}|rfddwA_WVN*)=<1r+JM*
zb8H-JoPS<;__|pBd1~$KZ0%r)dddw4UHMC9(3I_8QmRYnOK@BBx^lrEJW$|ubk}tg
zl=)MM6TOzPTYKM&ffCc()I=xpp=5NvZt~V!c14wMVFonG$f;;?ylAvD4Of?@$>6zd#Qn{ycBtT)P1=nEcCf}
z&8+-nZM7ZMIISJv`~vc_z6vsWDxi$H0-wCHs*1axgMz)Tye+2+kSpu(-}Wa(zcE8yuT
z<*eqYVCCp7i|})Gbe7kYk#n?Bb&=7NuvFpnbkC^GVoBe(X&ufa&Y1Ify*G|Z58dLTnw!BJq_T}S{!zAegYf_X-_vn4{Lcj
z&@Dj)RXH^uNqxA6wWNZCfSU)0ueGz8q@W}aQ6BtBa?4A&@^kQ5*h{NOs#!Rx>iW3J
z=qqZ=OLOSz$ys?SOMpTaZr<*k){=^X`ZBhDF7oDFc5n%2A9-b2TXShEGahaUAA36u
zxPX-=r>nObx2=z!wu>CXR!85>4IwAT!)L*xqv@ri>7v0Y>FDU-%cm`)X2BuvW-DQV
zu;les({|MqFymF@GqAIfmgVsFv+=dy(9%%Tkyp3Z(1Tl=Ia*j~Iw@&baq#GxDcKvC
zIjL({I`P;___%0D=<8be*eU=KCA1Z--Bs14-N2-gL|AGWxJU!@lmXq8bhFk~w3Y)m
z;L=hu@OAOC(o=JH(A4#^*YlLHapLyW*3)q0cUKYgR8lop)ONB|Rx|gubmTUd;PK+O
z;O3T72VZhas!DOWs(Z+}YZ*wWXsg+J`f;17E30~X3!1BFxqE2~NcnI$Xe)xE25v4A
zK!r~7%3xMYa&kI>#$}u>xcRIkWo!-9fwz!xv3JpN@v_tQ;8M3z^On-$L~t2+p;|HL
zQ|DJ#w%~Sgv2*8f)Ase^cJ;F4QB$_DR#TPNb~od2)K@igc2KZ!)RE@21Jgy4LzPcc
z)7;fhU!7aPN|HxV6(QvWTHq7lMwy|alpdd)f`YZ5ouz^vho+qk!qd{n&PAQW$v}pm
z*Gmc%LDeDcqo?NP?x7}c<*lO4Z|T9QD8tR^ZEeBj%I#`_FxRj$aCbM;<2Sd^P;&w0
zmF+e84eZ^Wxs@F?xzrsrIXU^f+#Kv>^=!SA930F+t&++L_MW;D5;puEin@{RP%6JeI29y@mm&97*y%Sd7`QTo=e72jfY1{8Qf?&sG93)af9A?NP4Jos(AYNIa^r@%DF3fqnsV;LiO-(
zANSkgp}zm^$btV8J>OJ9L%WA2FD;?vW3riv<)fut7b?d5g_SY8A
zBayIM%@i+se6Qtby2(T8rUDl9r?FN%48o)UUEFGENfUdFIbflZxu1DT5kX5k^84b+cJ61oPilyCtT{D@YUTyb&9AJlfw^2uvJm1^f4JH0=5WH27E&VkJlZ
zKfY$_=tn2G_wP%W093`bVZ`%yJ*W!dse
zP>+
zDt!Equeq`ddrN%+e!q$)Dl
zO&X{94HN%O-o@S*Sfk~}Qp3gCoWYj?4l}LB5vRi?x=HU;G6d9X&Q4aczvjsk23%bn
zeasZ9J55VGW%Nd74{s&un!Hb9(Zv@F3hOaUU`C~SoUBIm_)T7$lD!G^t}6qS{V#cM
zYTN=MMMaUY!-6l*?XYwe%-Vu35*HP0ldgnpztJH>LyNVoKEKqy(8qCU@jIw=80XS@
zqjCKy7T$-o;S$%`c68}UXshc&=zVX!YO5HsT>qmj0nfEWJC+st>jDA-q=t1)aG-Tg
ztNzqN>0|xP4!BQe=W{C(LZqHQY8Ak4ap^O9Sz0aHyybK3oivZmo
zaL(RB8CCmN%$ySFbhR~}+tR1&4YaJ@mVJrLxg%-9zHS);t{A*_L*hNLl4OSlt-R|d0X3zX7OVo8mhd@Awy`K?H&%zzG>49k4zxn}d@SyW}>
z^oJPo63VS7*KhjS6)eS8Qb^%kjJ-AgT8WU7?MX*z0?fk0LL0XSzcqo5{y?ZhHkRUj
zx^wFrs}vXQWU44i|pdZ+ZnkgwboYkqsKpDPoyvk)idw{F3WoT?!2?*+=mGRN{zQSYsc(~E_kIPC~&
zU@3`^K4V?qw#m0iaSAW!@xyQ_HTK3E+^sjJTgcna_o4`l##N{)$vGZ{Bsl6vsyF+%
zE`Cne-CX?SYI#vL`8;T^avcCnWD2M
zqVw5{klPS7q$572%43v-8A{@jm8C05Ze+U&Se=WpWu?$qJ6Qqs9O>0>`BC>Bw)NilRf6Ii77;L4el$&#(K=nceyOoEV11Rb(0z7-X~d-xE1Q-
zc!T+yB(2Dt#pn2PBJlEOVfGO2&Rl1hwmg!^lDSMZat9^uPTSRGNtZAytCEOBj^Dvr
zaj1Sd%e*@>U#pLD)1<{uP;etd2>bXtm~p1a=TX0%;T5A=y$gaqX%8DZ2Wxg+kfHDV
zz3kj$04|gB^#}hy&R&W3p1t1^qKSZBp@?#7$IqmJRtd`Y;1)$V<~eG;Vek9|e-74=
zkSqLnOOi5!8jUM~XRLLQHj9QQMCK>IuuV`i1D_CsXf9Q4K?L>V?<uP7jV2#`AP^qS!Ojhkum}G!XL6znc;8tIVmATj9?Za7#w|nR2tP82IwnD
zI_AqxASkJ12+RSZ%2)$FPoqTL0Dn9X>6FFe?eILvzI6
z;-Q@JPhbHmarahKe^-tF$QFqmFNg`+N1y~FHShvUvpB}3YDn19x!4CI8th|kbgJb2
zl~p%KRMn3-Kxf{EjEjITUW2Ma=Pg%I&u3@A=W{c=OKWoNGuV|p9W%832PoV5`|^Q<
z8k;-1Km1!j&iR@pWbY~G3ZHiK8?6(Jr95Zu
zX#G-s)e>MlAtsaY+6#cJ&C-mq1eilx5!_c#r9>dHd>H4TSXaDLmBm-bYpVlpabJV-
z>b&_Vt;D!~&Hfy$RYhm=zQ#qrAXH%wsZ*pz`hT{pvCF14rt;T{RapcaH5~2UK57Ve
z7#Ft}{o%HkG31WD<9T8v?J#=}G>3Ja<%VWHqzCQg`#L>_j;AuJJ)L23_(MOV54%t+YgWEo_mh$2>Di(2?j*
z&u=q#j6C}s|1ErF6B3u*fElKm(S0PVI#(E#va3FT$01rz9-H+rEOhhqKA8vxnZz9q
z(^fy@3*k*e=I5}J^?F96kN9YEvGT{`@%sJInYBToLrTdKDyf}?!8VxY>AO_!0_x%7OLrT28wMg$
zWdR+f`2w!XveA@72h7l9p_&paT;0t)I@vl}qf$Tj#m~>-y)NHI6MVvDi010tb-QvR
z-ZRBBHk}6Um%sN}TEhI8&BF3wK<`QA(v?W3YBHiZE1&MLJRJ_LW#hk1od$CdFjK?=
zzT|0kK6Kfo0FvH0UaZd4q4S)7^$pZOfi^ZnI7*&IWcU-~pmA(2;~<>5qj2!5^bD5K&z2Q&9
zo$0+MJ&!vXIovJc842#<1m<|<`05QbF7o(UL{@%9mThLk93BM6ma_wo(ki?;LJl8u
zfdPZa-l-;ZJ_}caM)v-
zcrPq!XU2QmTi3aw}}XI-c8}VY11a`o&6Fd!jFkE31Qi-nVf6
z>Xh#H6)QAL>Agc2E5VK7Bnuz9TcvusVt33O3=d~wcglVQ!+v7r?{qF-CqkSp6*zH!
zdpX#6`(vyUJy;ot`LVcDfx$g$zvHb6trb91zqS1WGHCi1y(LshO{jq$XZgeMdfURj
z$~7&K%EXD4h@H%~Ei;!vzIfgaC|vK;TJzHSThE_#=Rx~?i!Ue*_(~rKWu`9-_jhyJ
zYnf`X8IwloI!iwKD1Bb1&2ADF~v05`YuxJbVd%iK#d}L5ynD|8&Sv?9y|689f&-;f)I2U
z&_sToa~NhKLX>BFWl}rARKT?^k$DXUsNQ^#RcvTE4UMjeH@br-*p`3c#u5PXcO>Tors
zLeanLarIxA)@u_up0|I6Ix-o0n
z9tYk{gKgXtIh#?0)&@rlNNr*sUzTJh5gLA@-5}wOtR&$?VECi=$tW{-@Zp4_Yrck%
zqa_&|$N*?^*H80Cfo$rN@e&E94wG54|0%~C$)-<+_?M5SLJ<41XOqqK-E^G~mrtd;
zd@-6U8+Q*+uPc&R@j}q2|DM))UXCh}3@#b{X;H|nG;W)%l|ivf>63Hf%7wAIm)jA=
zG=yV}0m)P6tWc*B-+Kk@=QwQ?dSP133I5AI2X*9fX}Q8=L#5$qv`^JqoebaEo-^N1
zzE8CI@Q_M4lCLGy>?%~SvGf2;Jy*mw8J)_nJzbQ32Hd9Bb}AilUiRFc4ooCeG7aYCyc10wPG?r;J-8kHl?XuWj^zVoVb&Gi
zD$=RhzBA`8^uer$QN{2Q8W1mdzO3Htz#08g^!`LcjpHQlu$?yiQ=P_Z-6j*gR$u)@
zM_qQ~7(vbD`HqI0a)|;^9AftH+NmEf+r2!KGXKrJnz6TzjG@uw8^eLEe4?J`L=Wp$
zpOAUl&raRiHrpC%NNt`K^XA;kktLNs8V{5`@=kp%)X_3=eSXu!IoJ1$J?rI^7~ZAC
zB*}uGa^BCv&sxF0IE!Bc*x`EI_uCwfg`p+@j@;TBzk#l7=Q|>1JK*NAC5FSxlA-|%
zkb_vSB|6{xX)m8+`Ku?}+huR#I!!+HY6s^-=;h@{`NWD*+^7EUWQMEYu7=S~Zc?eo
zuLQ3(=?sd8dViNDyL#+X)xqZ(y;Km@SXYbsAmELQnRxrACpPE?G@&G-Xt~O}#HQz6p6qLadZu#{66;
zSUAY5Ft0rRl?oe9hpbJK^x#p;emLUJ2^>89czQ(u6MHjoXVbhVHb#{^C{mx$+ErFB
z+sMs`eUNZYOgh%M)sJ$fG-FNQ@TWo9&daefr5%j``i>-WFY&0;;o(@9Dm;bP);~Uw
za$2I#W53_{Qi^L3H8sDDn=KyY^c1}R%-+asYf}BwaI)=9SxWi^F8FPDD*0!H9-L%ySF9A
zEoSlolX=|t6IsKbo5cpggEdzwq>uPw)T&%eJfkM(wkC6^xbjt_0+PsH_gZUnIKID6
zr?f*58jg3pDvq&&IyO^g-|4+T;HCBA1;@>Tpz)i|5u>x84TE1<#?g%JZHhL(qw4Fl~y(xqjcwc9kH{mvJv_E6DgP*W|mOS=a&Pa9r|{|G4SugmO5s(4E80
z$*_0ZdGIUfMZBo_X(4HuX$`u$)Q9`*O!}>KjYr#|KD!HZ{k`0)qxsYWnWAy;whQ3d
z@zWnOc)9QN8-;)GQC;YIPFNqXTb1C&yv-PwNf6A4RqR}{^ud6P)9N^Ohq~?xUuf>J
zK>hYTx8=_fO~ErUi_EQ7=C#Z8s(Q
z^lPamcrR`Nj31+cj3WN{>>1$=#OUKrrW0GmNx}UGu{gU?OdW0Ke#8i~p8fSrf3-4K
zM&vIX@t$Q*b@o?r1ZvFDF+#tjjNYAx1X8kc*ucd|E>
zWW4^uNcy$z+9wkB5VmVQvy8hRs;my>YHs){lYP6!IB*`GV8+_3kt`DkV7p92P0htQ
z_bm!u%GXc2>q-jd)+?t-c|B?P`l)k8nUEMpFILLc#M*ZHEQv}_8zQg4$9gNo
z2Si|xyCMh|-ao}6<#>el(0l8gh@vTKXC%gu)4W5e-%#Wz?eXww5IelyaDA)TC|55n
zCza2NjIaFCpe3b`vUx{Q!D(SFhSR=ViSAB15&G>PIZFB8ibhMR?vs>1VJJ(VeKA38
zzPaZ~dbA`^I#YwIxbmPgzMV$z9pd%ioGdy2J3NjrU9eYyl~PYN7Gb@T7e!6uue
zoSQRNi9Ob))DP(Uqzc$Gm&YsjdDFp`lVIp7NK=t^@g}YJ<)qCHJg_EVRa!5|j%61i
z|BH$sT!UEQjcUN0Zat)p*(=8`u{dHgtP|gKW1_dj?7hP3x4N5Nk$EaGuZ6|(>So-n
zGn0Uaj*8F`v0#d?3;WV~3O_A(Q(Pau)nxJZvSu@g*@iQB0g$&4KXyIFPS&Tj$o_%m
zuQq3{1nUPegg;W|pWt_f!;GrbOy1e388`pDczXO3XZFL}2X;Lkp+)kSUNbbq+n&2g
zlYM3+d;kgx1f1WB9JYxKT(_dlP1V!_VL+b9QHRIrjJ3XFg`W-Y>Q-EpUmDSw;s&(1
zWF(B|oVe$-jFlyiR!a-S)L)JCI{FAAFotUta$}1$3Lq~>-YnlWKik^=@^)J;;OId3
zfcpeNqjbLU0*5}HNlKjp&u|{PqGRTL{D9_b72e5X$#TQPK%?g8!**Y<$=0&P>L6Ko
zZQ^9Ib!1``t~TXH9Vv^v5|}$;ecrexkU>x}cXV|0V!BlTkXL8gg6U{RG?BT*P~0hY
zI`>^&&S_}!OfA(|Aey;VtWHT1Z%1}w>6X6l{^93gtE7=5f98(;g3r2D_bWi?I%fw9
zUg^^b(>Aypq)f_(?%k#0qPt7QUg2)GFc&9wiEnAj9_jq?q4`4A?X23QD6RJ)(2L<4
zV-IPn_eU>}jz&7tqFgTfeShI>-7SNx`0}$$9tFnecJr(6-G82&_i%>j@^einzF0Lj
z%EO$_I5$Fc&+9Qh=`fe*?Bzj%X}p4#w#t?2mDl;qC&p~oSmK^^ETTC#PmCe5rLYhRGm_JZ=gL`XwcFfAiL`W3A)0LeAim)48?2^v#4H8D
zcbON1BPwp^L&VN4E3A(gEY2N_h??g1q{I&jj9oQ0*!tEJi)c$}6lAjKJKWd^pES_~
zZ#k@Gm7A^TCuf6zPjLQQ9?3RRIzTXuR+#EiGRVcZ({=$it24d=ZmW@1D~iW}*9X_7
zJ=d~D`cx#*qk#=|ZD7moT&y^Cp;}cF+8fbookUA@x
zd5{JsLHD=~+-EG0U7bl0alX?>qiS07DKHeb1?;2(2jtd;DwS=e^VgDwM(^$=z$WuN
z&54){#4j2=vbfeXC(q_fqSE=Dice`z+Gr%CcK7H64}4e&6g0Km?i3t6>Jt$g~J>IVFsMn9)KBpZTh6>zGSykF{@{NrSokNfS@RY
zP){*@@{-2=)JFmx(5rvw7d|l+nysS&J8`-!))Vvgcn8?LKHax8#YrroROm6Za~t?P
zlKn@CvdDF9_U$(ZNwCQpPpL$<1l1{8@8^Yh+hx*vTaX!1;)1oOOZ4;xhwLjvLA{-h
zkLQ_*{YM5g#2RV5s;t>eo(@PBnEF;gRD!De_lIK8Xb!8oq^TB5zLt`!gwOOUMnO`_
zrf!%HosDsCpXNlG<%CL#tbAl?|8M;z@SVry4n1R+EBMS!7K!*f?d01)|?6
zH+{7X*{OUWc=IglcN=Frj=bO(z^(nY9vS%PYwhKDdXga*Nr#oMLy_N*wC%{_6
zA^IGrs-PvC$xx%-Qa-jhReNp9@TficY5MTZJjK59;$*okn76|SGz^dGeO*uMF7v%}
zg-+Bys}znDig;>Hc^2FLm|CT%)98hypCpxNl#{F;J5=RmK3%dB)f0ked*PI$4-_cy
zob{2VY2Ri2^_&WsN{Z-kw{NeF;M9~PX=906u^X2xpVczm#hHPE_=KCa4wWeRcm*w=
z(~p?QgCB_fd29uwWsJ+H&TV|wEXm!^S=;I@`nnm@Z(-jExi0TiwEenXjF_mLW@28R
z5b>cHxbT3MSjUKWdPx)y_~sNrMd@1^9_X|Pb|kCnm~EFqRMYA(_H`Y!csr_YG>9iL
ztiAP|gw}2>E!<62jrqcULv;(H7?DU%;3g|TXX$K6{C6$@PrD0Bp`s(KhEaF>3Kwdb
z+>6y48AR>%6kk9Tak$|zJLh-j?`?g#r@iH~zWgAX{2^6S+f>Y2jaY;RGS|NiuL4TN
zZ=G?>P#69XpWQpWvs)meQJ83^QG&eRgqJ)*`nGcjlQ%0pZvU{O0Z9^$w$2jiaF;&J
z_mxvuJ9MnlUL^T}-c}N%ucuJnJDwV6BFcF7b`HCKo3LZAR%B(+PiKAATB14H8S3sG
z^iJ2)F56wY@X`45&-`CX9W|3&a-YbWkDpum(21g3eNH33CX0*TrW$*9!P@L_ahz1p
z6Ind>yijXtA8iKz=#X@b@&xCiZh=0Vl*T@7QIl5G`MY#;y_NWg%!!*iGVrzaREq)J
zCxuBdKCD`jBt}=L&?>xAn1Z40Qk7NoTpKD>vm+n=$2Ve-BHR54wJ+?AC0HNffM0qyhZwycHaK|VopndRn=rfGR#PFBV?ub
zvemDuvt)kf<0^Wl$iW~qmm&KF>w55DlyNErwS`tbc*z{JV-H4l;bZ2yxLQSf`M4$jGMT>m9k0G7_>%ZW)9wyJaF90#Q+?#{6B*)UbJIG;jz+nW
zr{3j~Q9XGl;(knc{Vws;SqstJ0n6wD`|c2q{=lwD!4bzJi>tyTFC`tj~$oR84vkF@~xJv_xOg#-UXl~lXF#|(o%+vkoH@l|E?kbbXrPD~UZu0q+
znv&u|d7{huyE2iC+C)UWlSGJ>uIJn_S>g$F@-ExcA_qg^;TWZL5bhhCwD<6qmPlp<
z`T!(yK%96K56=WVL*@u;#zj%R#)1c)ar~;xXw|G~
zP&6O}*1r57vBF=Fl8;i9v&n;7L3{JD|7XYwgrZP^v%kXNt5BWig^+bgQ0+~b??SI3
z>9LbTASY#c*6Ke3cBs~W2kaOs*+`YQkuZmcZkkwmko3s(r+1Y=0F2{NR8yg>9z{q|
z>n6bgfhw`mM-o^?5%#Zgo-`W@5F*>C4Dl5-j;u^+ADbKj{8=q-+~A8`$XT@75pGT?
zbR>|-OAg}>NXRjS%phMgcQAw`loA{xlg5jXJq
z(*6-2j2qC-rFcH=}ppG4KuY
zkFaYsPsjh1vRq*R&8Od*T%2R-V7NBQL4nt28`8xv0Ju{xuk$Q^|Hz$v$nKE}$fEGl
zc-NfG0w43Yl9>SlyyQ5m%K(ANJ$%xKAo%~V=^SFbZGDtxV8cwdSmW
z{|tVj2r8gS*(}=sE_E{WPr3HrKu4ahd`t#S8d70M0cxF(h#iExo3j9Y!*Gq?S0J*o
zx1IJ@Poa!n52y}hiLPk3sIq`T&R%V2{a4)I+7{`#C|I=wl${*b{sqLiZ+G_%DF5cX
z>w7!Zk;*s95du`n-_atJGn9eNZXAm^9}>;2Kt&2Oh+s
zvmj985)Fq3Fe9m5zlWfOf9MHxo6fX2eG
z;=ePZKfsifl>AbtO8v3oZIkzQ`O=+Q8gykqNWaD=BnKq)SR#twe=9RqfWGif4kSU$
z_r%4W9c|MBm{;$;ZKBjT`*-rfEE%YE6#4Szfj5R5zW@`{N7|0Sv%{tG$~LUbb&wAL
zX#JfCpiRbr<}}O=se1wZ0s%6U#I>pp=LiN6UH_fAHERN>Y!~N^1~(FXQZ5U~B5oyX
zl+RLEr=-1?(10{{zkm~?NO5uKnCm8hX`N|BCvi>}gl-zwm6+?tfv
zjtHc9h3$}>HT&*=X$|y8rH_Gn_*`ymOx#8>^`T7eF?6$SsN1%;OP}QK5uk5Pa?<>j
z#~@A)IREb-BFg02If0eLViPm|PC1}^#0eDLjXt|W_lh6fO-qkw(*Xu$86Bhpyg(t4bgAC}m#4oqQg%_KD?}JqRglf>mSNs-J|`CTX#gzt9i*SpbDYC5kJD
zF#^&_3_CM(WmLs+6akSR@lh3@d!=^SoE5rEzA`CWdtE)sih-MSThyK%%&?;Tr&Z+=oD_m7{M
zx_|)a7oIJZZ@0&2g+z@$-Mul2ze8Wa#Cn;@;CC}h$Ka2QD%;ID5Fu_n0ega-mm%yO
z^9PTb-}9n0^HR;fMXlu-d>xFCfxKXK=Cp73H;DBzI_QlFa0dP{(fp)Jpg*W(XB5?V
z*q_GJl72US4jR8_0Cd3+HU&}B+qeS_Trp3s6c)5!PHJ_1XXtFTL<~sR=PLnn?XsX4
zHTvEm6VT$52q-0bHS!s3qnu($r~*h*em-Ys4BX1b{49CBhv{CVkoS!nA8r`0KVUUT
z>1zI_@F{Kc*8mGuvbz4u#$(
zAlW7jD}xu?N4cE@vFpb%4JIC#GRj3QCPmX7Ii5fol>WfjnAsw^_d}`~&?``6oXiLE
zTMXg|PSVQmso#xv{%O4b)9=QEK;!ri&?7lsbUI-&v3f;?x9)Q)>Sbl{?kmg)OqB6g
z??u8K>OQ)Q(#_HW1CB%U^-lyt{k|(aHblQe66&RkyOya7B+CV0a3$u0YcOhV3
z1O3+qUSc*g%tlrEh>$wF+y+3Vo?*y@UCSBq6}1rlH8#M)(tO+XUxbVTy_?W3+`d8h
zs*?29*!&i%@jk?##)0lojf(-Z2~P%DzP$*3Ft3{6(laNi^sK4_FMac!DKmoT_IDt~Ve7R@EeMe1mN&rFniJ
z^YP=i5~ILWN9Z{#-X*wM3AOA;hAn0MkLJJ>c-GzaT^jU5A?jcDPlWi}RCnNd%~a9+
z^`Rk#AS&WC8M)iZ@aSaOGHWcf9J{-n>M4h3FOfxnSng|1=O6Wy#lyvs1WO8}4`Z`3
zQA%D}wt$KT>~P#Tpg?w$lZd%D2!3Dgh+UY0Nj$-{(XS(B7|4Ewt$Luri_aAV`DU$D
z@ur269Lzv~BFqy783{N-K+n;NHoh@ps1PJG!hOz`Hz5wAOCUf?O{M%1l-9Nzm%Qx;@9c@p^%BewW
z9mXQ_dSB!q_HPtvR7ai~Uq)a55-c5jYgH7DrIt;$#*8{w|A$p7w=eE|D=Eos=^qbc
z>`Q9*3kONWvZF{$&^~G1e~SB%7(S-%;b@et#g
z0?{6(oz{*`hEO1`sR~7;y>9tyiJDMSZg-xlrTzlxdY^H#RpQB;*rqDslKMEbs*i<>
zH(y_b?f8B!7z=jFPDn(Td}39W@l2PX<%!OLVXpM}yv^Cw77GyQpo
zZ6a`*O24WlF2o6)3Ywe`1z9Jp{RG5Ii^NCrqU)%E|
z-bY_gtuZTTyn26u#T4K0iokvZaup|x*XO%CZ>CA$!i93Dz3a=9G4!*FCW7*Mc@S^M
zVP$N`NG)cy8a>{cT*Fs3=)ytgq~jWvx&-8JFY7E$2<(njB6k`y?RO2njS9n>?S3d|
zOVU*2j7QFuRBEq&d3TNN(hVD%kl`1mvcgf~vlq-ACV6jP?5+920+ze(V&2}&eF>tB
znAR#;f#r~}m6xn#0$!`6H)5z(?-octj$BK}iR3^jh$uZFBp4se@$;u>x(H14x+EE-
z7_S*%s!8t(<7n7e-hEp7*{If*IiW4h2ywYoXo^B}+Jnp>pM2{;ZqFd$D#9fdf6n4L
z+~-l;<9U{i`J!Wa`0&=qcAd*b-MWH-9&M?J|AjnqFiZHUD9v<)i%+?uLEYV((w|TI
zuV$c^3y0K9zk(3jg(}cXQ&AA9J+qH`AP*AH0d~R7HdUFH?;sdn_>MtV1JCJ-7>Z37
zhIk)siTr{t5sa2!hThO7FQPQpzgen{JlYt0%A8whw)aHdhjTh={+rZEOJ0V@cZ|pa#ZQ=f35Mq@z1Tanjc+{sW)!P)Bf;Ss|~2d5muxvg*jijqLw|D0*L)z
zYpxHWP!YF)4%Hkxhy
zOYq-d;ZHYV4svAU1)8XuXPH3?UfoqaYMmuU4c4D^7PiqZ%waL&M2HnxNn>k~t}>N8
zsPh=?tp1Pj`;C#>|Xxye?v0go4MCgR26~?7v5Q4*M+QXVZi1vPeLV
z=)%fH@N!^&L7iM|E~G=zZH*4;QD*~{)LNvW5)m1`~982q!%YA
z`WN^H5YCEu+8^>pw#p1
zX+DS}0bN>Q--M4{HoRK7^&4BQ-lYsXT0)JskT4>fI}oMzuZsx5l8--vjZgqTvVrz*
z!GG@>{#WsZsARRj#t&rY{sp)rE4laoTknR~!G{0;vZ#D0K%i28v`(HNQ~PrWXq|k=
zLf|YM}MQsq#26j^ft3=4E-jdAQ=>7yY#A;`vl!uogly_Y)0;R{>brp
zY_vMjh{$u0GkrxCisjj!*>)+@!q@WlQOCWZaveU6bk=UsZjiG$*V9ts3X(qp06#z9
z6+!t%If)DYHef=n$KvIWuJ=D-0lJH(ACZFbfF1f9Ne)!ANfEWWEc1V=<`cVg{q_#C9>`)+qL?&p>AiCMiE|p08ks9;
zgy)bN1vhml{xiXy$jY+s;zzvzpavO|xn34V9anl7Q%vws
zZ|BKVGUHz_35s1KR(Pd9Wha+ab}QCFANlL&w1(p;wy!ATRwj2r+k|a;wFhNv_|)Qe
zA)VPJB(_$%L~h~2PDJ(C(FJE@plQUl5
zXQOvyAPWB>^u#{~LDQr|=Q-qV0r60gAAIVYqO8VPPaA6_yEUGnj`=K%
znkq-*(BsU1zkemKJ;3GpwMCiMgPbQB=CClTj*4MlT?XEw5Wok=0>J4ll{<5|61393
zssXo%8px&&I^TYer#^l1kWcE&%CifrE_aY{`v7fy_Vx7wtIK1YYjtY
zwmxkD$TQ3%(3nf?o9RM~lsqc(33L4S@M>xCI9FFx9R((tw1)-lQC
z`<+rr@!6Ctt((RIxDh5Q;q$zPv!`D_gX5TYytagXDM5pjHy~k{wc3Fc!fpiGWS(8!
z+ap5sf!UX@s@eFtcmp@aPJ}GyTa#r3EHg8#`yn74EojAm>XJf)Fu7C~A8g+H@h$xB
z%{C3**7fDsp!?(&4NoVbCfxpcW1Io^KG7ezRZOLbsRuZz*a?n&4KEVV$<0E|m_M#qwS@!pG3Gu$
zn5Ny+O1m>ChR5lpHd<(~Tr!3)_3R6S{4bofceq_|E9r|rzkoHbG|{edJ$hHjBN@c~
ztyo=o_z+|e4X@9?SM{gpl~tLT5gilc!_N@-eS{}IWC5=)Q}C*KVoIdS6Z$Pr1hlmBP-+
z@d~Xl;bU)}FEpc$^tSt9#F8K)Do5k7^=dekGG1*zZl<<+U!fcPG$5hNmZo>6JY&tb
zOKbH|WUcbcl>gE0`VqOX7~TwEi<(c@H;>)sjf}1??x+M*vnDWo>?w4W9Xrj7vAY4b
zM}!PYMM%A5xsQ}{`|k6CI>;phkmtfqwd<`*`NUtWQ)rDd$k889^Q6T`V@DGfAWgB8
z*wkGpEgv5+FF*be{rG4mSOnH=^}TM426rGofcvf}p=#bo*eS(ib&SZh5`+#B;&!};
z&Ag&561TJZ=ARg8$V{>fQZuc^;sL{F<8qZRZOL?R$FSL=TQghR?ahu@&J)}cxnzB7
z7KE1iOtP|NiRSzz6P5@WFkTLIyxJ;b>JZJ(6yy&Zg(!~SB$*w^6!O2}?G{sMgk+6b
z1tCT8tPT*z;TIg_?s8*sR7`SLU;``6rWj3+-YQ+?~-sKea|2qJt#kryn%*+ww{nfHE8gq{B|ZJWDDJjrOjHINCcT^fUtZ4>@2
z<;4kW$5-XFuqCBi1;(LnD15n@;QOJ!6CjgYaCbjI=5(gz#)pUjb_Gb`Sas*OtPBy>
zUw9^y2X7z(oa+3NWL#vA8;ZtJn!`YEkxtErDzV_9|1R0~;twJ}5Mb@0M3m(8xXQOT
zVE!UNfKa6q?y9Ies+8UlQI&QWz=tKX;0OY25R{n$TgR=~HkuOebOEa1V|+9;yql>1
z7XWZ2|3;~(?a80Qph6Hq6N-bCO`~=thue$ZhN41(F98a5`z7E_P#GFx0;8DP0L+g4
zyxq^D->@(Mlz9EdOU&~`aHo3BeyQrWc3G-FAHN02@&jRvA0{A+
z;8_a^Ax+q;^c|pPZ>qOEIrHdNA=w~goldAnmCPN+Eej4;hAWA)LzgvaQ}wAZKuaKo
z?Y;L0g`;pau#ZU;tFv`XUgh_Dkp70pCGCG3(4;b`0WhCU`{DUez-s)^1!4)lUP1`2
zbVYL0qc`DyQAFk%;sF9DMJ`AN87ot~3!`#vM+CrA@FFs|W`B@`s
zZ?e3UOibGVHUy7Yd%#Hu?L|R0?D}*tHvQlz;|f%f7Q~wCD$u~G22@+_0`5cv_a1*l
z=}VONN!k+4ET8LaY-KQG8?dWl8;$=%S^pmj0-@e-khZ|mNMp%<_QRg-R
zyOe~cNV`FHcOVM$9MzQ#ZChk8EjS1X>@fthnH~^4(*)-Jzh@j!oQVjpWL!iw9?DDJ
zL%*T{){?E~M*nwkEy|r9^;`^fqH_hy>*NVBWQ_<3&ZYa`*_Vj~2_44VL#4X3A8t|z
z+7R6j(m=nFgGN`HXmk8~xD5nfGST?6j22LLIqTV8d6&Jbr>G%5~h&59fzR6Yl-L
zO0GN{%C(J4%5u_}BC-zVpu?GI!-$Y|B!iMAvXvGj4vnSGd|8Ur&_R}(DJsh+F@$6-
zGfb&06*+CFL}hECk?q{iJ9T~E_vd$A?;r1Vz1RCZ_jBL(^W69Q-oN{&7h$Yw5=G!O
z8kKrC#!34ea#53k-)BiTAJqpO64||JlXY)?>`(<3|7lQ#@GCsP3noF|cl-=4W0>_@$qZVFhUSCe||D^)e+BJn}+kn
zS2b!?22`HYlVA4T3KZX!BDeD(uc*I^68T-ZkwuYx$`eQz>+`a6uKe|3)J5k)p>k#d
z>7-uY0q;7a;+|ryHoY|8d#sl1+Nk0y&{_!!3ecS{uTnhlj
z@X*+>H5wXm)-u{Ddpp=Fn&M1e>V9&OnLWM{zPR?CV!vVg8RI*PuK!ke*8f$*k;tn@
z$5krFUi`MNKRU}1ZR$)Lf$Y*(SjX`gDulMDHpvb&vS&2LPKR~-tImIMf~t~-zLL?G
zL0-wJ8yMhKt;=W5d+cgv+s<9xlhp@1mkt(ZS`I3n>mD~KeU&74xoOSm0s*o4*YI=A
z96KKk+lveIv>E#Q#00Xo)ko5Gr`O!H;mt&+5uGfHdcmhB#Z
zYft#qmY(avc*NO>m$+dgj`*X-3q31Kk~Hf52c&}CtL|uaxsiQ#etaGtV<>9;wbk~-
z=R8;LF2T?pv%7z3Y11$E=FzhT|LhwzDx#l=&T$kT#Sc8-NH^GLdGjy)NUwTTo
zdO_-?SdLn?AHck**=yBz(F5Om1h-e-;!O6h3LCy|R4aC5ONes_p^PhzyW+?vlt_85
z-h&@|R;F8#-s_LJ4wC8BaGisu!Rq1sD6<6alf*%F})Id5iMOYn{iG`Dp
zb!oYCw?tpvQO9nH!<#FY1Z9@u^F_@r_f0SY%PTT+dlOrrjm4ZRl2PXB-M#d_Zp5gT
z9=pzJwDkLSMQ37=v)=Pc*?WE4O25zj_)|=zW{VgeeqnV!@=?*kx%}ni)9lyg<(4Lg
zpP4w*!c+kPc0HkTh@7T2L}Obf-`q}M%nCj@d3!oPeUr-j4f%ZEa{HK%-<~U&24qu+H&=WqB;Ves-bm)fY$<@Oxf}+US@*DjAxsvfckM7z9@}u5;UM=evwBt+8m5;wRcS$!F+VoNcEJYNs1S_G|Sk-lg
zmJpOVEi19_iUKpO^@2;5K3Z9ATP+yXGawenqXe*e%j_8y%W&m&nANl
zi4m=Wob1if1~xR6yg(+G{JX}np$_BMb~5pt@aH7nDJ5}-U4}Ld1(PoFp9wE3rteP-
z4`}Wc!VV`!l-)>P9|LI%}j4=7MVrP%&&Y1=(^D=;
zgps|gYuE)Xw-n6H%T(ZuQIJul+aXqi5Op|XLQYjSJYDlKrh`R^*U~q23&0OJrgOfP
z7#aIWKzIghAlbRU0j1*o2o;s3ecU0lDS*2QigPlDYOA*}Yr7Av*QIf(@DKxQU?Th%
z+GH4IUywVs$NEL0Va!Ee;lZQEqssL``?*;Xw)`|lrf;T@9U&Vi5Q3%lEw>P5SVL0g
zHt^OTi;p=70PvtD42tOy!YIwUD9JKI_H(B~l-5h=$g{iMB20~%TkLrG7`;Rpgw@6s
zZ|7jvb#T(8Nb0|`L?RG#b)nikoSe>vX%7%ViW?g(G(!!9h?cb{MR505bI&yZ_ryvR
zt@rQ>Z3N*{p+e3FO<1bZlK36!ixS~8$C&8E(!%^`0}Lq$kGN&9ftEm54HLYal(|~5BO%`n250u+m$dS
zbSuEgJD}mFkmaUME!l4&N>fKFyym1ZA2MA@?)PuNqJjLK(yYeRQ!IbC|wg1kOTsT-~n2JvK>bd
zaBd^VYn7w|73@v$qGQV<<}%^LtPJzlg?axJruChMq!A{X&I>VSc*%v3GR1-|g(?hQ
zxt*4Pp0RAfSL(CFT8aHQu=u=;bty(wqf9d|dz5sTR(6CU3EH6E4i>du?_$xl?A)J<
zaN3y;ym^S6;pYV;CXURBn@NBCgg;>q&cmYh{rokNE|6k`w>NFJG=&NJFE$BBYwiq7)z*>Hh_kN&Xr|Mjlhcg5`Q$
z6!9{Zq@Y`6%w|YK!pqzTC`$^-j#HQq{SdQX2HY=EEcr2JQ(c5I!thSi+iu`66G9ji
zVUhew*ktnJm+Z9{nGwJk3>hmDrjrtY{8cBL@FB16j0Z9oT=Gafrr)h#PLi}qmV+q6
z0&W>As3h#cPhr%MFgV*Oi~lH$*uNGDv*gTUEKApw0AZv)AeS;HgD{<^i8C0vjXZ*K
z$uDKwYb`RB6hDWOX|C!)^zP%t`_x40UMDtcf;6xR%v!+> This script will require sudo for certain commands
+
+The password option is the *secret* password key, with which the disk will be encrypted, you will need to pass it to launch the VM.  
+
 ```shell
-bash ./build_debian_image.sh -o ~/destination-image.img --password your-password -r $ROOT_DIR
+bash ./build_debian_image.sh  --rootfs-dir  $ROOT_DIR -o ~/destination-image.img --password your-password
 ```
 
-> If you need debuging you can pass the -x option to bash before the script name
+> Tip: To debug the image creation, pass the `-x` option to bash in front of the script name
 
 ## To test and further customise you image you can also boot it inside qemu
 ```shell
diff --git a/examples/example_confidential_image/setup_debian_rootfs.sh b/examples/example_confidential_image/setup_debian_rootfs.sh
index f87683569..e3d320b1b 100644
--- a/examples/example_confidential_image/setup_debian_rootfs.sh
+++ b/examples/example_confidential_image/setup_debian_rootfs.sh
@@ -106,9 +106,9 @@ update-initramfs -u
 # Generate system SSH keys
 ssh-keygen -A
 
-# Example to add a sudo user
-useradd -m -s /bin/bash username
-echo 'username:password' | chpasswd
-usermod -aG sudo username
+### Example to add a user with sudo right
+#useradd -m -s /bin/bash username
+#echo 'username:password' | chpasswd
+#usermod -aG sudo username
 
 umount /tmp
\ No newline at end of file

From e2fcccd9fa507fa5a729a073596e216f5cd4e53e Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Mon, 29 Jul 2024 13:13:29 +0200
Subject: [PATCH 817/990] Enable Qemu support by default (#662)

Problem: If a user wants to launch a Qemu instance inside a CRN, by default the support is disabled.

Solution: Enable the support by default and disable snapshots because aren't used.

Co-authored-by: Andres D. Molins 
---
 src/aleph/vm/conf.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index a192a9b4e..98e3772e9 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -248,7 +248,7 @@ class Settings(BaseSettings):
     )
 
     SNAPSHOT_FREQUENCY: int = Field(
-        default=60,
+        default=0,
         description="Snapshot frequency interval in minutes. It will create a VM snapshot every X minutes. "
         "If set to zero, snapshots are disabled.",
     )
@@ -261,7 +261,7 @@ class Settings(BaseSettings):
     # hashlib.sha256(b"secret-token").hexdigest()
     ALLOCATION_TOKEN_HASH = "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda"
 
-    ENABLE_QEMU_SUPPORT: bool = Field(default=False)
+    ENABLE_QEMU_SUPPORT: bool = Field(default=True)
     INSTANCE_DEFAULT_HYPERVISOR: Optional[HypervisorType] = Field(
         default=HypervisorType.firecracker,  # User Firecracker
         description="Default hypervisor to use on running instances, can be Firecracker or QEmu",

From 307cab98f0fcc7a24066f13b735632301ff710bd Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 7 Aug 2024 15:33:54 +0200
Subject: [PATCH 818/990] Fix: Droplet with Ubuntu 24.04 was not tested

---
 .github/workflows/test-on-droplets-matrix.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index adc04e0aa..876495a35 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -52,6 +52,13 @@ jobs:
             package_name: "aleph-vm.ubuntu-22.04.deb"
             concurrency_group: "droplet-aleph-vm-ubuntu-22-04"
 
+          - os_name: "Ubuntu 24.04"
+            os_image: "ubuntu-24-04-x64"
+            alias: "ubuntu-24-04"
+            package_build_command: "all-podman-ubuntu-2404"
+            package_name: "aleph-vm.ubuntu-24.04.deb"
+            concurrency_group: "droplet-aleph-vm-ubuntu-24-04"
+
         # Check compatibility with all supported runtimes.
         check_vm:
           - alias: "runtime-6770"  # Old runtime, using Debian 11

From 7989177d5b004c44a20439fef37cbd5cdfaf3875 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 9 Aug 2024 12:11:05 +0200
Subject: [PATCH 819/990] Update confidential README.md

---
 examples/example_confidential_image/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/example_confidential_image/README.md b/examples/example_confidential_image/README.md
index 5618db260..0e91e6949 100644
--- a/examples/example_confidential_image/README.md
+++ b/examples/example_confidential_image/README.md
@@ -47,7 +47,7 @@ sudo cp --archive /mnt/debian/* ${ROOT_DIR}
 Clean up the mount
 ```shell
 sudo guestunmount /mnt/debian
-sudo rm -r /mnt/debian
+sudo rmdir /mnt/debian
 ```
 
 
@@ -76,4 +76,4 @@ sudo qemu-system-x86_64 \
 
 > Once you have entered your password you might have to wait a minute or so for the disk to decrypt and boot.
 
-To exit qemu : press Ctrl a, x and then [Enter]
\ No newline at end of file
+To exit qemu : press Ctrl a, x and then [Enter]

From 902cf85f4437b0bde608c90942ec15eeadeedc45 Mon Sep 17 00:00:00 2001
From: philogicae 
Date: Wed, 14 Aug 2024 12:24:31 +0300
Subject: [PATCH 820/990] Fix wrong balance endpoint (#666)

---
 src/aleph/vm/orchestrator/payment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index b93a90e45..65e642e01 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -29,7 +29,7 @@ async def fetch_balance_of_address(address: str) -> Decimal:
     """
 
     async with aiohttp.ClientSession() as session:
-        url = f"{settings.API_SERVER}/api/v0/{address}/balance"
+        url = f"{settings.API_SERVER}/api/v0/addresses/{address}/balance"
         resp = await session.get(url)
 
         # Consider the balance as null if the address is not found

From 1af9c7598ae129e95a2e79dbb27f193030fd9370 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 16 Aug 2024 19:06:11 +0200
Subject: [PATCH 821/990] CoCo image: Improve example user creation (#669)

---
 .../setup_debian_rootfs.sh                    | 26 ++++++++++++++++---
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/examples/example_confidential_image/setup_debian_rootfs.sh b/examples/example_confidential_image/setup_debian_rootfs.sh
index e3d320b1b..ff732dfd8 100644
--- a/examples/example_confidential_image/setup_debian_rootfs.sh
+++ b/examples/example_confidential_image/setup_debian_rootfs.sh
@@ -107,8 +107,26 @@ update-initramfs -u
 ssh-keygen -A
 
 ### Example to add a user with sudo right
-#useradd -m -s /bin/bash username
-#echo 'username:password' | chpasswd
-#usermod -aG sudo username
-
+#USER="username"
+#PASSWORD="password"
+#SSH_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArQslTrAf9A... user@example.com"
+
+## Create a new user with a home directory and Bash shell
+#useradd -m -s /bin/bash "$USER"
+#
+## Set the user's password
+#echo "$USER:$PASSWORD" | chpasswd
+#
+## Add the user to the sudo group
+#usermod -aG sudo "$USER"
+#
+## Install ssh key
+#USER_HOME="/home/$USER"
+#mkdir -p "$USER_HOME/.ssh"
+#chmod 700 "$USER_HOME/.ssh"
+#echo "$SSH_KEY" >> "$USER_HOME/.ssh/authorized_keys"
+#chmod 600 "$USER_HOME/.ssh/authorized_keys"
+#chown -R $USER:$USER "$USER_HOME/.ssh"
+
+### END example
 umount /tmp
\ No newline at end of file

From e83d45b5f4b6c98568f581365a40f5ab3fb2ba94 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 16 Aug 2024 17:12:39 +0200
Subject: [PATCH 822/990] Problem: using vm_id in qemu socket could cause
 concurrency issue in case the id was reused

Solution: Base the socket name on the vm_hash
---
 src/aleph/vm/controllers/qemu/instance.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index e3c2e5435..be4e2def2 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -166,7 +166,7 @@ async def configure(self):
         """Configure the VM by saving controller service configuration"""
 
         logger.debug(f"Making  Qemu configuration: {self} ")
-        monitor_socket_path = settings.EXECUTION_ROOT / (str(self.vm_id) + "-monitor.socket")
+        monitor_socket_path = settings.EXECUTION_ROOT / (str(self.vm_hash) + "-monitor.socket")
 
         cloud_init_drive = await self._create_cloud_init_drive()
 
@@ -218,7 +218,7 @@ def save_controller_configuration(self):
 
     @property
     def qmp_socket_path(self) -> Path:
-        return settings.EXECUTION_ROOT / f"{self.vm_id}-qmp.socket"
+        return settings.EXECUTION_ROOT / f"{self.vm_hash}-qmp.socket"
 
     async def start(self):
         # Start via systemd not here

From 0a62fba8ce832a21dd8f396859db79770ce9dd5c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 16 Aug 2024 17:14:34 +0200
Subject: [PATCH 823/990] Problem: vm_id allocation algorithm could allocate
 already in use id (and thus network interface)

Issue happened with systemd run controller since we could have an id allocated but the controller process not started or running at that moment
---
 src/aleph/vm/models.py | 4 ++++
 src/aleph/vm/pool.py   | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 6f87b2363..6d6b77d97 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -96,6 +96,10 @@ def is_running(self) -> bool:
             else bool(self.times.starting_at and not self.times.stopping_at)
         )
 
+    @property
+    def is_allocated(self) -> bool:
+        return bool(self.times.starting_at and not self.times.stopping_at)
+
     @property
     def is_stopping(self) -> bool:
         return bool(self.times.stopping_at and not self.times.stopped_at)
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 01e5afd72..2ecb673df 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -161,7 +161,9 @@ def get_unique_vm_id(self) -> int:
             #
             # We therefore recycle vm_id values from executions that are not running
             # anymore.
-            currently_used_vm_ids = {execution.vm_id for execution in self.executions.values() if execution.is_running}
+            currently_used_vm_ids = {
+                execution.vm_id for execution in self.executions.values() if execution.is_allocated
+            }
             for i in range(settings.START_ID_INDEX, 255**2):
                 if i not in currently_used_vm_ids:
                     return i

From 189124bfed75a25fd26d4a8c8d7d21cd42230cce Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 16 Aug 2024 18:02:50 +0200
Subject: [PATCH 824/990] CI: Add more debugging info

---
 .github/workflows/test-using-pytest.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-using-pytest.yml b/.github/workflows/test-using-pytest.yml
index e618ab71f..3b8cb97a8 100644
--- a/.github/workflows/test-using-pytest.yml
+++ b/.github/workflows/test-using-pytest.yml
@@ -52,7 +52,8 @@ jobs:
           chmod +x /opt/firecracker/firecracker
           chmod +x /opt/firecracker/jailer
 
-          find /opt
+          # this produces a 33 MB log
+          #  find /opt
 
       - name: "Build custom runtimes"
         run: |
@@ -71,6 +72,11 @@ jobs:
         run: |
           sudo python3 -m pip install hatch hatch-vcs coverage
           sudo hatch run testing:cov
+      - name: Output modules used and their version
+        if:  always()
+        run: |
+           sudo hatch -e testing run pip freeze
+
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4.0.1

From 76224992eff490c72bb60135d50cd20db5af417c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 16 Aug 2024 18:22:16 +0200
Subject: [PATCH 825/990] CI : Force the eth_typing depency that was causing
 issue

---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index 67850ab88..ff5e8e382 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -87,6 +87,7 @@ check = "aleph-vm controller run {args:--help}"
 type = "virtual"
 system-packages = true
 dependencies = [
+  "eth_typing==4.3.1", # Temp fix for bug in CI with 5.0.0
   "pytest==8.2.1",
   "pytest-cov==5.0.0",
   "pytest-mock==3.14.0",

From ae784065526f5012924336212f7f001f2f69ba0e Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 16 Aug 2024 21:07:14 +0200
Subject: [PATCH 826/990] Add duration info to pytest

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index ff5e8e382..ede2589b2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -96,7 +96,7 @@ dependencies = [
 ]
 [tool.hatch.envs.testing.scripts]
 test = "pytest {args:tests}"
-test-cov = "pytest --cov {args:tests}"
+test-cov = "pytest --durations=10 --cov  {args:tests}"
 cov-report = [
   "- coverage combine",
   "coverage report",

From ff6c119ec9f5014d806bf54a02cc98c91abc70e7 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 20 Aug 2024 09:45:04 +0200
Subject: [PATCH 827/990] Problem: Vm execution failed due to network interface
 (#596)

Occasionally, VM creation failed because the assigned TAP network interface already existed, likely due to improper teardown from a previous execution or a concurrency issue.

Displayed Error:
OSError: [Errno 16] Device or resource busy

This caused a retry loop, blocking the process.

Solution:
When assigning a VM ID, check that the network interface for that VM doesn't already exist. This acts as a double check for various issues.
---
 src/aleph/vm/network/hostnetwork.py | 6 ++++++
 src/aleph/vm/pool.py                | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/src/aleph/vm/network/hostnetwork.py b/src/aleph/vm/network/hostnetwork.py
index 0503efdc6..c2965e466 100644
--- a/src/aleph/vm/network/hostnetwork.py
+++ b/src/aleph/vm/network/hostnetwork.py
@@ -3,6 +3,7 @@
 from pathlib import Path
 from typing import Optional, Protocol
 
+import pyroute2
 from aleph_message.models import ItemHash
 
 from aleph.vm.conf import IPv6AllocationPolicy
@@ -136,6 +137,7 @@ def __init__(
         self.ipv4_forwarding_enabled = ipv4_forwarding_enabled
         self.ipv6_forwarding_enabled = ipv6_forwarding_enabled
         self.use_ndp_proxy = use_ndp_proxy
+        self.ndb = pyroute2.NDB()
 
         if not self.ipv4_address_pool.is_private:
             logger.warning(f"Using a network range that is not private: {self.ipv4_address_pool}")
@@ -220,3 +222,7 @@ async def create_tap(self, vm_id: int, interface: TapInterface):
         """Create TAP interface to be used by VM"""
         await interface.create()
         setup_nftables_for_vm(vm_id, interface)
+
+    def interface_exists(self, vm_id: int):
+        interface_name = f"vmtap{vm_id}"
+        return self.ndb.interfaces.exists(interface_name)
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 2ecb673df..839b1a3cc 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -165,6 +165,12 @@ def get_unique_vm_id(self) -> int:
                 execution.vm_id for execution in self.executions.values() if execution.is_allocated
             }
             for i in range(settings.START_ID_INDEX, 255**2):
+
+                if self.network:
+                    # Check the network interface don't already exists, otherwise it will cause a crash
+                    if self.network.interface_exists(i):
+                        continue
+
                 if i not in currently_used_vm_ids:
                     return i
             else:

From 885ff7590d46fcc7d68777496c827084b753dadb Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 20 Aug 2024 09:49:06 +0200
Subject: [PATCH 828/990] Do not reuse the id of any vm in pool.executions

---
 src/aleph/vm/models.py | 4 ----
 src/aleph/vm/pool.py   | 4 +---
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 6d6b77d97..6f87b2363 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -96,10 +96,6 @@ def is_running(self) -> bool:
             else bool(self.times.starting_at and not self.times.stopping_at)
         )
 
-    @property
-    def is_allocated(self) -> bool:
-        return bool(self.times.starting_at and not self.times.stopping_at)
-
     @property
     def is_stopping(self) -> bool:
         return bool(self.times.stopping_at and not self.times.stopped_at)
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 839b1a3cc..8c5f9fd8b 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -161,9 +161,7 @@ def get_unique_vm_id(self) -> int:
             #
             # We therefore recycle vm_id values from executions that are not running
             # anymore.
-            currently_used_vm_ids = {
-                execution.vm_id for execution in self.executions.values() if execution.is_allocated
-            }
+            currently_used_vm_ids = {execution.vm_id for execution in self.executions.values()}
             for i in range(settings.START_ID_INDEX, 255**2):
 
                 if self.network:

From 2f68012ade5862cc33c71196f2b19ffc09d68793 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 21 Aug 2024 13:35:22 +0200
Subject: [PATCH 829/990] Problem: Login token was not display with default
 conf (#673)

Solution: Display logging token at info logging level
---
 src/aleph/vm/orchestrator/supervisor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 09cb79ede..91edf2dc1 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -175,7 +175,7 @@ def run():
         app["sev_client"] = sev_client
         # TODO: Review and check sevctl first initialization steps, like (sevctl generate and sevctl provision)
 
-    logger.debug(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
+    logger.info(f"Login to /about pages {protocol}://{hostname}/about/login?token={secret_token}")
 
     try:
         if settings.WATCH_FOR_MESSAGES:

From 5e7d68632f63d6787e599789f6bce670be63f0bd Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 21 Aug 2024 16:19:24 +0200
Subject: [PATCH 830/990] Problem: status_check_fastapi endpoint raising eror
 (#676)

Endpoint `/status/check/fastapi` was raising error when there was no
internet inside vm

This was caused by `check_internet` raising an error instead of just
returning False

Note: Contrarely to what was previously understood the diagnostic vm don't
return a headers when the result is False.

Previous stacktrace
```
ValueError: The server cannot connect to Internet
  File "aiohttp/web_app.py", line 537, in _handle
    resp = await handler(request)
  File "aiohttp/web_middlewares.py", line 114, in impl
    return await handler(request)
  File "aleph/vm/orchestrator/supervisor.py", line 70, in server_version_middleware
    resp: web.StreamResponse = await handler(request)
  File "aleph/vm/orchestrator/views/__init__.py", line 215, in status_check_fastapi
    "internet": await status.check_internet(session, fastapi_vm_id),
  File "aleph/vm/orchestrator/status.py", line 124, in check_internet
    raise ValueError("The server cannot connect to Internet")
```

Sentry issue : https://alephim.sentry.io/issues/5654330290/
---
 src/aleph/vm/orchestrator/status.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py
index f09127f03..12692f6a9 100644
--- a/src/aleph/vm/orchestrator/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -120,13 +120,6 @@ async def check_internet(session: ClientSession, vm_id: ItemHash) -> bool:
     try:
         response: dict = await get_json_from_vm(session, vm_id, "/internet")
 
-        if "headers" not in response:
-            raise ValueError("The server cannot connect to Internet")
-
-        # The HTTP Header "Server" must always be present in the result.
-        if "Server" not in response["headers"]:
-            raise ValueError("Server header not found in the result.")
-
         # The diagnostic VM returns HTTP 200 with {"result": False} when cannot connect to the internet.
         # else it forwards the return code if its own test endpoint.
         return response.get("result") == HTTPOk.status_code

From 414ef8fbb4bc310c4c94f9aca27bac8b751c0331 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 21 Aug 2024 17:21:59 +0200
Subject: [PATCH 831/990] Fix is_confidential property (#674)

This was causing an error in the monitor payment task

AttributeError: 'FunctionEnvironment' object has no attribute 'trusted_execution'
  File "aleph/vm/utils/__init__.py", line 90, in run_and_log_exception
    return await coro
  File "aleph/vm/orchestrator/tasks.py", line 154, in monitor_payments
    executions = [execution for execution in executions if execution.is_confidential]
  File "aleph/vm/orchestrator/tasks.py", line 154, in 
    executions = [execution for execution in executions if execution.is_confidential]
  File "aleph/vm/models.py", line 113, in is_confidential
    return True if self.message.environment.trusted_execution else False
---
 src/aleph/vm/models.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 6f87b2363..42bc8c70e 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -110,7 +110,8 @@ def is_instance(self) -> bool:
 
     @property
     def is_confidential(self) -> bool:
-        return True if self.message.environment.trusted_execution else False
+        # FunctionEnvironment has no trusted_execution
+        return True if getattr(self.message.environment, "trusted_execution", None) else False
 
     @property
     def hypervisor(self) -> HypervisorType:

From 2414198abb508239984f36ddc5c59bedee01f037 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 21 Aug 2024 18:25:29 +0200
Subject: [PATCH 832/990] Problem: Websocked auth for fail user was not
 returning error (#675)

Due to a previous refactoring the code wasn't reachable
and thus the code only hang if the user was not the correct one

Solution: Code correction
---
 src/aleph/vm/orchestrator/views/operator.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 930053a33..48fe7d036 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -135,14 +135,16 @@ async def authenticate_websocket_for_vm_or_403(execution: VmExecution, vm_hash:
         if is_sender_authorized(authenticated_sender, execution.message):
             logger.debug(f"Accepted request to access logs by {authenticated_sender} on {vm_hash}")
             return True
-
-            logger.debug(f"Denied request to access logs by {authenticated_sender} on {vm_hash}")
-            await ws.send_json({"status": "failed", "reason": "unauthorized sender"})
-            raise web.HTTPForbidden(body="Unauthorized sender")
     except Exception as error:
+        # Error occurred (invalid auth packet or other
         await ws.send_json({"status": "failed", "reason": str(error)})
         raise web.HTTPForbidden(body="Unauthorized sender")
 
+    # Auth was valid but not the correct user
+    logger.debug(f"Denied request to access logs by {authenticated_sender} on {vm_hash}")
+    await ws.send_json({"status": "failed", "reason": "unauthorized sender"})
+    raise web.HTTPForbidden(body="Unauthorized sender")
+
 
 @cors_allow_all
 @require_jwk_authentication
@@ -175,7 +177,6 @@ async def operate_expire(request: web.Request, authenticated_sender: str) -> web
 @require_jwk_authentication
 async def operate_confidential_initialize(request: web.Request, authenticated_sender: str) -> web.Response:
     """Start the confidential virtual machine if possible."""
-    # TODO: Add user authentication
     vm_hash = get_itemhash_or_400(request.match_info)
 
     pool: VmPool = request.app["vm_pool"]
@@ -219,7 +220,6 @@ async def operate_confidential_initialize(request: web.Request, authenticated_se
 @require_jwk_authentication
 async def operate_stop(request: web.Request, authenticated_sender: str) -> web.Response:
     """Stop the virtual machine, smoothly if possible."""
-    # TODO: Add user authentication
     vm_hash = get_itemhash_or_400(request.match_info)
 
     pool: VmPool = request.app["vm_pool"]

From 0b4fbfddde51988cda36ff2fcb448867f2cbfdc9 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Thu, 22 Aug 2024 09:36:11 +0200
Subject: [PATCH 833/990] Solve failing tests on main branch (#678)

Fix: Solve failing test removing it because is not used.
---
 tests/supervisor/test_status.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/tests/supervisor/test_status.py b/tests/supervisor/test_status.py
index e3232f1bd..0e0449dbf 100644
--- a/tests/supervisor/test_status.py
+++ b/tests/supervisor/test_status.py
@@ -6,21 +6,6 @@
 from aleph.vm.orchestrator.status import check_internet
 
 
-@pytest.mark.asyncio
-async def test_check_internet_no_server_header():
-    vm_id = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")
-
-    mock_session = Mock()
-    mock_session.get = MagicMock()
-    mock_session.get.__aenter__.return_value.json = AsyncMock(return_value={"result": 200})
-
-    # A "Server" header is always expected in the result from the VM.
-    # If it is not present, the diagnostic VM is not working correctly
-    # and an error must be raised.
-    with pytest.raises(ValueError):
-        await check_internet(mock_session, vm_id)
-
-
 @pytest.mark.asyncio
 async def test_check_internet_wrong_result_code():
     vm_id = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")

From 6af63ce551008993a4b75c1d3871c03a3bc5f2f9 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Fri, 23 Aug 2024 13:22:08 +0200
Subject: [PATCH 834/990] Check message status before checking the payment
 (#679)

* Fix: Solve failing test removing it because is not used.

* Problem: If a user allocates a VM and later forgets the VM, the payment task fails because cannot get the price for that execution.

Solution: Check the message status before checking the price and remove the execution if it is forgotten or on a different status than `processed`.

* Fix: Solve code style issues.

* Fix: Explain the reason to use a direct API call instead using the connector.
---
 src/aleph/vm/orchestrator/messages.py | 20 +++++++++++++++++++-
 src/aleph/vm/orchestrator/tasks.py    | 12 +++++++++++-
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/messages.py b/src/aleph/vm/orchestrator/messages.py
index 3ed4f5cc2..5ae67102c 100644
--- a/src/aleph/vm/orchestrator/messages.py
+++ b/src/aleph/vm/orchestrator/messages.py
@@ -1,10 +1,12 @@
 import asyncio
 import copy
 
-from aiohttp import ClientConnectorError, ClientResponseError
+from aiohttp import ClientConnectorError, ClientResponseError, ClientSession
 from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable
 from aleph_message.models import ExecutableMessage, ItemHash, MessageType
+from aleph_message.status import MessageStatus
 
+from aleph.vm.conf import settings
 from aleph.vm.storage import get_latest_amend, get_message
 
 
@@ -69,3 +71,19 @@ async def load_updated_message(
     message = copy.deepcopy(original_message)
     await update_message(message)
     return message, original_message
+
+
+async def get_message_status(item_hash: ItemHash) -> MessageStatus:
+    """
+    Fetch the status of an execution from the reference API server.
+    We use a normal API call to the CCN instead to use the connector because we want to get the updated status of the
+    message and bypass the messages cache.
+    """
+    async with ClientSession() as session:
+        url = f"{settings.API_SERVER}/api/v0/messages/{item_hash}"
+        resp = await session.get(url)
+        # Raise an error if the request failed
+        resp.raise_for_status()
+
+        resp_data = await resp.json()
+        return resp_data["status"]
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 006de6e6b..fab864a6d 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -16,13 +16,14 @@
     ProgramMessage,
     parse_message,
 )
+from aleph_message.status import MessageStatus
 from yarl import URL
 
 from aleph.vm.conf import settings
 from aleph.vm.pool import VmPool
 from aleph.vm.utils import create_task_log_exceptions
 
-from .messages import load_updated_message
+from .messages import get_message_status, load_updated_message
 from .payment import (
     compute_required_balance,
     compute_required_flow,
@@ -148,6 +149,14 @@ async def monitor_payments(app: web.Application):
     while True:
         await asyncio.sleep(settings.PAYMENT_MONITOR_INTERVAL)
 
+        # Check if the executions continues existing or are forgotten before checking the payment
+        for vm_hash in pool.executions.keys():
+            message_status = await get_message_status(vm_hash)
+            if message_status != MessageStatus.PROCESSED:
+                logger.debug(f"Stopping {vm_hash} execution due to {message_status} message status")
+                await pool.stop_vm(vm_hash)
+                pool.forget_vm(vm_hash)
+
         # Check if the balance held in the wallet is sufficient holder tier resources (Not do it yet)
         for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.hold).items():
             for chain, executions in chains.items():
@@ -171,6 +180,7 @@ async def monitor_payments(app: web.Application):
                 logger.debug(
                     f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
                 )
+
                 required_stream = await compute_required_flow(executions)
                 logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
                 # Stop executions until the required stream is reached

From 4aa662b696e897aa8c015cfc99db16c9b8e825df Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Fri, 23 Aug 2024 13:27:21 +0200
Subject: [PATCH 835/990] Solve duplicated network issues for ephemeral VMs
 (#680)

* Fix: Solve failing test removing it because is not used.

* Problem: If the service restarts, the diagnostic VM fails for network issues.

Solution: Loading already loaded VMs filtering by only persistent ones.

* Fix: Replaced interface check by interface remove and re-creation.

* Fix: Ensure to delete the IPv6 address first before trying to delete the interface to prevent if the deletion fails.

* Fix: Also delete the IPv4 ip to prevent 2 interfaces with the same IPv4.

---------

Co-authored-by: Andres D. Molins 
---
 src/aleph/vm/network/interfaces.py | 15 +++++++++++++++
 src/aleph/vm/pool.py               | 13 +++++--------
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index f51e5a732..b3fc14029 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -61,6 +61,19 @@ def add_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6
         logger.error(f"Unknown exception while adding address {ip} to interface {device_name}: {e}")
 
 
+def delete_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6Interface]):
+    """Delete an IP address to the given interface."""
+    interface_index: list[int] = ipr.link_lookup(ifname=device_name)
+    if not interface_index:
+        raise MissingInterfaceError(f"Interface {device_name} does not exist, can't delete address {ip} to it.")
+    try:
+        ipr.addr("del", index=interface_index[0], address=str(ip.ip), mask=ip.network.prefixlen)
+    except NetlinkError as e:
+        logger.error(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
+    except OSError as e:
+        logger.error(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
+
+
 def set_link_up(ipr: IPRoute, device_name: str):
     """Set the given interface up."""
     interface_index: list[int] = ipr.link_lookup(ifname=device_name)
@@ -154,4 +167,6 @@ async def delete(self) -> None:
         if self.ndp_proxy:
             await self.ndp_proxy.delete_range(self.device_name)
         with IPRoute() as ipr:
+            delete_ip_address(ipr, self.device_name, self.host_ip)
+            delete_ip_address(ipr, self.device_name, self.host_ipv6)
             delete_tap_interface(ipr, self.device_name)
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 8c5f9fd8b..48e9c6b8c 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -115,6 +115,9 @@ async def create_a_vm(
                 if self.network:
                     vm_type = VmType.from_message_content(message)
                     tap_interface = await self.network.prepare_tap(vm_id, vm_hash, vm_type)
+                    # If the network interface already exists, remove it and then re-create it.
+                    if self.network.interface_exists(vm_id):
+                        await tap_interface.delete()
                     await self.network.create_tap(vm_id, tap_interface)
                 else:
                     tap_interface = None
@@ -163,12 +166,6 @@ def get_unique_vm_id(self) -> int:
             # anymore.
             currently_used_vm_ids = {execution.vm_id for execution in self.executions.values()}
             for i in range(settings.START_ID_INDEX, 255**2):
-
-                if self.network:
-                    # Check the network interface don't already exists, otherwise it will cause a crash
-                    if self.network.interface_exists(i):
-                        continue
-
                 if i not in currently_used_vm_ids:
                     return i
             else:
@@ -229,8 +226,8 @@ async def load_persistent_executions(self):
         for saved_execution in saved_executions:
             vm_hash = ItemHash(saved_execution.vm_hash)
 
-            if vm_hash in self.executions:
-                # The execution is already loaded, skip it
+            if vm_hash in self.executions or not saved_execution.persistent:
+                # The execution is already loaded or isn't persistent, skip it
                 continue
 
             vm_id = saved_execution.vm_id

From cd6463cd9a200e34f477ccaaaaa94d8a4520e085 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Mon, 26 Aug 2024 14:55:13 +0200
Subject: [PATCH 836/990] Fix: Update new `aleph-message` package version.
 (#683)

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index ede2589b2..9b1fd06ac 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,7 +29,7 @@ dependencies = [
   "aiodns==3.1.0",
   "setproctitle==1.3.3",
   "pyyaml==6.0.1",
-  "aleph-message==0.4.7",
+  "aleph-message==0.4.9",
   "eth-account~=0.10",
   "sentry-sdk==1.31.0",
   "aioredis==1.3.1",

From d66de42767e5ee70092c9354a55bd4ad9117fb65 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Tue, 27 Aug 2024 17:24:52 +0200
Subject: [PATCH 837/990] Update `aleph_message` package on packaging steps
 (#684)

Fix: Update new `aleph-message` package version on packaging steps.
---
 docker/vm_supervisor-dev.dockerfile                  | 2 +-
 examples/volumes/Dockerfile                          | 2 +-
 packaging/Makefile                                   | 2 +-
 runtimes/aleph-debian-12-python/create_disk_image.sh | 2 +-
 src/aleph/vm/orchestrator/README.md                  | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 77718f3da..14514f569 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -19,7 +19,7 @@ RUN curl -fsSL -o /opt/firecracker/vmlinux.bin https://s3.amazonaws.com/spec.ccf
 RUN ln /opt/firecracker/release-*/firecracker-v* /opt/firecracker/firecracker
 RUN ln /opt/firecracker/release-*/jailer-v* /opt/firecracker/jailer
 
-RUN pip3 install typing-extensions 'aleph-message==0.4.7'
+RUN pip3 install typing-extensions 'aleph-message==0.4.9'
 
 RUN mkdir -p /var/lib/aleph/vm/jailer
 
diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile
index f3aad1e18..8f57a0622 100644
--- a/examples/volumes/Dockerfile
+++ b/examples/volumes/Dockerfile
@@ -6,6 +6,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 RUN python3 -m venv /opt/venv
-RUN /opt/venv/bin/pip install 'aleph-message==0.4.7'
+RUN /opt/venv/bin/pip install 'aleph-message==0.4.9'
 
 CMD mksquashfs /opt/venv /mnt/volume-venv.squashfs
diff --git a/packaging/Makefile b/packaging/Makefile
index 71a96d579..73cc23289 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.7' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo  target/bin/sevctl
diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index 859d678d5..10c8ae404 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -36,7 +36,7 @@ locale-gen en_US.UTF-8
 
 echo "Pip installing aleph-sdk-python"
 mkdir -p /opt/aleph/libs
-pip3 install --target /opt/aleph/libs 'aleph-sdk-python==0.9.0' 'aleph-message==0.4.7' 'fastapi~=0.109.2'
+pip3 install --target /opt/aleph/libs 'aleph-sdk-python==1.0.0' 'aleph-message==0.4.9' 'fastapi~=0.109.2'
 
 # Compile Python code to bytecode for faster execution
 # -o2 is needed to compile with optimization level 2 which is what we launch init1.py ("python -OO")
diff --git a/src/aleph/vm/orchestrator/README.md b/src/aleph/vm/orchestrator/README.md
index 95430423a..c1d22ea0f 100644
--- a/src/aleph/vm/orchestrator/README.md
+++ b/src/aleph/vm/orchestrator/README.md
@@ -86,7 +86,7 @@ is used to parse and validate Aleph messages.
 ```shell
 apt install -y --no-install-recommends --no-install-suggests python3-pip
 pip3 install pydantic[dotenv]
-pip3 install 'aleph-message==0.4.7'
+pip3 install 'aleph-message==0.4.9'
 ```
 
 ### 2.f. Create the jailer working directory:

From b74d05ffa90eee504cdcd86cf5f9c515146e0387 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 28 Aug 2024 09:41:42 +0200
Subject: [PATCH 838/990] Provide a template for new PRs (#667)

---
 .github/PULL_REQUEST_TEMPLATE.md | 33 ++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 .github/PULL_REQUEST_TEMPLATE.md

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 000000000..bcf764608
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,33 @@
+Explain what problem this PR is resolving
+
+Related ClickUp, GitHub or Jira tickets : ALEPH-XXX
+
+## Self proofreading checklist
+
+- [ ] The new code clear, easy to read and well commented.
+- [ ] New code does not duplicate the functions of builtin or popular libraries.
+- [ ] An LLM was used to review the new code and look for simplifications.
+- [ ] New classes and functions contain docstrings explaining what they provide.
+- [ ] All new code is covered by relevant tests.
+- [ ] Documentation has been updated regarding these changes.
+
+## Changes
+
+Explain the changes that were made. The idea is not to list exhaustively all the changes made (GitHub already provides a full diff), but to help the reviewers better understand:
+- which specific file changes go together, e.g: when creating a table in the front-end, there usually is a config file that goes with it
+- the reasoning behind some changes, e.g: deleted files because they are now redundant
+- the behaviour to expect, e.g: tooltip has purple background color because the client likes it so, changed a key in the API response to be consistent with other endpoints
+
+## How to test
+
+Explain how to test your PR.
+If a specific config is required explain it here (account, data entry, ...)
+
+## Print screen / video
+
+Upload here screenshots or videos showing the changes if relevant.
+
+## Notes
+
+Things that the reviewers should know: known bugs that are out of the scope of the PR, other trade-offs that were made.
+If the PR depends on a PR in another repo, or merges into another PR (i.o. main), it should also be mentioned here

From 46063fbf0019e9fe49ece5bda9ae9276f224821a Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 2 Sep 2024 12:16:52 +0200
Subject: [PATCH 839/990] Fix CI slowness and correct  execution tests (#686)

* Problem: Execution tests were very slow

Solution: This was due to an import in the test app that is somehow
very slow but only during testing.

Haven't figured out why it is slow, but have implemented a workaround
that delay the import so it's not hit during the tests

* Fix 'real' executions test were testing the fake VM

This was due to as settings contamination which made it runn the FAKE_DATA_PROGRAM instead of the real one

Also correct some things that made the test not run (load_update_mesage
instead of get_message)

* Correct the Workflow name

It was the same name as an other workflow which caused issue in github

* Execution test were failing on Python 3.12

Due to change in behaviour of unix_socket.wait_closed

* Symlink don't work so make a copy instead

* add vm-connector in test runner

* Increase timeout for ci

* Update comment src/aleph/vm/hypervisors/firecracker/microvm.py


Co-authored-by: Hugo Herter 
---
 .github/workflows/test-using-pytest.yml       |  8 ++++-
 examples/example_fastapi/main.py              |  6 +++-
 runtimes/aleph-debian-11-python/init1.py      |  1 +
 .../vm/hypervisors/firecracker/microvm.py     | 18 ++++++++--
 src/aleph/vm/storage.py                       |  1 +
 tests/supervisor/test_execution.py            | 36 ++++++++++++-------
 6 files changed, 54 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/test-using-pytest.yml b/.github/workflows/test-using-pytest.yml
index 3b8cb97a8..2817922ef 100644
--- a/.github/workflows/test-using-pytest.yml
+++ b/.github/workflows/test-using-pytest.yml
@@ -1,4 +1,4 @@
-name: "Test on DigitalOcean Droplets"
+name: "py.test and linting"
 
 on:
   push
@@ -7,6 +7,12 @@ jobs:
   tests-python:
     name: "Test Python code"
     runs-on: ubuntu-22.04
+    services:
+      # Run vm connector for the execution tests
+      vm-connector:
+        image: alephim/vm-connector:alpha
+        ports:
+          - 4021:4021
 
     steps:
       - uses: actions/checkout@v4
diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py
index 81055c723..44caaf458 100644
--- a/examples/example_fastapi/main.py
+++ b/examples/example_fastapi/main.py
@@ -25,7 +25,6 @@
 from pydantic import BaseModel, HttpUrl
 from starlette.responses import JSONResponse
 
-from aleph.sdk.chains.ethereum import get_fallback_account
 from aleph.sdk.chains.remote import RemoteAccount
 from aleph.sdk.client import AlephHttpClient, AuthenticatedAlephHttpClient
 from aleph.sdk.query.filters import MessageFilter
@@ -292,6 +291,7 @@ async def post_with_remote_account():
 @app.post("/post_a_message_local_account")
 async def post_with_local_account():
     """Post a message on the Aleph.im network using a local private key."""
+    from aleph.sdk.chains.ethereum import get_fallback_account
 
     account = get_fallback_account()
 
@@ -326,6 +326,8 @@ async def post_with_local_account():
 
 @app.post("/post_a_file")
 async def post_a_file():
+    from aleph.sdk.chains.ethereum import get_fallback_account
+
     account = get_fallback_account()
     file_path = Path(__file__).absolute()
     async with AuthenticatedAlephHttpClient(
@@ -351,6 +353,8 @@ async def post_a_file():
 async def sign_a_message():
     """Sign a message using a locally managed account within the virtual machine."""
     # FIXME: Broken, fixing this depends on https://github.com/aleph-im/aleph-sdk-python/pull/120
+    from aleph.sdk.chains.ethereum import get_fallback_account
+
     account = get_fallback_account()
     message = {"hello": "world", "chain": "ETH"}
     signed_message = await account.sign_message(message)
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
index f41128a8b..11c4a7dd0 100644
--- a/runtimes/aleph-debian-11-python/init1.py
+++ b/runtimes/aleph-debian-11-python/init1.py
@@ -247,6 +247,7 @@ async def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> A
         module = __import__(module_name)
         for level in module_name.split(".")[1:]:
             module = getattr(module, level)
+        logger.debug("import done")
         app = getattr(module, app_name)
     elif encoding == Encoding.plain:
         # Execute the code and extract the entrypoint
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index e5a7c94dc..d85b80071 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -1,4 +1,5 @@
 import asyncio
+import errno
 import json
 import logging
 import os.path
@@ -318,7 +319,8 @@ def enable_rootfs(self, path_on_host: Path) -> Path:
     def enable_file_rootfs(self, path_on_host: Path) -> Path:
         """Make a rootfs available to the VM.
 
-        Creates a symlink to the rootfs file if jailer is in use.
+        If jailer is in use, try to create a hardlink
+        If it is not possible to create a link because the dir are in separate device made a copy.
         """
         if self.use_jailer:
             rootfs_filename = Path(path_on_host).name
@@ -327,6 +329,13 @@ def enable_file_rootfs(self, path_on_host: Path) -> Path:
                 os.link(path_on_host, f"{self.jailer_path}/{jailer_path_on_host}")
             except FileExistsError:
                 logger.debug(f"File {jailer_path_on_host} already exists")
+            except OSError as err:
+                if err.errno == errno.EXDEV:
+                    # Invalid cross-device link: cannot make hard link between partition.
+                    # In this case, copy the file instead:
+                    shutil.copyfile(path_on_host, f"{self.jailer_path}/{jailer_path_on_host}")
+                else:
+                    raise
             return Path(jailer_path_on_host)
         else:
             return path_on_host
@@ -489,7 +498,12 @@ async def teardown(self):
         if self._unix_socket:
             logger.debug("Closing unix socket")
             self._unix_socket.close()
-            await self._unix_socket.wait_closed()
+            try:
+                await asyncio.wait_for(self._unix_socket.wait_closed(), 2)
+            except asyncio.TimeoutError:
+                # In  Python < 3.11 wait_closed() was broken and returned immediatly
+                # It is supposedly fixed in Python 3.12.1, but it hangs indefinitely during tests.
+                logger.info("f{self} unix socket closing timeout")
 
         logger.debug("Removing files")
         if self.config_file_path:
diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index eb652b02b..239a71586 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -148,6 +148,7 @@ async def get_message(ref: str) -> Union[ProgramMessage, InstanceMessage]:
         cache_path = settings.FAKE_INSTANCE_MESSAGE
     elif settings.FAKE_DATA_PROGRAM:
         cache_path = settings.FAKE_DATA_MESSAGE
+        logger.debug("Using the fake data message")
     else:
         cache_path = (Path(settings.MESSAGE_CACHE) / ref).with_suffix(".json")
         url = f"{settings.CONNECTOR_URL}/download/message/{ref}"
diff --git a/tests/supervisor/test_execution.py b/tests/supervisor/test_execution.py
index afaa82ce7..7f64b5a8f 100644
--- a/tests/supervisor/test_execution.py
+++ b/tests/supervisor/test_execution.py
@@ -4,29 +4,35 @@
 import pytest
 from aleph_message.models import ItemHash
 
-from aleph.vm.conf import settings
+from aleph.vm.conf import Settings, settings
 from aleph.vm.controllers.firecracker import AlephFirecrackerProgram
 from aleph.vm.models import VmExecution
 from aleph.vm.orchestrator import metrics
+from aleph.vm.orchestrator.messages import load_updated_message
 from aleph.vm.storage import get_message
 
 
 @pytest.mark.asyncio
-async def test_create_execution():
+async def test_create_execution(mocker):
     """
     Create a new VM execution and check that it starts properly.
     """
+    mock_settings = Settings()
+    mocker.patch("aleph.vm.conf.settings", new=mock_settings)
+    mocker.patch("aleph.vm.storage.settings", new=mock_settings)
+    mocker.patch("aleph.vm.controllers.firecracker.executable.settings", new=mock_settings)
+    mocker.patch("aleph.vm.controllers.firecracker.program.settings", new=mock_settings)
 
-    settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM
-    settings.ALLOW_VM_NETWORKING = False
-    settings.USE_JAILER = False
+    mock_settings.FAKE_DATA_PROGRAM = mock_settings.BENCHMARK_FAKE_DATA_PROGRAM
+    mock_settings.ALLOW_VM_NETWORKING = False
+    mock_settings.USE_JAILER = False
 
     logging.basicConfig(level=logging.DEBUG)
-    settings.PRINT_SYSTEM_LOGS = True
+    mock_settings.PRINT_SYSTEM_LOGS = True
 
     # Ensure that the settings are correct and required files present.
-    settings.setup()
-    settings.check()
+    mock_settings.setup()
+    mock_settings.check()
 
     # The database is required for the metrics and is currently not optional.
     engine = metrics.setup_engine()
@@ -57,6 +63,7 @@ async def test_create_execution():
     await execution.stop()
 
 
+# This test depends on having a vm-connector running on port 4021
 @pytest.mark.asyncio
 async def test_create_execution_online(vm_hash: ItemHash = None):
     """
@@ -73,29 +80,34 @@ async def test_create_execution_online(vm_hash: ItemHash = None):
     engine = metrics.setup_engine()
     await metrics.create_tables(engine)
 
-    message = await get_message(ref=vm_hash)
+    message, original_message = await load_updated_message(vm_hash)
 
     execution = VmExecution(
         vm_hash=vm_hash,
         message=message.content,
-        original=message.content,
+        original=original_message.content,
         snapshot_manager=None,
         systemd_manager=None,
         persistent=False,
     )
 
-    # Downloading the resources required may take some time, limit it to 10 seconds
-    await asyncio.wait_for(execution.prepare(), timeout=30)
+    # Downloading the resources required may take some time, limit it to 120 seconds
+    # since it is a bit slow in GitHub Actions
+    await asyncio.wait_for(execution.prepare(), timeout=120)
 
     vm = execution.create(vm_id=3, tap_interface=None)
+
     # Test that the VM is created correctly. It is not started yet.
     assert isinstance(vm, AlephFirecrackerProgram)
+    vm.enable_console = True
+    vm.fvm.enable_log = True
     assert vm.vm_id == 3
 
     await execution.start()
     await execution.stop()
 
 
+# This test depends on having a vm-connector running on port 4021
 @pytest.mark.asyncio
 async def test_create_execution_legacy():
     """

From 91d60278d48465a14b8c210b407e4a3436c4a19a Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 3 Sep 2024 16:17:55 +0200
Subject: [PATCH 840/990] Problem: ping: invalid value (`2.0' near `.0') (#688)

Symptoms:
Could not allocate a VM on some ubuntu server because the
wait_for_init/ping
was failing

```
2024-09-03 12:18:47,259 | DEBUG | command: ping -c 1 -W 2.0 172.16.4.2
2024-09-03 12:18:47,259 | ERROR | Command failed with error code 1:
    stdin = None
    command = ['ping', '-c', '1', '-W', '2.0', '172.16.4.2']
    stdout = b"ping: invalid value (`2.0' near `.0')\n"
2024-09-03 12:18:47,260 | ERROR |
Traceback (most recent call last):
  File "/home/olivier/pycharm/aleph-vm/src/aleph/vm/utils/__init__.py", line 186, in ping
    await run_in_subprocess(["ping", "-c", str(packets), "-W", str(timeout), host], check=True)
  File "/home/olivier/pycharm/aleph-vm/src/aleph/vm/utils/__init__.py", line 121, in run_in_subprocess
    raise subprocess.CalledProcessError(process.returncode, str(command), stderr.decode())
subprocess.CalledProcessError: Command '['ping', '-c', '1', '-W', '2.0', '172.16.4.2']' returned non-zero exit status 1.
```

Causes:
The root cause seems to be that the ping command from the  deb package
inetutils-ping 2.5-3ubuntu4 doesn't accept a float for it's -W argument

While the ping command  from the package 'iputils-ping' which we use on
other server accept it.

Solution:
Convert the argument to a int since we didn't use the float part
This allow compatibility with both version of the binary
---
 src/aleph/vm/controllers/firecracker/instance.py | 2 +-
 src/aleph/vm/controllers/qemu/instance.py        | 2 +-
 src/aleph/vm/utils/__init__.py                   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index 4ab5711cf..4f936b7bb 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -133,7 +133,7 @@ async def wait_for_init(self) -> None:
         ip = ip.split("/", 1)[0]
 
         attempts = 30
-        timeout_seconds = 2.0
+        timeout_seconds = 2
 
         for attempt in range(attempts):
             try:
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index be4e2def2..3f4f5ba3d 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -235,7 +235,7 @@ async def wait_for_init(self) -> None:
         ip = ip.split("/", 1)[0]
 
         attempts = 30
-        timeout_seconds = 2.0
+        timeout_seconds = 2
 
         for attempt in range(attempts):
             try:
diff --git a/src/aleph/vm/utils/__init__.py b/src/aleph/vm/utils/__init__.py
index d96b519f1..30b7ce90e 100644
--- a/src/aleph/vm/utils/__init__.py
+++ b/src/aleph/vm/utils/__init__.py
@@ -177,7 +177,7 @@ class HostNotFoundError(Exception):
     pass
 
 
-async def ping(host: str, packets: int, timeout: float):
+async def ping(host: str, packets: int, timeout: int):
     """
     Waits for a host to respond to a ping request.
     """

From 02affa3e3e9c59c5ec6d49d7ff9383663a36227e Mon Sep 17 00:00:00 2001
From: 1yam <40899431+1yam@users.noreply.github.com>
Date: Tue, 3 Sep 2024 17:02:15 +0200
Subject: [PATCH 841/990] Feature: Allow PAYG on base (#685)

INFO: the settings `PAYMENT_RPC_API` has been renamed to `RPC_AVAX`

Problem:
Base chain isn't supported.

Solutions:
adding src/aleph/vm/orchestrator/chain.py to store Available Chains
Display available_payments in status_public_config
Adding checks that the chains sent is in the STREAM_CHAINS
Fix: use chain_info.super_token instead of settings.PAYMENT_SUPER_TOKEN
Update dependency superfluid to aleph-superfluid==0.2.1
Fix: wrong logic in monitor_payments for payg

Co-authored-by: nesitor 
Co-authored-by: Olivier Le Thanh Duong 
---
 packaging/Makefile                          |  2 +-
 pyproject.toml                              |  2 +-
 src/aleph/vm/conf.py                        | 30 +++++----
 src/aleph/vm/orchestrator/chain.py          | 67 +++++++++++++++++++++
 src/aleph/vm/orchestrator/payment.py        | 19 ++++--
 src/aleph/vm/orchestrator/tasks.py          |  2 +-
 src/aleph/vm/orchestrator/views/__init__.py | 13 +++-
 7 files changed, 112 insertions(+), 23 deletions(-)
 create mode 100644 src/aleph/vm/orchestrator/chain.py

diff --git a/packaging/Makefile b/packaging/Makefile
index 73cc23289..7e4a395cc 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'superfluid==0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo  target/bin/sevctl
diff --git a/pyproject.toml b/pyproject.toml
index 9b1fd06ac..aa74d1de7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -44,7 +44,7 @@ dependencies = [
   "dbus-python==1.3.2",
   "systemd-python==235",
   "systemd-python==235",
-  "superfluid~=0.2.1",
+  "aleph-superfluid~=0.2.1",
   "sqlalchemy[asyncio]>=2.0",
   "aiosqlite==0.19.0",
   "alembic==1.13.1",
diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 98e3772e9..3ef96127b 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -11,11 +11,13 @@
 from subprocess import CalledProcessError, check_output
 from typing import Any, Literal, NewType, Optional, Union
 
+from aleph_message.models import Chain
 from aleph_message.models.execution.environment import HypervisorType
 from pydantic import BaseSettings, Field, HttpUrl
 from pydantic.env_settings import DotenvType, env_file_sentinel
 from pydantic.typing import StrPath
 
+from aleph.vm.orchestrator.chain import STREAM_CHAINS, ChainInfo
 from aleph.vm.utils import (
     check_amd_sev_es_supported,
     check_amd_sev_supported,
@@ -224,22 +226,17 @@ class Settings(BaseSettings):
         description="Address of the account receiving payments",
     )
     # This address is the ALEPH SuperToken on SuperFluid Testnet
-    PAYMENT_SUPER_TOKEN: str = Field(
-        default="0xc0Fbc4967259786C743361a5885ef49380473dCF",  # Mainnet
-        # default="0x1290248e01ed2f9f863a9752a8aad396ef3a1b00",  # Testnet
-        description="Address of the ALEPH SuperToken on SuperFluid",
-    )
     PAYMENT_PRICING_AGGREGATE: str = ""  # TODO: Missing
 
-    PAYMENT_RPC_API: HttpUrl = Field(
-        default="https://api.avax.network/ext/bc/C/rpc",
-        # default="https://api.avax-test.network/ext/bc/C/rpc",
-        description="Default to Avalanche Testnet RPC",
+    # Use to check PAYG payment
+    RPC_AVAX: HttpUrl = Field(
+        default=STREAM_CHAINS[Chain.AVAX].rpc,
+        description="RPC API Endpoint for AVAX chain",
     )
-    PAYMENT_CHAIN_ID: int = Field(
-        default=43114,  # Avalanche Mainnet
-        # default=43113,  # Avalanche Fuji Testnet
-        description="Avalanche chain ID",
+
+    RPC_BASE: HttpUrl = Field(
+        default=STREAM_CHAINS[Chain.BASE].rpc,
+        description="RPC API Endpoint for BASE chain",
     )
 
     PAYMENT_BUFFER: Decimal = Field(
@@ -401,6 +398,13 @@ def check(self):
 
     def setup(self):
         """Setup the environment defined by the settings. Call this method after loading the settings."""
+
+        # Update chain RPC
+        STREAM_CHAINS[Chain.AVAX].rpc = str(self.RPC_AVAX)
+        STREAM_CHAINS[Chain.BASE].rpc = str(self.RPC_BASE)
+
+        logger.info(STREAM_CHAINS)
+
         os.makedirs(self.MESSAGE_CACHE, exist_ok=True)
         os.makedirs(self.CODE_CACHE, exist_ok=True)
         os.makedirs(self.RUNTIME_CACHE, exist_ok=True)
diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py
new file mode 100644
index 000000000..2cedd8162
--- /dev/null
+++ b/src/aleph/vm/orchestrator/chain.py
@@ -0,0 +1,67 @@
+import logging
+from typing import Dict, Optional, Union
+
+from aleph_message.models import Chain
+from pydantic import BaseModel, root_validator
+
+logger = logging.getLogger(__name__)
+
+
+class ChainInfo(BaseModel):
+    """
+    A chain information.
+    """
+
+    chain_id: int
+    rpc: str
+    standard_token: Optional[str] = None
+    super_token: Optional[str] = None
+    testnet: bool = False
+    active: bool = True
+
+    @property
+    def token(self) -> Optional[str]:
+        return self.super_token or self.standard_token
+
+    @root_validator(pre=True)
+    def check_tokens(cls, values):
+        if not values.get("standard_token") and not values.get("super_token"):
+            raise ValueError("At least one of standard_token or super_token must be provided.")
+        return values
+
+
+STREAM_CHAINS: Dict[Union[Chain, str], ChainInfo] = {
+    # TESTNETS
+    "SEPOLIA": ChainInfo(
+        chain_id=11155111,
+        rpc="https://eth-sepolia.public.blastapi.io",
+        standard_token="0xc4bf5cbdabe595361438f8c6a187bdc330539c60",
+        super_token="0x22064a21fee226d8ffb8818e7627d5ff6d0fc33a",
+        active=False,
+        testnet=True,
+    ),
+    # MAINNETS
+    Chain.ETH: ChainInfo(
+        chain_id=1,
+        rpc="https://eth-mainnet.public.blastapi.io",
+        standard_token="0x27702a26126e0B3702af63Ee09aC4d1A084EF628",
+        active=False,
+    ),
+    Chain.AVAX: ChainInfo(
+        chain_id=43114,
+        rpc="https://api.avax.network/ext/bc/C/rpc",
+        super_token="0xc0Fbc4967259786C743361a5885ef49380473dCF",
+    ),
+    Chain.BASE: ChainInfo(
+        chain_id=8453,
+        rpc="https://base-mainnet.public.blastapi.io",
+        super_token="0xc0Fbc4967259786C743361a5885ef49380473dCF",
+    ),
+}
+
+
+def get_chain(chain: str) -> ChainInfo:
+    try:
+        return STREAM_CHAINS[chain]
+    except KeyError as error:
+        raise ValueError(f"Unknown chain id for chain {chain}")
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 65e642e01..420754ecc 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -14,6 +14,8 @@
 from aleph.vm.models import VmExecution
 from aleph.vm.utils import to_normalized_address
 
+from .chain import ChainInfo, get_chain
+
 logger = logging.getLogger(__name__)
 
 
@@ -87,18 +89,25 @@ class InvalidAddressError(ValueError):
     pass
 
 
-async def get_stream(sender: str, receiver: str, chain) -> Decimal:
+class InvalidChainError(ValueError):
+    pass
+
+
+async def get_stream(sender: str, receiver: str, chain: str) -> Decimal:
     """
     Get the stream of the user from the Superfluid API.
     See https://community.aleph.im/t/pay-as-you-go-using-superfluid/98/11
     """
-    chain_id = settings.PAYMENT_CHAIN_ID
-    superfluid_instance = CFA_V1(settings.PAYMENT_RPC_API, chain_id)
+    chain_info: ChainInfo = get_chain(chain=chain)
+    if not chain_info.active:
+        raise InvalidChainError(f"Chain : {chain} is not active for superfluid")
+
+    superfluid_instance = CFA_V1(chain_info.rpc, chain_info.chain_id)
 
     try:
-        super_token: HexAddress = to_normalized_address(settings.PAYMENT_SUPER_TOKEN)
+        super_token: HexAddress = to_normalized_address(chain_info.super_token)
     except ValueError as error:
-        raise InvalidAddressError(f"Invalid token address '{settings.PAYMENT_SUPER_TOKEN}' - {error.args}") from error
+        raise InvalidAddressError(f"Invalid token address '{chain_info.super_token}' - {error.args}") from error
 
     try:
         sender_address: HexAddress = to_normalized_address(sender)
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index fab864a6d..3f468785d 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -184,7 +184,7 @@ async def monitor_payments(app: web.Application):
                 required_stream = await compute_required_flow(executions)
                 logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
                 # Stop executions until the required stream is reached
-                while stream < (required_stream + settings.PAYMENT_BUFFER):
+                while (stream + settings.PAYMENT_BUFFER) < required_stream:
                     try:
                         last_execution = executions.pop(-1)
                     except IndexError:  # Empty list
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 614314461..c99b0e38c 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -24,10 +24,12 @@
 from aleph.vm.controllers.firecracker.program import FileTooLargeError
 from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInitError
 from aleph.vm.orchestrator import payment, status
+from aleph.vm.orchestrator.chain import STREAM_CHAINS, ChainInfo
 from aleph.vm.orchestrator.messages import try_get_message
 from aleph.vm.orchestrator.metrics import get_execution_records
 from aleph.vm.orchestrator.payment import (
     InvalidAddressError,
+    InvalidChainError,
     fetch_execution_flow_price,
     get_stream,
 )
@@ -299,6 +301,11 @@ async def status_check_version(request: web.Request):
 @cors_allow_all
 async def status_public_config(request: web.Request):
     """Expose the public fields from the configuration"""
+
+    available_payments = {
+        str(chain_name): chain_info for chain_name, chain_info in STREAM_CHAINS.items() if chain_info.active
+    }
+
     return web.json_response(
         {
             "DOMAIN_NAME": settings.DOMAIN_NAME,
@@ -329,8 +336,7 @@ async def status_public_config(request: web.Request):
             },
             "payment": {
                 "PAYMENT_RECEIVER_ADDRESS": settings.PAYMENT_RECEIVER_ADDRESS,
-                "PAYMENT_SUPER_TOKEN": settings.PAYMENT_SUPER_TOKEN,
-                "PAYMENT_CHAIN_ID": settings.PAYMENT_CHAIN_ID,
+                "AVAILABLE_PAYMENTS": available_payments,
                 "PAYMENT_MONITOR_INTERVAL": settings.PAYMENT_MONITOR_INTERVAL,
             },
             "computing": {
@@ -494,6 +500,9 @@ async def notify_allocation(request: web.Request):
         except InvalidAddressError as error:
             logger.warning(f"Invalid address {error}", exc_info=True)
             return web.HTTPBadRequest(reason=f"Invalid address {error}")
+        except InvalidChainError as error:
+            logger.warning(f"Invalid chain {error}", exc_info=True)
+            return web.HTTPBadRequest(reason=f"Invalid Chain {error}")
 
         if not active_flow:
             raise web.HTTPPaymentRequired(reason="Empty payment stream for this instance")

From 1c888e21cea275290e6d965e076378e132a0f90b Mon Sep 17 00:00:00 2001
From: Laurent Peuch 
Date: Tue, 3 Sep 2024 20:51:15 +0200
Subject: [PATCH 842/990] ci/fix(test-using-pytest): ensure hatch is always
 installed when needed

---
 .github/workflows/test-using-pytest.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-using-pytest.yml b/.github/workflows/test-using-pytest.yml
index 2817922ef..0ca8ec032 100644
--- a/.github/workflows/test-using-pytest.yml
+++ b/.github/workflows/test-using-pytest.yml
@@ -72,16 +72,19 @@ jobs:
       - name: "Build example volume"
         run: |
           cd examples/volumes && bash build_squashfs.sh
-        
+
       # Unit tests create and delete network interfaces, and therefore require to run as root
       - name: Run unit tests
         run: |
           sudo python3 -m pip install hatch hatch-vcs coverage
           sudo hatch run testing:cov
+
       - name: Output modules used and their version
         if:  always()
         run: |
-           sudo hatch -e testing run pip freeze
+          # re-install hatch in case previous job failed and hatch didn't get installed
+          sudo python3 -m pip install hatch hatch-vcs coverage
+          sudo hatch -e testing run pip freeze
 
 
       - name: Upload coverage reports to Codecov
@@ -107,4 +110,4 @@ jobs:
       - name: Run Shellcheck on all shell scripts
         run: |
           find ./ -type f -name "*.sh" -exec shellcheck {} \;
-  
+

From fbbbf3dac32f14ead98007005288c93cb1f31afe Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 4 Sep 2024 18:08:42 +0200
Subject: [PATCH 843/990] Fix: Debian 11 Bullseye is deprecated, remove it

Debian 11 provided Python 3.9.

This branch removes the support for both Debian 11 and Python 3.9.
The oldest distribution supported is now Ubuntu 22.04 with Python 3.10.

That that mentions of Debian 11 were replaced in
some example files that were not maintained and
the change has not been tested. These remain to
serve as examples for developers.
---
 .github/workflows/build-deb-package.yml       |   9 +-
 .github/workflows/test-on-droplets-matrix.yml |   7 -
 docker/vm_supervisor-dev.dockerfile           |   2 +-
 examples/example_http_js/Dockerfile           |   2 +-
 examples/example_http_rust/Dockerfile         |   2 +-
 examples/volumes/Dockerfile                   |   2 +-
 packaging/Makefile                            |  35 +-
 packaging/debian-11.dockerfile                |  16 -
 .../{bullseye => bookworm}/conf/distributions |   2 +-
 pyproject.toml                                |   2 +-
 .../create_disk_image.sh                      | 106 ---
 runtimes/aleph-debian-11-python/init0.sh      |  56 --
 runtimes/aleph-debian-11-python/init1.py      | 623 -----------------
 runtimes/aleph-debian-11-python/loading.html  | 346 ----------
 .../aleph-debian-11-python/update_inits.sh    |  14 -
 runtimes/aleph-debian-12-python/init1.py      | 624 +++++++++++++++++-
 runtimes/aleph-debian-12-python/loading.html  | 347 +++++++++-
 .../aleph-debian-12-python/update_inits.sh    |  15 +-
 .../instance-rootfs/create-debian-11-disk.sh  |  55 --
 .../create-debian-11-qemu-disk.sh             |  18 -
 20 files changed, 1001 insertions(+), 1282 deletions(-)
 delete mode 100644 packaging/debian-11.dockerfile
 rename packaging/repositories/{bullseye => bookworm}/conf/distributions (92%)
 delete mode 100755 runtimes/aleph-debian-11-python/create_disk_image.sh
 delete mode 100644 runtimes/aleph-debian-11-python/init0.sh
 delete mode 100644 runtimes/aleph-debian-11-python/init1.py
 delete mode 100644 runtimes/aleph-debian-11-python/loading.html
 delete mode 100755 runtimes/aleph-debian-11-python/update_inits.sh
 mode change 120000 => 100644 runtimes/aleph-debian-12-python/init1.py
 mode change 120000 => 100644 runtimes/aleph-debian-12-python/loading.html
 mode change 120000 => 100755 runtimes/aleph-debian-12-python/update_inits.sh
 delete mode 100755 runtimes/instance-rootfs/create-debian-11-disk.sh
 delete mode 100755 runtimes/instance-rootfs/create-debian-11-qemu-disk.sh

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index ca4776b8b..544b523d5 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -9,11 +9,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["debian-11", "debian-12", "ubuntu-22.04"]
+        os: ["debian-12", "ubuntu-22.04", "ubuntu-24.04"]
         include:
-          - os: "debian-11"
-            make_target: "all-podman-debian-11"
-            artifact_name: "aleph-vm.debian-11.deb"
           - os: "debian-12"
             make_target: "all-podman-debian-12"
             artifact_name: "aleph-vm.debian-12.deb"
@@ -56,10 +53,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        os: ["debian-11", "debian-12"]
+        os: ["debian-12"]
         include:
-          - os: "debian-11"
-            artifact_name: "aleph-debian-11-python.squashfs"
           - os: "debian-12"
             artifact_name: "aleph-debian-12-python.squashfs"
     steps:
diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index 876495a35..84b3c1e15 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -31,13 +31,6 @@ jobs:
 
         # Check compatibility with all supported OSes.
         os_config:
-          - os_name: "Debian 11"
-            os_image: "debian-11-x64"
-            alias: "debian-11"
-            package_build_command: "all-podman-debian-11"
-            package_name: "aleph-vm.debian-11.deb"
-            concurrency_group: "droplet-aleph-vm-debian-11"
-
           - os_name: "Debian 12"
             os_image: "debian-12-x64"
             alias: "debian-12"
diff --git a/docker/vm_supervisor-dev.dockerfile b/docker/vm_supervisor-dev.dockerfile
index 14514f569..3db28b836 100644
--- a/docker/vm_supervisor-dev.dockerfile
+++ b/docker/vm_supervisor-dev.dockerfile
@@ -1,6 +1,6 @@
 # This is mainly a copy of the installation instructions from [orchestrator/README.md]
 
-FROM debian:bullseye
+FROM debian:bookworm
 
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     sudo acl curl squashfs-tools git \
diff --git a/examples/example_http_js/Dockerfile b/examples/example_http_js/Dockerfile
index 4916b01b1..cf2167653 100644
--- a/examples/example_http_js/Dockerfile
+++ b/examples/example_http_js/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:16-bullseye
+FROM node:16-bookworm
 
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
      libsecp256k1-dev \
diff --git a/examples/example_http_rust/Dockerfile b/examples/example_http_rust/Dockerfile
index 7144e041e..09bcab15b 100644
--- a/examples/example_http_rust/Dockerfile
+++ b/examples/example_http_rust/Dockerfile
@@ -1,4 +1,4 @@
-FROM rust:bullseye
+FROM rust:bookworm
 
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
      libsecp256k1-dev \
diff --git a/examples/volumes/Dockerfile b/examples/volumes/Dockerfile
index 8f57a0622..21a66c82c 100644
--- a/examples/volumes/Dockerfile
+++ b/examples/volumes/Dockerfile
@@ -1,4 +1,4 @@
-FROM debian:bullseye
+FROM debian:bookworm
 
 RUN apt-get update && apt-get -y upgrade && apt-get install -y \
     python3-venv \
diff --git a/packaging/Makefile b/packaging/Makefile
index 7e4a395cc..f22788c02 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -70,17 +70,6 @@ clean:
 	rm -fr ./aleph-vm/opt/aleph-vm/
 	rm -fr ./sevctl/target/
 
-all-podman-debian-11: version
-	cd .. && podman build -t localhost/aleph-vm-packaging-debian-11:latest -f ./packaging/debian-11.dockerfile .
-	mkdir -p ./target
-	podman run --rm -ti \
-		-w /opt/packaging \
-		-v ./target:/opt/packaging/target \
-		localhost/aleph-vm-packaging-debian-11:latest \
-		make
-	file target/aleph-vm.deb
-	mv target/aleph-vm.deb target/aleph-vm.debian-11.deb
-
 all-podman-debian-12: version
 	cd .. && podman build -t localhost/aleph-vm-packaging-debian-12:latest -f ./packaging/debian-12.dockerfile .
 	mkdir -p ./target
@@ -116,22 +105,13 @@ all-podman-ubuntu-2404: version
 	file target/aleph-vm.deb
 	mv target/aleph-vm.deb target/aleph-vm.ubuntu-24.04.deb
 
-# extract Python requirements from Debian 11 container
-requirements-debian-11: all-podman-debian-11
-	podman run --rm -ti \
-		-v ./target/aleph-vm.debian-11.deb:/opt/packaging/target/aleph-vm.deb:ro \
-		-v ./extract_requirements.sh:/opt/extract_requirements.sh:ro \
-		-v ./requirements-debian-11.txt:/mnt/requirements-debian-11.txt \
-		debian:bullseye \
-		bash -c "/opt/extract_requirements.sh /mnt/requirements-debian-11.txt"
-
 # extract Python requirements from Debian 12 container
 requirements-debian-12: all-podman-debian-12
 	podman run --rm -ti \
 		-v ./target/aleph-vm.debian-12.deb:/opt/packaging/target/aleph-vm.deb:ro \
 		-v ./extract_requirements.sh:/opt/extract_requirements.sh:ro \
 		-v ./requirements-debian-12.txt:/mnt/requirements-debian-12.txt \
-		debian:bullseye \
+		debian:bookworm \
 		bash -c "/opt/extract_requirements.sh /mnt/requirements-debian-12.txt"
 
 # extract Python requirements from Ubuntu 22.04 container
@@ -153,13 +133,18 @@ requirements-ubuntu-24.04: all-podman-ubuntu-2404
 		bash -c "/opt/extract_requirements.sh /mnt/requirements-ubuntu-24.04.txt"
 
 # run on host in order to sign with GPG
-repository-bullseye:
-	cd ./repositories/bullseye && reprepro -Vb . includedeb bullseye ../../target/aleph-vm.debian-11.deb && cd ..
+repository-bookworm:
+	cd ./repositories/bookworm && reprepro -Vb . includedeb bookworm ../../target/aleph-vm.debian-12.deb && cd ..
 
 # run on host in order to sign with GPG
 repository-jammy:
 	cd ./repositories/jammy && reprepro -Vb . includedeb jammy ../../target/aleph-vm.ubuntu-22.04.deb && cd ..
 
-repositories: repository-bullseye repository-jammy
+# run on host in order to sign with GPG
+repository-noble:
+	cd ./repositories/noble && reprepro -Vb . includedeb noble ../../target/aleph-vm.ubuntu-24.04.deb && cd ..
+
+repositories: repository-bookworm repository-jammy repository-noble
+ 
+all-podman: all-podman-debian-12 all-podman-ubuntu-2204 all-podman-ubuntu-2404 repositories
 
-all-podman: all-podman-debian-11 all-podman-ubuntu-2204 repositories
diff --git a/packaging/debian-11.dockerfile b/packaging/debian-11.dockerfile
deleted file mode 100644
index ebe903ef6..000000000
--- a/packaging/debian-11.dockerfile
+++ /dev/null
@@ -1,16 +0,0 @@
-FROM rust:1.79.0-bullseye
-
-RUN apt-get update && apt-get -y upgrade && apt-get install -y \
-    make \
-    git \
-    curl \
-    sudo \
-    python3-pip \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /opt
-COPY ../src/aleph/ ./src/aleph
-COPY ../packaging ./packaging
-COPY ../kernels ./kernels
-
-COPY ../examples/ ./examples
diff --git a/packaging/repositories/bullseye/conf/distributions b/packaging/repositories/bookworm/conf/distributions
similarity index 92%
rename from packaging/repositories/bullseye/conf/distributions
rename to packaging/repositories/bookworm/conf/distributions
index a1d0ecfd6..3891c6001 100644
--- a/packaging/repositories/bullseye/conf/distributions
+++ b/packaging/repositories/bookworm/conf/distributions
@@ -1,7 +1,7 @@
 Origin: Aleph-IM
 Label: aleph-im
 Suite: stable
-Codename: bullseye
+Codename: bookworm
 Version: 3.0
 Architectures: amd64 source
 Components: contrib
diff --git a/pyproject.toml b/pyproject.toml
index aa74d1de7..33457e454 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -148,7 +148,7 @@ norecursedirs = [
 ]
 
 [tool.black]
-target-version = ["py39"]
+target-version = ["py310"]
 line-length = 120
 #skip-string-normalization = true
 
diff --git a/runtimes/aleph-debian-11-python/create_disk_image.sh b/runtimes/aleph-debian-11-python/create_disk_image.sh
deleted file mode 100755
index 2f426b903..000000000
--- a/runtimes/aleph-debian-11-python/create_disk_image.sh
+++ /dev/null
@@ -1,106 +0,0 @@
-#!/bin/sh
-
-rm ./rootfs.squashfs
-
-set -euf
-
-rm -fr ./rootfs
-mkdir ./rootfs
-
-debootstrap --variant=minbase bullseye ./rootfs http://deb.debian.org/debian/
-
-chroot ./rootfs /bin/sh < /etc/locale.gen
-locale-gen en_US.UTF-8
-
-pip3 install 'fastapi~=0.103.1'
-
-echo "Pip installing aleph-client"
-pip3 install 'aleph-sdk-python==0.7.0'
-
-# Compile Python code to bytecode for faster execution
-# -o2 is needed to compile with optimization level 2 which is what we launch init1.py ("python -OO")
-# otherwise they are not used
-python3 -m compileall -o 2 -f /usr/local/lib/python3.9
-
-
-echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
-echo "PasswordAuthentication no" >> /etc/ssh/sshd_config
-echo "ChallengeResponseAuthentication no" >> /etc/ssh/sshd_config
-echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
-
-mkdir -p /overlay
-
-# Set up a login terminal on the serial console (ttyS0):
-ln -s agetty /etc/init.d/agetty.ttyS0
-echo ttyS0 > /etc/securetty
-EOT
-
-
-# Generate SSH host keys
-#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
-#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key
-#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key
-#systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t ed25519 -f /etc/ssh/ssh_host_ed25519_key
-
-cat < ./rootfs/etc/inittab
-# /etc/inittab
-
-::sysinit:/sbin/init sysinit
-::sysinit:/sbin/init boot
-::wait:/sbin/init default
-
-# Set up a couple of getty's
-tty1::respawn:/sbin/getty 38400 tty1
-tty2::respawn:/sbin/getty 38400 tty2
-tty3::respawn:/sbin/getty 38400 tty3
-tty4::respawn:/sbin/getty 38400 tty4
-tty5::respawn:/sbin/getty 38400 tty5
-tty6::respawn:/sbin/getty 38400 tty6
-
-# Put a getty on the serial port
-ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100
-
-# Stuff to do for the 3-finger salute
-::ctrlaltdel:/sbin/reboot
-
-# Stuff to do before rebooting
-::shutdown:/sbin/init shutdown
-EOT
-
-# Reduce size
-rm -fr ./rootfs/root/.cache
-rm -fr ./rootfs/var/cache
-mkdir -p ./rootfs/var/cache/apt/archives/partial
-rm -fr ./rootfs/usr/share/doc
-rm -fr ./rootfs/usr/share/man
-rm -fr ./rootfs/var/lib/apt/lists/
-
-# Custom init
-cp ./init0.sh ./rootfs/sbin/init
-cp ./init1.py ./rootfs/root/init1.py
-cp ./loading.html ./rootfs/root/loading.html
-chmod +x ./rootfs/sbin/init
-chmod +x ./rootfs/root/init1.py
-
-mksquashfs ./rootfs/ ./rootfs.squashfs
diff --git a/runtimes/aleph-debian-11-python/init0.sh b/runtimes/aleph-debian-11-python/init0.sh
deleted file mode 100644
index 75659b0b9..000000000
--- a/runtimes/aleph-debian-11-python/init0.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/sh
-
-set -euf
-
-mount -t proc proc /proc -o nosuid,noexec,nodev
-
-log() {
-    echo "$(awk '{print $1}' /proc/uptime)" '|S' "$@"
-}
-log "init0.sh is launching"
-
-# Switch root from read-only ext4 to to read-write overlay
-mkdir -p /overlay
-/bin/mount -t tmpfs -o noatime,mode=0755 tmpfs /overlay
-mkdir -p /overlay/root /overlay/work
-/bin/mount -o noatime,lowerdir=/,upperdir=/overlay/root,workdir=/overlay/work -t overlay "overlayfs:/overlay/root" /mnt
-mkdir -p /mnt/rom
-pivot_root /mnt /mnt/rom
-
-mount --move /rom/proc /proc
-mount --move /rom/dev /dev
-
-mkdir -p /dev/pts
-mkdir -p /dev/shm
-
-mount -t sysfs sys /sys -o nosuid,noexec,nodev
-mount -t tmpfs run /run -o mode=0755,nosuid,nodev
-#mount -t devtmpfs dev /dev -o mode=0755,nosuid
-mount -t devpts devpts /dev/pts -o mode=0620,gid=5,nosuid,noexec
-mount -t tmpfs shm /dev/shm -omode=1777,nosuid,nodev
-
-# Required by Docker
-cgroupfs-mount
-update-alternatives --set iptables /usr/sbin/iptables-legacy
-update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
-
-# Enable the following to force the storage driver used by Docker.
-# See https://docs.docker.com/storage/storagedriver/select-storage-driver/
-#echo '{\n"storage-driver": "overlay2"\n}\n' > /etc/docker/daemon.json
-
-# List block devices
-lsblk
-
-#cat /proc/sys/kernel/random/entropy_avail
-
-# TODO: Move in init1
-mkdir -p /run/sshd
-/usr/sbin/sshd &
-log "SSH UP"
-
-log "Setup socat"
-socat UNIX-LISTEN:/tmp/socat-socket,fork,reuseaddr VSOCK-CONNECT:2:53 &
-log "Socat ready"
-
-# Replace this script with the manager
-exec /root/init1.py
diff --git a/runtimes/aleph-debian-11-python/init1.py b/runtimes/aleph-debian-11-python/init1.py
deleted file mode 100644
index 11c4a7dd0..000000000
--- a/runtimes/aleph-debian-11-python/init1.py
+++ /dev/null
@@ -1,623 +0,0 @@
-#!/usr/bin/python3 -OO
-import base64
-import logging
-from pathlib import Path
-
-logging.basicConfig(
-    level=logging.DEBUG,
-    format="%(relativeCreated)4f |V %(levelname)s | %(message)s",
-)
-logger = logging.getLogger(__name__)
-
-logger.debug("Imports starting")
-
-import asyncio
-import ctypes
-import os
-import socket
-import subprocess
-import sys
-import traceback
-from collections.abc import AsyncIterable
-from contextlib import redirect_stdout
-from dataclasses import dataclass, field
-from enum import Enum
-from io import StringIO
-from os import system
-from shutil import make_archive
-from typing import Any, Literal, NewType, Optional, Union, cast
-
-import aiohttp
-import msgpack
-
-logger.debug("Imports finished")
-
-__version__ = "2.0.0"
-ASGIApplication = NewType("ASGIApplication", Any)  # type: ignore
-
-
-class Encoding(str, Enum):
-    plain = "plain"
-    zip = "zip"
-    squashfs = "squashfs"
-
-
-class Interface(str, Enum):
-    asgi = "asgi"
-    executable = "executable"
-
-
-class ShutdownException(Exception):
-    pass
-
-
-@dataclass
-class Volume:
-    mount: str
-    device: str
-    read_only: bool
-
-
-@dataclass
-class ConfigurationPayload:
-    input_data: bytes
-    interface: Interface
-    vm_hash: str
-    code: bytes
-    encoding: Encoding
-    entrypoint: str
-    ip: Optional[str] = None
-    ipv6: Optional[str] = None
-    route: Optional[str] = None
-    ipv6_gateway: Optional[str] = None
-    dns_servers: list[str] = field(default_factory=list)
-    volumes: list[Volume] = field(default_factory=list)
-    variables: Optional[dict[str, str]] = None
-    authorized_keys: Optional[list[str]] = None
-
-
-@dataclass
-class RunCodePayload:
-    scope: dict
-
-
-# Open a socket to receive instructions from the host
-s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
-s.bind((socket.VMADDR_CID_ANY, 52))
-s.listen()
-
-# Send the host that we are ready
-s0 = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
-s0.connect((2, 52))
-s0.sendall(msgpack.dumps({"version": __version__}))
-s0.close()
-
-# Configure aleph-client to use the guest API
-os.environ["ALEPH_INIT_VERSION"] = __version__
-os.environ["ALEPH_API_HOST"] = "http://localhost"
-os.environ["ALEPH_API_UNIX_SOCKET"] = "/tmp/socat-socket"
-os.environ["ALEPH_REMOTE_CRYPTO_HOST"] = "http://localhost"
-os.environ["ALEPH_REMOTE_CRYPTO_UNIX_SOCKET"] = "/tmp/socat-socket"
-
-logger.debug("init1.py is launching")
-
-
-def setup_hostname(hostname: str):
-    os.environ["ALEPH_ADDRESS_TO_USE"] = hostname
-    system(f"hostname {hostname}")
-
-
-def setup_variables(variables: Optional[dict[str, str]]):
-    if variables is None:
-        return
-    for key, value in variables.items():
-        os.environ[key] = value
-
-
-def setup_network(
-    ipv4: Optional[str],
-    ipv6: Optional[str],
-    ipv4_gateway: Optional[str],
-    ipv6_gateway: Optional[str],
-    dns_servers: Optional[list[str]] = None,
-):
-    """Setup the system with info from the host."""
-    dns_servers = dns_servers or []
-    if not os.path.exists("/sys/class/net/eth0"):
-        logger.error("No network interface eth0")
-        return
-
-    # Configure loopback networking
-    system("ip addr add 127.0.0.1/8 dev lo brd + scope host")
-    system("ip addr add ::1/128 dev lo")
-    system("ip link set lo up")
-
-    # Forward compatibility with future supervisors that pass the mask with the IP.
-    if ipv4 and ("/" not in ipv4):
-        logger.warning("Not passing the mask with the IP is deprecated and will be unsupported")
-        ipv4 = f"{ipv4}/24"
-
-    addresses = [ip for ip in [ipv4, ipv6] if ip]
-    gateways = [gateway for gateway in [ipv4_gateway, ipv6_gateway] if gateway]
-
-    for address in addresses:
-        system(f"ip addr add {address} dev eth0")
-
-    # Interface must be up before a route can use it
-    if addresses:
-        system("ip link set eth0 up")
-    else:
-        logger.debug("No ip address provided")
-
-    for gateway in gateways:
-        system(f"ip route add default via {gateway} dev eth0")
-
-    if not gateways:
-        logger.debug("No ip gateway provided")
-
-    with open("/etc/resolv.conf", "wb") as resolvconf_fd:
-        for server in dns_servers:
-            resolvconf_fd.write(f"nameserver {server}\n".encode())
-
-
-def setup_input_data(input_data: bytes):
-    logger.debug("Extracting data")
-    if input_data:
-        # Unzip in /data
-        if not os.path.exists("/opt/input.zip"):
-            open("/opt/input.zip", "wb").write(input_data)
-            os.makedirs("/data", exist_ok=True)
-            os.system("unzip -q /opt/input.zip -d /data")
-
-
-def setup_authorized_keys(authorized_keys: list[str]) -> None:
-    path = Path("/root/.ssh/authorized_keys")
-    path.parent.mkdir(exist_ok=True)
-    path.write_text("\n".join(key for key in authorized_keys))
-
-
-def setup_volumes(volumes: list[Volume]):
-    for volume in volumes:
-        logger.debug(f"Mounting /dev/{volume.device} on {volume.mount}")
-        os.makedirs(volume.mount, exist_ok=True)
-        if volume.read_only:
-            system(f"mount -t squashfs -o ro /dev/{volume.device} {volume.mount}")
-        else:
-            system(f"mount -o rw /dev/{volume.device} {volume.mount}")
-
-    system("mount")
-
-
-async def wait_for_lifespan_event_completion(
-    application: ASGIApplication, event: Union[Literal["startup", "shutdown"]]
-):
-    """
-    Send the startup lifespan signal to the ASGI app.
-    Specification: https://asgi.readthedocs.io/en/latest/specs/lifespan.html
-    """
-
-    lifespan_completion = asyncio.Event()
-
-    async def receive():
-        return {
-            "type": f"lifespan.{event}",
-        }
-
-    async def send(response: dict):
-        response_type = response.get("type")
-        if response_type == f"lifespan.{event}.complete":
-            lifespan_completion.set()
-            return
-        else:
-            logger.warning(f"Unexpected response to {event}: {response_type}")
-
-    while not lifespan_completion.is_set():
-        await application(
-            scope={
-                "type": "lifespan",
-            },
-            receive=receive,
-            send=send,
-        )
-
-
-async def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplication:
-    # Allow importing packages from /opt/packages, give it priority
-    sys.path.insert(0, "/opt/packages")
-
-    logger.debug("Extracting code")
-    app: ASGIApplication
-    if encoding == Encoding.squashfs:
-        sys.path.insert(0, "/opt/code")
-        module_name, app_name = entrypoint.split(":", 1)
-        logger.debug("import module")
-        module = __import__(module_name)
-        for level in module_name.split(".")[1:]:
-            module = getattr(module, level)
-        app = getattr(module, app_name)
-    elif encoding == Encoding.zip:
-        # Unzip in /opt and import the entrypoint from there
-        if not os.path.exists("/opt/archive.zip"):
-            open("/opt/archive.zip", "wb").write(code)
-            logger.debug("Run unzip")
-            os.system("unzip -q /opt/archive.zip -d /opt")
-        sys.path.insert(0, "/opt")
-        module_name, app_name = entrypoint.split(":", 1)
-        logger.debug("import module")
-        module = __import__(module_name)
-        for level in module_name.split(".")[1:]:
-            module = getattr(module, level)
-        logger.debug("import done")
-        app = getattr(module, app_name)
-    elif encoding == Encoding.plain:
-        # Execute the code and extract the entrypoint
-        locals: dict[str, Any] = {}
-        exec(code, globals(), locals)
-        app = locals[entrypoint]
-    else:
-        raise ValueError(f"Unknown encoding '{encoding}'")
-    await wait_for_lifespan_event_completion(application=app, event="startup")
-    return ASGIApplication(app)
-
-
-def setup_code_executable(code: bytes, encoding: Encoding, entrypoint: str) -> subprocess.Popen:
-    logger.debug("Extracting code")
-    if encoding == Encoding.squashfs:
-        path = f"/opt/code/{entrypoint}"
-        if not os.path.isfile(path):
-            os.system("find /opt/code/")
-            raise FileNotFoundError(f"No such file: {path}")
-        os.system(f"chmod +x {path}")
-    elif encoding == Encoding.zip:
-        open("/opt/archive.zip", "wb").write(code)
-        logger.debug("Run unzip")
-        os.makedirs("/opt/code", exist_ok=True)
-        os.system("unzip /opt/archive.zip -d /opt/code")
-        path = f"/opt/code/{entrypoint}"
-        if not os.path.isfile(path):
-            os.system("find /opt/code")
-            raise FileNotFoundError(f"No such file: {path}")
-        os.system(f"chmod +x {path}")
-    elif encoding == Encoding.plain:
-        os.makedirs("/opt/code", exist_ok=True)
-        path = f"/opt/code/executable {entrypoint}"
-        open(path, "wb").write(code)
-        os.system(f"chmod +x {path}")
-    else:
-        raise ValueError(f"Unknown encoding '{encoding}'. This should never happen.")
-
-    process = subprocess.Popen(path)
-    return process
-
-
-async def setup_code(
-    code: bytes,
-    encoding: Encoding,
-    entrypoint: str,
-    interface: Interface,
-) -> Union[ASGIApplication, subprocess.Popen]:
-    if interface == Interface.asgi:
-        return await setup_code_asgi(code=code, encoding=encoding, entrypoint=entrypoint)
-    elif interface == Interface.executable:
-        return setup_code_executable(code=code, encoding=encoding, entrypoint=entrypoint)
-    else:
-        raise ValueError("Invalid interface. This should never happen.")
-
-
-async def run_python_code_http(application: ASGIApplication, scope: dict) -> tuple[dict, dict, str, Optional[bytes]]:
-    logger.debug("Running code")
-    with StringIO() as buf, redirect_stdout(buf):
-        # Execute in the same process, saves ~20ms than a subprocess
-
-        # The body should not be part of the ASGI scope itself
-        scope_body: bytes = scope.pop("body")
-
-        async def receive():
-            type_ = "http.request" if scope["type"] in ("http", "websocket") else "aleph.message"
-            return {"type": type_, "body": scope_body, "more_body": False}
-
-        send_queue: asyncio.Queue = asyncio.Queue()
-
-        async def send(dico):
-            await send_queue.put(dico)
-
-        # TODO: Better error handling
-        logger.debug("Awaiting application...")
-        await application(scope, receive, send)
-
-        logger.debug("Waiting for headers")
-        headers: dict
-        if scope["type"] == "http":
-            headers = await send_queue.get()
-        else:
-            headers = {}
-
-        logger.debug("Waiting for body")
-        response_body: dict = await send_queue.get()
-
-        logger.debug("Waiting for buffer")
-        output = buf.getvalue()
-
-        logger.debug(f"Headers {headers}")
-        logger.debug(f"Body {response_body}")
-        logger.debug(f"Output {output}")
-
-    logger.debug("Getting output data")
-    output_data: bytes
-    if os.path.isdir("/data") and os.listdir("/data"):
-        make_archive("/opt/output", "zip", "/data")
-        with open("/opt/output.zip", "rb") as output_zipfile:
-            output_data = output_zipfile.read()
-    else:
-        output_data = b""
-
-    logger.debug("Returning result")
-    return headers, response_body, output, output_data
-
-
-async def make_request(session, scope):
-    async with session.request(
-        scope["method"],
-        url="http://localhost:8080{}".format(scope["path"]),
-        params=scope["query_string"],
-        headers=[(a.decode("utf-8"), b.decode("utf-8")) for a, b in scope["headers"]],
-        data=scope.get("body", None),
-    ) as resp:
-        headers = {
-            "headers": [(a.encode("utf-8"), b.encode("utf-8")) for a, b in resp.headers.items()],
-            "status": resp.status,
-        }
-        body = {"body": await resp.content.read()}
-    return headers, body
-
-
-def show_loading():
-    body = {"body": Path("/root/loading.html").read_text()}
-    headers = {
-        "headers": [
-            [b"Content-Type", b"text/html"],
-            [b"Connection", b"keep-alive"],
-            [b"Keep-Alive", b"timeout=5"],
-            [b"Transfer-Encoding", b"chunked"],
-        ],
-        "status": 503,
-    }
-    return headers, body
-
-
-async def run_executable_http(scope: dict) -> tuple[dict, dict, str, Optional[bytes]]:
-    logger.debug("Calling localhost")
-
-    tries = 0
-    headers = None
-    body = None
-
-    timeout = aiohttp.ClientTimeout(total=5)
-    async with aiohttp.ClientSession(timeout=timeout) as session:
-        while not body:
-            try:
-                tries += 1
-                headers, body = await make_request(session, scope)
-            except aiohttp.ClientConnectorError:
-                if tries > 20:
-                    headers, body = show_loading()
-                await asyncio.sleep(0.05)
-
-    output = ""  # Process stdout is not captured per request
-    output_data = None
-    logger.debug("Returning result")
-    return headers, body, output, output_data
-
-
-async def process_instruction(
-    instruction: bytes,
-    interface: Interface,
-    application: Union[ASGIApplication, subprocess.Popen],
-) -> AsyncIterable[bytes]:
-    if instruction == b"halt":
-        logger.info("Received halt command")
-        system("sync")
-        logger.debug("Filesystems synced")
-        if isinstance(application, subprocess.Popen):
-            application.terminate()
-            logger.debug("Application terminated")
-            # application.communicate()
-        else:
-            await wait_for_lifespan_event_completion(application=application, event="shutdown")
-        yield b"STOP\n"
-        logger.debug("Supervisor informed of halt")
-        raise ShutdownException
-    elif instruction.startswith(b"!"):
-        # Execute shell commands in the form `!ls /`
-        msg = instruction[1:].decode()
-        try:
-            process_output = subprocess.check_output(msg, stderr=subprocess.STDOUT, shell=True)
-            yield process_output
-        except subprocess.CalledProcessError as error:
-            yield str(error).encode() + b"\n" + error.output
-    else:
-        # Python
-        logger.debug("msgpack.loads (")
-        msg_ = msgpack.loads(instruction, raw=False)
-        logger.debug("msgpack.loads )")
-        payload = RunCodePayload(**msg_)
-
-        output: Optional[str] = None
-        try:
-            headers: dict
-            body: dict
-            output_data: Optional[bytes]
-
-            if interface == Interface.asgi:
-                application = cast(ASGIApplication, application)
-                headers, body, output, output_data = await run_python_code_http(
-                    application=application, scope=payload.scope
-                )
-            elif interface == Interface.executable:
-                headers, body, output, output_data = await run_executable_http(scope=payload.scope)
-            else:
-                raise ValueError("Unknown interface. This should never happen")
-
-            result = {
-                "headers": headers,
-                "body": body,
-                "output": output,
-                "output_data": output_data,
-            }
-            yield msgpack.dumps(result, use_bin_type=True)
-        except Exception as error:
-            yield msgpack.dumps(
-                {
-                    "error": str(error),
-                    "traceback": str(traceback.format_exc()),
-                    "output": output,
-                }
-            )
-
-
-def receive_data_length(client) -> int:
-    """Receive the length of the data to follow."""
-    buffer = b""
-    for _ in range(9):
-        byte = client.recv(1)
-        if byte == b"\n":
-            break
-        else:
-            buffer += byte
-    return int(buffer)
-
-
-def load_configuration(data: bytes) -> ConfigurationPayload:
-    msg_ = msgpack.loads(data, raw=False)
-    msg_["volumes"] = [Volume(**volume_dict) for volume_dict in msg_.get("volumes")]
-    return ConfigurationPayload(**msg_)
-
-
-def receive_config(client) -> ConfigurationPayload:
-    length = receive_data_length(client)
-    data = b""
-    while len(data) < length:
-        data += client.recv(1024 * 1024)
-    return load_configuration(data)
-
-
-def setup_system(config: ConfigurationPayload):
-    # Linux host names are limited to 63 characters. We therefore use the base32 representation
-    # of the item_hash instead of its common base16 representation.
-    item_hash_binary: bytes = base64.b16decode(config.vm_hash.encode().upper())
-    hostname = base64.b32encode(item_hash_binary).decode().strip("=").lower()
-    setup_hostname(hostname)
-
-    setup_variables(config.variables)
-    setup_volumes(config.volumes)
-    setup_network(
-        ipv4=config.ip,
-        ipv6=config.ipv6,
-        ipv4_gateway=config.route,
-        ipv6_gateway=config.ipv6_gateway,
-        dns_servers=config.dns_servers,
-    )
-    setup_input_data(config.input_data)
-    if authorized_keys := config.authorized_keys:
-        setup_authorized_keys(authorized_keys)
-    logger.debug("Setup finished")
-
-
-def umount_volumes(volumes: list[Volume]):
-    "Umount user related filesystems"
-    system("sync")
-    for volume in volumes:
-        logger.debug(f"Umounting /dev/{volume.device} on {volume.mount}")
-        system(f"umount {volume.mount}")
-
-
-async def main() -> None:
-    client, addr = s.accept()
-
-    logger.debug("Receiving setup...")
-    config = receive_config(client)
-    setup_system(config)
-
-    try:
-        app: Union[ASGIApplication, subprocess.Popen] = await setup_code(
-            config.code, config.encoding, config.entrypoint, config.interface
-        )
-        client.send(msgpack.dumps({"success": True}))
-    except Exception as error:
-        client.send(
-            msgpack.dumps(
-                {
-                    "success": False,
-                    "error": str(error),
-                    "traceback": str(traceback.format_exc()),
-                }
-            )
-        )
-        logger.exception("Program could not be started")
-        raise
-
-    class ServerReference:
-        "Reference used to close the server from within `handle_instruction"
-        server: asyncio.AbstractServer
-
-    server_reference = ServerReference()
-
-    async def handle_instruction(reader, writer):
-        data = await reader.read(1000_1000)  # Max 1 Mo
-
-        logger.debug("Init received msg")
-        if logger.level <= logging.DEBUG:
-            data_to_print = f"{data[:500]}..." if len(data) > 500 else data
-            logger.debug(f"<<<\n\n{data_to_print}\n\n>>>")
-
-        try:
-            async for result in process_instruction(instruction=data, interface=config.interface, application=app):
-                writer.write(result)
-                await writer.drain()
-
-                logger.debug("Instruction processed")
-        except ShutdownException:
-            logger.info("Initiating shutdown")
-            writer.write(b"STOPZ\n")
-            await writer.drain()
-            logger.debug("Shutdown confirmed to supervisor")
-            server_reference.server.close()
-            logger.debug("Supervisor socket server closed")
-        finally:
-            writer.close()
-
-    server = await asyncio.start_server(handle_instruction, sock=s)
-    server_reference.server = server
-
-    addr = server.sockets[0].getsockname()
-    print(f"Serving on {addr}")
-
-    try:
-        async with server:
-            await server.serve_forever()
-    except asyncio.CancelledError:
-        logger.debug("Server was properly cancelled")
-    finally:
-        logger.warning("System shutdown")
-        server.close()
-        logger.debug("Server closed")
-        umount_volumes(config.volumes)
-        logger.debug("User volumes unmounted")
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.DEBUG)
-    asyncio.run(main())
-
-    logger.info("Unmounting system filesystems")
-    system("umount /dev/shm")
-    system("umount /dev/pts")
-    system("umount -a")
-
-    logger.info("Sending reboot syscall")
-    # Send reboot syscall, see man page
-    # https://man7.org/linux/man-pages/man2/reboot.2.html
-    libc = ctypes.CDLL(None)
-    libc.syscall(169, 0xFEE1DEAD, 672274793, 0x4321FEDC, None)
-    # The exit should not happen due to system halt.
-    sys.exit(0)
diff --git a/runtimes/aleph-debian-11-python/loading.html b/runtimes/aleph-debian-11-python/loading.html
deleted file mode 100644
index da9128c40..000000000
--- a/runtimes/aleph-debian-11-python/loading.html
+++ /dev/null
@@ -1,346 +0,0 @@
-
-
-  VM Loading
-  
-
-
-
-
-
              
-  
              
-    
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-    
-  
              
-  
              
-    
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-      
-      
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-      
-      
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-      
-      
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-      
-      
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-      
-      
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-      
-      
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-      
-      
-        
-          
-          
-        
-      
-    
-  
              
-  
              
-    
              
-      
              Whoops!
              
-      
              Seems like your VM is still loading, please wait...
              
-    
              
-
-    Refresh!
-  
              
-
-
              
-
-
diff --git a/runtimes/aleph-debian-11-python/update_inits.sh b/runtimes/aleph-debian-11-python/update_inits.sh
deleted file mode 100755
index 55a1c99b1..000000000
--- a/runtimes/aleph-debian-11-python/update_inits.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-rm ./rootfs.squashfs
-
-set -euf
-
-cp ./init0.sh ./rootfs/sbin/init
-cp ./init1.py ./rootfs/root/init1.py
-chmod +x ./rootfs/sbin/init
-chmod +x ./rootfs/root/init1.py
-
-mksquashfs ./rootfs/ ./rootfs.squashfs
-
-echo "OK"
diff --git a/runtimes/aleph-debian-12-python/init1.py b/runtimes/aleph-debian-12-python/init1.py
deleted file mode 120000
index 7f48acafa..000000000
--- a/runtimes/aleph-debian-12-python/init1.py
+++ /dev/null
@@ -1 +0,0 @@
-../aleph-debian-11-python/init1.py
\ No newline at end of file
diff --git a/runtimes/aleph-debian-12-python/init1.py b/runtimes/aleph-debian-12-python/init1.py
new file mode 100644
index 000000000..11c4a7dd0
--- /dev/null
+++ b/runtimes/aleph-debian-12-python/init1.py
@@ -0,0 +1,623 @@
+#!/usr/bin/python3 -OO
+import base64
+import logging
+from pathlib import Path
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    format="%(relativeCreated)4f |V %(levelname)s | %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+logger.debug("Imports starting")
+
+import asyncio
+import ctypes
+import os
+import socket
+import subprocess
+import sys
+import traceback
+from collections.abc import AsyncIterable
+from contextlib import redirect_stdout
+from dataclasses import dataclass, field
+from enum import Enum
+from io import StringIO
+from os import system
+from shutil import make_archive
+from typing import Any, Literal, NewType, Optional, Union, cast
+
+import aiohttp
+import msgpack
+
+logger.debug("Imports finished")
+
+__version__ = "2.0.0"
+ASGIApplication = NewType("ASGIApplication", Any)  # type: ignore
+
+
+class Encoding(str, Enum):
+    plain = "plain"
+    zip = "zip"
+    squashfs = "squashfs"
+
+
+class Interface(str, Enum):
+    asgi = "asgi"
+    executable = "executable"
+
+
+class ShutdownException(Exception):
+    pass
+
+
+@dataclass
+class Volume:
+    mount: str
+    device: str
+    read_only: bool
+
+
+@dataclass
+class ConfigurationPayload:
+    input_data: bytes
+    interface: Interface
+    vm_hash: str
+    code: bytes
+    encoding: Encoding
+    entrypoint: str
+    ip: Optional[str] = None
+    ipv6: Optional[str] = None
+    route: Optional[str] = None
+    ipv6_gateway: Optional[str] = None
+    dns_servers: list[str] = field(default_factory=list)
+    volumes: list[Volume] = field(default_factory=list)
+    variables: Optional[dict[str, str]] = None
+    authorized_keys: Optional[list[str]] = None
+
+
+@dataclass
+class RunCodePayload:
+    scope: dict
+
+
+# Open a socket to receive instructions from the host
+s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
+s.bind((socket.VMADDR_CID_ANY, 52))
+s.listen()
+
+# Send the host that we are ready
+s0 = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
+s0.connect((2, 52))
+s0.sendall(msgpack.dumps({"version": __version__}))
+s0.close()
+
+# Configure aleph-client to use the guest API
+os.environ["ALEPH_INIT_VERSION"] = __version__
+os.environ["ALEPH_API_HOST"] = "http://localhost"
+os.environ["ALEPH_API_UNIX_SOCKET"] = "/tmp/socat-socket"
+os.environ["ALEPH_REMOTE_CRYPTO_HOST"] = "http://localhost"
+os.environ["ALEPH_REMOTE_CRYPTO_UNIX_SOCKET"] = "/tmp/socat-socket"
+
+logger.debug("init1.py is launching")
+
+
+def setup_hostname(hostname: str):
+    os.environ["ALEPH_ADDRESS_TO_USE"] = hostname
+    system(f"hostname {hostname}")
+
+
+def setup_variables(variables: Optional[dict[str, str]]):
+    if variables is None:
+        return
+    for key, value in variables.items():
+        os.environ[key] = value
+
+
+def setup_network(
+    ipv4: Optional[str],
+    ipv6: Optional[str],
+    ipv4_gateway: Optional[str],
+    ipv6_gateway: Optional[str],
+    dns_servers: Optional[list[str]] = None,
+):
+    """Setup the system with info from the host."""
+    dns_servers = dns_servers or []
+    if not os.path.exists("/sys/class/net/eth0"):
+        logger.error("No network interface eth0")
+        return
+
+    # Configure loopback networking
+    system("ip addr add 127.0.0.1/8 dev lo brd + scope host")
+    system("ip addr add ::1/128 dev lo")
+    system("ip link set lo up")
+
+    # Forward compatibility with future supervisors that pass the mask with the IP.
+    if ipv4 and ("/" not in ipv4):
+        logger.warning("Not passing the mask with the IP is deprecated and will be unsupported")
+        ipv4 = f"{ipv4}/24"
+
+    addresses = [ip for ip in [ipv4, ipv6] if ip]
+    gateways = [gateway for gateway in [ipv4_gateway, ipv6_gateway] if gateway]
+
+    for address in addresses:
+        system(f"ip addr add {address} dev eth0")
+
+    # Interface must be up before a route can use it
+    if addresses:
+        system("ip link set eth0 up")
+    else:
+        logger.debug("No ip address provided")
+
+    for gateway in gateways:
+        system(f"ip route add default via {gateway} dev eth0")
+
+    if not gateways:
+        logger.debug("No ip gateway provided")
+
+    with open("/etc/resolv.conf", "wb") as resolvconf_fd:
+        for server in dns_servers:
+            resolvconf_fd.write(f"nameserver {server}\n".encode())
+
+
+def setup_input_data(input_data: bytes):
+    logger.debug("Extracting data")
+    if input_data:
+        # Unzip in /data
+        if not os.path.exists("/opt/input.zip"):
+            open("/opt/input.zip", "wb").write(input_data)
+            os.makedirs("/data", exist_ok=True)
+            os.system("unzip -q /opt/input.zip -d /data")
+
+
+def setup_authorized_keys(authorized_keys: list[str]) -> None:
+    path = Path("/root/.ssh/authorized_keys")
+    path.parent.mkdir(exist_ok=True)
+    path.write_text("\n".join(key for key in authorized_keys))
+
+
+def setup_volumes(volumes: list[Volume]):
+    for volume in volumes:
+        logger.debug(f"Mounting /dev/{volume.device} on {volume.mount}")
+        os.makedirs(volume.mount, exist_ok=True)
+        if volume.read_only:
+            system(f"mount -t squashfs -o ro /dev/{volume.device} {volume.mount}")
+        else:
+            system(f"mount -o rw /dev/{volume.device} {volume.mount}")
+
+    system("mount")
+
+
+async def wait_for_lifespan_event_completion(
+    application: ASGIApplication, event: Union[Literal["startup", "shutdown"]]
+):
+    """
+    Send the startup lifespan signal to the ASGI app.
+    Specification: https://asgi.readthedocs.io/en/latest/specs/lifespan.html
+    """
+
+    lifespan_completion = asyncio.Event()
+
+    async def receive():
+        return {
+            "type": f"lifespan.{event}",
+        }
+
+    async def send(response: dict):
+        response_type = response.get("type")
+        if response_type == f"lifespan.{event}.complete":
+            lifespan_completion.set()
+            return
+        else:
+            logger.warning(f"Unexpected response to {event}: {response_type}")
+
+    while not lifespan_completion.is_set():
+        await application(
+            scope={
+                "type": "lifespan",
+            },
+            receive=receive,
+            send=send,
+        )
+
+
+async def setup_code_asgi(code: bytes, encoding: Encoding, entrypoint: str) -> ASGIApplication:
+    # Allow importing packages from /opt/packages, give it priority
+    sys.path.insert(0, "/opt/packages")
+
+    logger.debug("Extracting code")
+    app: ASGIApplication
+    if encoding == Encoding.squashfs:
+        sys.path.insert(0, "/opt/code")
+        module_name, app_name = entrypoint.split(":", 1)
+        logger.debug("import module")
+        module = __import__(module_name)
+        for level in module_name.split(".")[1:]:
+            module = getattr(module, level)
+        app = getattr(module, app_name)
+    elif encoding == Encoding.zip:
+        # Unzip in /opt and import the entrypoint from there
+        if not os.path.exists("/opt/archive.zip"):
+            open("/opt/archive.zip", "wb").write(code)
+            logger.debug("Run unzip")
+            os.system("unzip -q /opt/archive.zip -d /opt")
+        sys.path.insert(0, "/opt")
+        module_name, app_name = entrypoint.split(":", 1)
+        logger.debug("import module")
+        module = __import__(module_name)
+        for level in module_name.split(".")[1:]:
+            module = getattr(module, level)
+        logger.debug("import done")
+        app = getattr(module, app_name)
+    elif encoding == Encoding.plain:
+        # Execute the code and extract the entrypoint
+        locals: dict[str, Any] = {}
+        exec(code, globals(), locals)
+        app = locals[entrypoint]
+    else:
+        raise ValueError(f"Unknown encoding '{encoding}'")
+    await wait_for_lifespan_event_completion(application=app, event="startup")
+    return ASGIApplication(app)
+
+
+def setup_code_executable(code: bytes, encoding: Encoding, entrypoint: str) -> subprocess.Popen:
+    logger.debug("Extracting code")
+    if encoding == Encoding.squashfs:
+        path = f"/opt/code/{entrypoint}"
+        if not os.path.isfile(path):
+            os.system("find /opt/code/")
+            raise FileNotFoundError(f"No such file: {path}")
+        os.system(f"chmod +x {path}")
+    elif encoding == Encoding.zip:
+        open("/opt/archive.zip", "wb").write(code)
+        logger.debug("Run unzip")
+        os.makedirs("/opt/code", exist_ok=True)
+        os.system("unzip /opt/archive.zip -d /opt/code")
+        path = f"/opt/code/{entrypoint}"
+        if not os.path.isfile(path):
+            os.system("find /opt/code")
+            raise FileNotFoundError(f"No such file: {path}")
+        os.system(f"chmod +x {path}")
+    elif encoding == Encoding.plain:
+        os.makedirs("/opt/code", exist_ok=True)
+        path = f"/opt/code/executable {entrypoint}"
+        open(path, "wb").write(code)
+        os.system(f"chmod +x {path}")
+    else:
+        raise ValueError(f"Unknown encoding '{encoding}'. This should never happen.")
+
+    process = subprocess.Popen(path)
+    return process
+
+
+async def setup_code(
+    code: bytes,
+    encoding: Encoding,
+    entrypoint: str,
+    interface: Interface,
+) -> Union[ASGIApplication, subprocess.Popen]:
+    if interface == Interface.asgi:
+        return await setup_code_asgi(code=code, encoding=encoding, entrypoint=entrypoint)
+    elif interface == Interface.executable:
+        return setup_code_executable(code=code, encoding=encoding, entrypoint=entrypoint)
+    else:
+        raise ValueError("Invalid interface. This should never happen.")
+
+
+async def run_python_code_http(application: ASGIApplication, scope: dict) -> tuple[dict, dict, str, Optional[bytes]]:
+    logger.debug("Running code")
+    with StringIO() as buf, redirect_stdout(buf):
+        # Execute in the same process, saves ~20ms than a subprocess
+
+        # The body should not be part of the ASGI scope itself
+        scope_body: bytes = scope.pop("body")
+
+        async def receive():
+            type_ = "http.request" if scope["type"] in ("http", "websocket") else "aleph.message"
+            return {"type": type_, "body": scope_body, "more_body": False}
+
+        send_queue: asyncio.Queue = asyncio.Queue()
+
+        async def send(dico):
+            await send_queue.put(dico)
+
+        # TODO: Better error handling
+        logger.debug("Awaiting application...")
+        await application(scope, receive, send)
+
+        logger.debug("Waiting for headers")
+        headers: dict
+        if scope["type"] == "http":
+            headers = await send_queue.get()
+        else:
+            headers = {}
+
+        logger.debug("Waiting for body")
+        response_body: dict = await send_queue.get()
+
+        logger.debug("Waiting for buffer")
+        output = buf.getvalue()
+
+        logger.debug(f"Headers {headers}")
+        logger.debug(f"Body {response_body}")
+        logger.debug(f"Output {output}")
+
+    logger.debug("Getting output data")
+    output_data: bytes
+    if os.path.isdir("/data") and os.listdir("/data"):
+        make_archive("/opt/output", "zip", "/data")
+        with open("/opt/output.zip", "rb") as output_zipfile:
+            output_data = output_zipfile.read()
+    else:
+        output_data = b""
+
+    logger.debug("Returning result")
+    return headers, response_body, output, output_data
+
+
+async def make_request(session, scope):
+    async with session.request(
+        scope["method"],
+        url="http://localhost:8080{}".format(scope["path"]),
+        params=scope["query_string"],
+        headers=[(a.decode("utf-8"), b.decode("utf-8")) for a, b in scope["headers"]],
+        data=scope.get("body", None),
+    ) as resp:
+        headers = {
+            "headers": [(a.encode("utf-8"), b.encode("utf-8")) for a, b in resp.headers.items()],
+            "status": resp.status,
+        }
+        body = {"body": await resp.content.read()}
+    return headers, body
+
+
+def show_loading():
+    body = {"body": Path("/root/loading.html").read_text()}
+    headers = {
+        "headers": [
+            [b"Content-Type", b"text/html"],
+            [b"Connection", b"keep-alive"],
+            [b"Keep-Alive", b"timeout=5"],
+            [b"Transfer-Encoding", b"chunked"],
+        ],
+        "status": 503,
+    }
+    return headers, body
+
+
+async def run_executable_http(scope: dict) -> tuple[dict, dict, str, Optional[bytes]]:
+    logger.debug("Calling localhost")
+
+    tries = 0
+    headers = None
+    body = None
+
+    timeout = aiohttp.ClientTimeout(total=5)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        while not body:
+            try:
+                tries += 1
+                headers, body = await make_request(session, scope)
+            except aiohttp.ClientConnectorError:
+                if tries > 20:
+                    headers, body = show_loading()
+                await asyncio.sleep(0.05)
+
+    output = ""  # Process stdout is not captured per request
+    output_data = None
+    logger.debug("Returning result")
+    return headers, body, output, output_data
+
+
+async def process_instruction(
+    instruction: bytes,
+    interface: Interface,
+    application: Union[ASGIApplication, subprocess.Popen],
+) -> AsyncIterable[bytes]:
+    if instruction == b"halt":
+        logger.info("Received halt command")
+        system("sync")
+        logger.debug("Filesystems synced")
+        if isinstance(application, subprocess.Popen):
+            application.terminate()
+            logger.debug("Application terminated")
+            # application.communicate()
+        else:
+            await wait_for_lifespan_event_completion(application=application, event="shutdown")
+        yield b"STOP\n"
+        logger.debug("Supervisor informed of halt")
+        raise ShutdownException
+    elif instruction.startswith(b"!"):
+        # Execute shell commands in the form `!ls /`
+        msg = instruction[1:].decode()
+        try:
+            process_output = subprocess.check_output(msg, stderr=subprocess.STDOUT, shell=True)
+            yield process_output
+        except subprocess.CalledProcessError as error:
+            yield str(error).encode() + b"\n" + error.output
+    else:
+        # Python
+        logger.debug("msgpack.loads (")
+        msg_ = msgpack.loads(instruction, raw=False)
+        logger.debug("msgpack.loads )")
+        payload = RunCodePayload(**msg_)
+
+        output: Optional[str] = None
+        try:
+            headers: dict
+            body: dict
+            output_data: Optional[bytes]
+
+            if interface == Interface.asgi:
+                application = cast(ASGIApplication, application)
+                headers, body, output, output_data = await run_python_code_http(
+                    application=application, scope=payload.scope
+                )
+            elif interface == Interface.executable:
+                headers, body, output, output_data = await run_executable_http(scope=payload.scope)
+            else:
+                raise ValueError("Unknown interface. This should never happen")
+
+            result = {
+                "headers": headers,
+                "body": body,
+                "output": output,
+                "output_data": output_data,
+            }
+            yield msgpack.dumps(result, use_bin_type=True)
+        except Exception as error:
+            yield msgpack.dumps(
+                {
+                    "error": str(error),
+                    "traceback": str(traceback.format_exc()),
+                    "output": output,
+                }
+            )
+
+
+def receive_data_length(client) -> int:
+    """Receive the length of the data to follow."""
+    buffer = b""
+    for _ in range(9):
+        byte = client.recv(1)
+        if byte == b"\n":
+            break
+        else:
+            buffer += byte
+    return int(buffer)
+
+
+def load_configuration(data: bytes) -> ConfigurationPayload:
+    msg_ = msgpack.loads(data, raw=False)
+    msg_["volumes"] = [Volume(**volume_dict) for volume_dict in msg_.get("volumes")]
+    return ConfigurationPayload(**msg_)
+
+
+def receive_config(client) -> ConfigurationPayload:
+    length = receive_data_length(client)
+    data = b""
+    while len(data) < length:
+        data += client.recv(1024 * 1024)
+    return load_configuration(data)
+
+
+def setup_system(config: ConfigurationPayload):
+    # Linux host names are limited to 63 characters. We therefore use the base32 representation
+    # of the item_hash instead of its common base16 representation.
+    item_hash_binary: bytes = base64.b16decode(config.vm_hash.encode().upper())
+    hostname = base64.b32encode(item_hash_binary).decode().strip("=").lower()
+    setup_hostname(hostname)
+
+    setup_variables(config.variables)
+    setup_volumes(config.volumes)
+    setup_network(
+        ipv4=config.ip,
+        ipv6=config.ipv6,
+        ipv4_gateway=config.route,
+        ipv6_gateway=config.ipv6_gateway,
+        dns_servers=config.dns_servers,
+    )
+    setup_input_data(config.input_data)
+    if authorized_keys := config.authorized_keys:
+        setup_authorized_keys(authorized_keys)
+    logger.debug("Setup finished")
+
+
+def umount_volumes(volumes: list[Volume]):
+    "Umount user related filesystems"
+    system("sync")
+    for volume in volumes:
+        logger.debug(f"Umounting /dev/{volume.device} on {volume.mount}")
+        system(f"umount {volume.mount}")
+
+
+async def main() -> None:
+    client, addr = s.accept()
+
+    logger.debug("Receiving setup...")
+    config = receive_config(client)
+    setup_system(config)
+
+    try:
+        app: Union[ASGIApplication, subprocess.Popen] = await setup_code(
+            config.code, config.encoding, config.entrypoint, config.interface
+        )
+        client.send(msgpack.dumps({"success": True}))
+    except Exception as error:
+        client.send(
+            msgpack.dumps(
+                {
+                    "success": False,
+                    "error": str(error),
+                    "traceback": str(traceback.format_exc()),
+                }
+            )
+        )
+        logger.exception("Program could not be started")
+        raise
+
+    class ServerReference:
+        "Reference used to close the server from within `handle_instruction"
+        server: asyncio.AbstractServer
+
+    server_reference = ServerReference()
+
+    async def handle_instruction(reader, writer):
+        data = await reader.read(1000_1000)  # Max 1 Mo
+
+        logger.debug("Init received msg")
+        if logger.level <= logging.DEBUG:
+            data_to_print = f"{data[:500]}..." if len(data) > 500 else data
+            logger.debug(f"<<<\n\n{data_to_print}\n\n>>>")
+
+        try:
+            async for result in process_instruction(instruction=data, interface=config.interface, application=app):
+                writer.write(result)
+                await writer.drain()
+
+                logger.debug("Instruction processed")
+        except ShutdownException:
+            logger.info("Initiating shutdown")
+            writer.write(b"STOPZ\n")
+            await writer.drain()
+            logger.debug("Shutdown confirmed to supervisor")
+            server_reference.server.close()
+            logger.debug("Supervisor socket server closed")
+        finally:
+            writer.close()
+
+    server = await asyncio.start_server(handle_instruction, sock=s)
+    server_reference.server = server
+
+    addr = server.sockets[0].getsockname()
+    print(f"Serving on {addr}")
+
+    try:
+        async with server:
+            await server.serve_forever()
+    except asyncio.CancelledError:
+        logger.debug("Server was properly cancelled")
+    finally:
+        logger.warning("System shutdown")
+        server.close()
+        logger.debug("Server closed")
+        umount_volumes(config.volumes)
+        logger.debug("User volumes unmounted")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
+    asyncio.run(main())
+
+    logger.info("Unmounting system filesystems")
+    system("umount /dev/shm")
+    system("umount /dev/pts")
+    system("umount -a")
+
+    logger.info("Sending reboot syscall")
+    # Send reboot syscall, see man page
+    # https://man7.org/linux/man-pages/man2/reboot.2.html
+    libc = ctypes.CDLL(None)
+    libc.syscall(169, 0xFEE1DEAD, 672274793, 0x4321FEDC, None)
+    # The exit should not happen due to system halt.
+    sys.exit(0)
diff --git a/runtimes/aleph-debian-12-python/loading.html b/runtimes/aleph-debian-12-python/loading.html
deleted file mode 120000
index 926fba036..000000000
--- a/runtimes/aleph-debian-12-python/loading.html
+++ /dev/null
@@ -1 +0,0 @@
-../aleph-debian-11-python/loading.html
\ No newline at end of file
diff --git a/runtimes/aleph-debian-12-python/loading.html b/runtimes/aleph-debian-12-python/loading.html
new file mode 100644
index 000000000..da9128c40
--- /dev/null
+++ b/runtimes/aleph-debian-12-python/loading.html
@@ -0,0 +1,346 @@
+
+
+  VM Loading
+  
+
+
+
+
+
              
+  
              
+    
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+    
+  
              
+  
              
+    
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
+      
+        
+          
+          
+        
+      
+    
+  
              
+  
              
+    
              
+      
              Whoops!
              
+      
              Seems like your VM is still loading, please wait...
              
+    
              
+
+    Refresh!
+  
              
+
+
              
+
+
diff --git a/runtimes/aleph-debian-12-python/update_inits.sh b/runtimes/aleph-debian-12-python/update_inits.sh
deleted file mode 120000
index 757431761..000000000
--- a/runtimes/aleph-debian-12-python/update_inits.sh
+++ /dev/null
@@ -1 +0,0 @@
-../aleph-debian-11-python/update_inits.sh
\ No newline at end of file
diff --git a/runtimes/aleph-debian-12-python/update_inits.sh b/runtimes/aleph-debian-12-python/update_inits.sh
new file mode 100755
index 000000000..55a1c99b1
--- /dev/null
+++ b/runtimes/aleph-debian-12-python/update_inits.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+rm ./rootfs.squashfs
+
+set -euf
+
+cp ./init0.sh ./rootfs/sbin/init
+cp ./init1.py ./rootfs/root/init1.py
+chmod +x ./rootfs/sbin/init
+chmod +x ./rootfs/root/init1.py
+
+mksquashfs ./rootfs/ ./rootfs.squashfs
+
+echo "OK"
diff --git a/runtimes/instance-rootfs/create-debian-11-disk.sh b/runtimes/instance-rootfs/create-debian-11-disk.sh
deleted file mode 100755
index 1ee49c8b0..000000000
--- a/runtimes/instance-rootfs/create-debian-11-disk.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/bash
-
-set -euf
-
-# Variables
-ROOTFS_FILE="./debian-11.btrfs"
-MOUNT_ORIGIN_DIR="/mnt/debian"
-MOUNT_DIR="/mnt/vm"
-IMAGE_URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-genericcloud-amd64.tar.xz"
-IMAGE_NAME="debian-11-genericcloud.tar.xz"
-IMAGE_RAW_NAME="disk.raw"
-
-# Cleanup previous run
-umount "$MOUNT_ORIGIN_DIR" || true
-umount "$MOUNT_DIR" || true
-rm -f "$ROOTFS_FILE"
-
-# Prepare directories
-mkdir -p "$MOUNT_ORIGIN_DIR"
-mkdir -p "$MOUNT_DIR"
-
-# Download Debian image
-echo "Downloading Debian 11 image"
-curl -L "$IMAGE_URL" -o "$IMAGE_NAME"
-
-# Allocate 1GB rootfs.btrfs file
-echo "Allocate 1GB $ROOTFS_FILE file"
-fallocate -l 1G "$ROOTFS_FILE"
-mkfs.btrfs -m single --label root "$ROOTFS_FILE"
-mount "$ROOTFS_FILE" "$MOUNT_DIR"
-
-# Extract Debian image
-echo "Extracting Debian 11 image"
-tar xvf "$IMAGE_NAME"
-
-# Mount first partition of Debian Image
-LOOPDISK=$(losetup --find --show $IMAGE_RAW_NAME)
-partx -u "$LOOPDISK"
-mount "$LOOPDISK"p1 "$MOUNT_ORIGIN_DIR"
-
-# Fix boot partition missing
-sed -i '$d' "$MOUNT_ORIGIN_DIR"/etc/fstab
-
-# Copy Debian image to rootfs
-echo "Copying Debian 11 image to $ROOTFS_FILE file"
-cp -vap "$MOUNT_ORIGIN_DIR/." "$MOUNT_DIR"
-
-# Cleanup and unmount
-umount "$MOUNT_ORIGIN_DIR"
-partx -d "$LOOPDISK"
-losetup -d "$LOOPDISK"
-umount "$MOUNT_DIR"
-
-rm "$IMAGE_RAW_NAME"
-rm "$IMAGE_NAME"
diff --git a/runtimes/instance-rootfs/create-debian-11-qemu-disk.sh b/runtimes/instance-rootfs/create-debian-11-qemu-disk.sh
deleted file mode 100755
index 59bef20b5..000000000
--- a/runtimes/instance-rootfs/create-debian-11-qemu-disk.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -euf
-
-# Variables
-ROOTFS_FILENAME="./rootfs.img"
-IMAGE_URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-genericcloud-amd64.qcow2"
-IMAGE_NAME="debian-11-genericcloud-amd64.qcow2"
-
-# Cleanup previous run
-rm -f "$ROOTFS_FILENAME"
-
-# Download Ubuntu image
-echo "Downloading Debian 11 image"
-curl -L "$IMAGE_URL" -o "$IMAGE_NAME"
-
-# Rename final file
-mv "$IMAGE_NAME" "$ROOTFS_FILENAME"

From c518253300b8b0c183d3919baf39b87f80abed99 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 3 Sep 2024 17:20:57 +0200
Subject: [PATCH 844/990] Fix: Automated fixes with `ruff check src --fix`

---
 src/aleph/vm/conf.py                          | 38 +++++++--------
 src/aleph/vm/controllers/__main__.py          |  3 +-
 src/aleph/vm/controllers/configuration.py     | 15 +++---
 .../vm/controllers/firecracker/executable.py  | 28 +++++------
 .../vm/controllers/firecracker/instance.py    | 13 +++--
 .../firecracker/snapshot_manager.py           |  3 +-
 .../vm/controllers/firecracker/snapshots.py   |  3 +-
 src/aleph/vm/controllers/interface.py         | 16 +++----
 src/aleph/vm/controllers/qemu/cloudinit.py    |  3 +-
 src/aleph/vm/controllers/qemu/instance.py     | 16 +++----
 .../controllers/qemu_confidential/instance.py | 10 ++--
 src/aleph/vm/guest_api/__main__.py            | 15 +++---
 .../vm/hypervisors/firecracker/config.py      |  5 +-
 .../vm/hypervisors/firecracker/microvm.py     | 20 ++++----
 src/aleph/vm/hypervisors/qemu/qemuvm.py       |  8 ++--
 .../hypervisors/qemu_confidential/qemuvm.py   |  1 -
 src/aleph/vm/models.py                        | 47 +++++++++----------
 src/aleph/vm/network/hostnetwork.py           |  8 ++--
 src/aleph/vm/network/interfaces.py            |  7 ++-
 src/aleph/vm/orchestrator/chain.py            | 11 ++---
 src/aleph/vm/orchestrator/cli.py              |  7 +--
 src/aleph/vm/orchestrator/payment.py          |  5 +-
 src/aleph/vm/orchestrator/pubsub.py           |  3 +-
 src/aleph/vm/orchestrator/resources.py        |  5 +-
 src/aleph/vm/orchestrator/run.py              | 12 ++---
 src/aleph/vm/orchestrator/supervisor.py       |  3 +-
 src/aleph/vm/orchestrator/views/__init__.py   |  4 +-
 .../vm/orchestrator/views/authentication.py   | 11 ++---
 .../vm/orchestrator/views/host_status.py      | 10 ++--
 src/aleph/vm/pool.py                          | 14 +++---
 src/aleph/vm/storage.py                       |  7 ++-
 src/aleph/vm/utils/__init__.py                | 10 ++--
 src/aleph/vm/utils/logs.py                    |  3 +-
 src/aleph/vm/version.py                       |  7 ++-
 34 files changed, 175 insertions(+), 196 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 3ef96127b..9c86b6014 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -9,7 +9,7 @@
 from os.path import abspath, exists, isdir, isfile, join
 from pathlib import Path
 from subprocess import CalledProcessError, check_output
-from typing import Any, Literal, NewType, Optional, Union
+from typing import Any, Literal, NewType
 
 from aleph_message.models import Chain
 from aleph_message.models.execution.environment import HypervisorType
@@ -17,7 +17,7 @@
 from pydantic.env_settings import DotenvType, env_file_sentinel
 from pydantic.typing import StrPath
 
-from aleph.vm.orchestrator.chain import STREAM_CHAINS, ChainInfo
+from aleph.vm.orchestrator.chain import STREAM_CHAINS
 from aleph.vm.utils import (
     check_amd_sev_es_supported,
     check_amd_sev_supported,
@@ -85,7 +85,7 @@ def resolvectl_dns_servers_ipv4(interface: str) -> Iterable[str]:
             yield server
 
 
-def get_default_interface() -> Optional[str]:
+def get_default_interface() -> str | None:
     """Returns the default network interface"""
     with open("/proc/net/route") as f:
         for line in f.readlines():
@@ -150,7 +150,7 @@ class Settings(BaseSettings):
 
     # Networking does not work inside Docker/Podman
     ALLOW_VM_NETWORKING = True
-    NETWORK_INTERFACE: Optional[str] = None
+    NETWORK_INTERFACE: str | None = None
     IPV4_ADDRESS_POOL = Field(
         default="172.16.0.0/12",
         description="IPv4 address range used to provide networks to VMs.",
@@ -179,8 +179,8 @@ class Settings(BaseSettings):
         description="Use the Neighbor Discovery Protocol Proxy to respond to Router Solicitation for instances on IPv6",
     )
 
-    DNS_RESOLUTION: Optional[DnsResolver] = DnsResolver.detect
-    DNS_NAMESERVERS: Optional[list[str]] = None
+    DNS_RESOLUTION: DnsResolver | None = DnsResolver.detect
+    DNS_NAMESERVERS: list[str] | None = None
 
     FIRECRACKER_PATH = Path("/opt/firecracker/firecracker")
     JAILER_PATH = Path("/opt/firecracker/jailer")
@@ -259,7 +259,7 @@ class Settings(BaseSettings):
     ALLOCATION_TOKEN_HASH = "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda"
 
     ENABLE_QEMU_SUPPORT: bool = Field(default=True)
-    INSTANCE_DEFAULT_HYPERVISOR: Optional[HypervisorType] = Field(
+    INSTANCE_DEFAULT_HYPERVISOR: HypervisorType | None = Field(
         default=HypervisorType.firecracker,  # User Firecracker
         description="Default hypervisor to use on running instances, can be Firecracker or QEmu",
     )
@@ -279,19 +279,17 @@ class Settings(BaseSettings):
 
     # Tests on programs
 
-    FAKE_DATA_PROGRAM: Optional[Path] = None
+    FAKE_DATA_PROGRAM: Path | None = None
     BENCHMARK_FAKE_DATA_PROGRAM = Path(abspath(join(__file__, "../../../../examples/example_fastapi")))
 
     FAKE_DATA_MESSAGE = Path(abspath(join(__file__, "../../../../examples/program_message_from_aleph.json")))
-    FAKE_DATA_DATA: Optional[Path] = Path(abspath(join(__file__, "../../../../examples/data/")))
+    FAKE_DATA_DATA: Path | None = Path(abspath(join(__file__, "../../../../examples/data/")))
     FAKE_DATA_RUNTIME = Path(abspath(join(__file__, "../../../../runtimes/aleph-debian-12-python/rootfs.squashfs")))
-    FAKE_DATA_VOLUME: Optional[Path] = Path(
-        abspath(join(__file__, "../../../../examples/volumes/volume-venv.squashfs"))
-    )
+    FAKE_DATA_VOLUME: Path | None = Path(abspath(join(__file__, "../../../../examples/volumes/volume-venv.squashfs")))
 
     # Tests on instances
 
-    TEST_INSTANCE_ID: Optional[str] = Field(
+    TEST_INSTANCE_ID: str | None = Field(
         default=None,  # TODO: Use a valid item_hash here
         description="Identifier of the instance message used when testing the launch of an instance from the network",
     )
@@ -312,11 +310,11 @@ class Settings(BaseSettings):
 
     # Developer options
 
-    SENTRY_DSN: Optional[str] = None
+    SENTRY_DSN: str | None = None
     SENTRY_TRACES_SAMPLE_RATE: float = Field(ge=0, le=1.0, default=0.1)
-    DEVELOPER_SSH_KEYS: Optional[list[str]] = []
+    DEVELOPER_SSH_KEYS: list[str] | None = []
     # Using an object here forces the value to come from Python code and not from an environment variable.
-    USE_DEVELOPER_SSH_KEYS: Union[Literal[False], object] = False
+    USE_DEVELOPER_SSH_KEYS: Literal[False] | object = False
 
     # Fields
     SENSITIVE_FIELDS: list[str] = Field(
@@ -461,10 +459,10 @@ def display(self) -> str:
 
     def __init__(
         self,
-        _env_file: Optional[DotenvType] = env_file_sentinel,
-        _env_file_encoding: Optional[str] = None,
-        _env_nested_delimiter: Optional[str] = None,
-        _secrets_dir: Optional[StrPath] = None,
+        _env_file: DotenvType | None = env_file_sentinel,
+        _env_file_encoding: str | None = None,
+        _env_nested_delimiter: str | None = None,
+        _secrets_dir: StrPath | None = None,
         **values: Any,
     ) -> None:
         super().__init__(_env_file, _env_file_encoding, _env_nested_delimiter, _secrets_dir, **values)
diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index 90475086f..19701c4bf 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -6,7 +6,6 @@
 import sys
 from asyncio.subprocess import Process
 from pathlib import Path
-from typing import Union
 
 from aleph.vm.hypervisors.firecracker.microvm import MicroVM
 from aleph.vm.hypervisors.qemu.qemuvm import QemuVM
@@ -85,7 +84,7 @@ async def execute_persistent_vm(config: Configuration):
     return execution, process
 
 
-async def handle_persistent_vm(config: Configuration, execution: Union[MicroVM, QemuVM], process: Process):
+async def handle_persistent_vm(config: Configuration, execution: MicroVM | QemuVM, process: Process):
     # Catch the terminating signal and send a proper message to the vm to stop it so it close files properly
     loop = asyncio.get_event_loop()
 
diff --git a/src/aleph/vm/controllers/configuration.py b/src/aleph/vm/controllers/configuration.py
index 32f869bea..da10d8395 100644
--- a/src/aleph/vm/controllers/configuration.py
+++ b/src/aleph/vm/controllers/configuration.py
@@ -1,7 +1,6 @@
 import logging
 from enum import Enum
 from pathlib import Path
-from typing import List, Optional, Union
 
 from pydantic import BaseModel
 
@@ -26,26 +25,26 @@ class QemuVMHostVolume(BaseModel):
 
 class QemuVMConfiguration(BaseModel):
     qemu_bin_path: str
-    cloud_init_drive_path: Optional[str]
+    cloud_init_drive_path: str | None
     image_path: str
     monitor_socket_path: Path
     qmp_socket_path: Path
     vcpu_count: int
     mem_size_mb: int
-    interface_name: Optional[str]
-    host_volumes: List[QemuVMHostVolume]
+    interface_name: str | None
+    host_volumes: list[QemuVMHostVolume]
 
 
 class QemuConfidentialVMConfiguration(BaseModel):
     qemu_bin_path: str
-    cloud_init_drive_path: Optional[str]
+    cloud_init_drive_path: str | None
     image_path: str
     monitor_socket_path: Path
     qmp_socket_path: Path
     vcpu_count: int
     mem_size_mb: int
-    interface_name: Optional[str]
-    host_volumes: List[QemuVMHostVolume]
+    interface_name: str | None
+    host_volumes: list[QemuVMHostVolume]
     ovmf_path: Path
     sev_session_file: Path
     sev_dh_cert_file: Path
@@ -61,7 +60,7 @@ class Configuration(BaseModel):
     vm_id: int
     vm_hash: str
     settings: Settings
-    vm_configuration: Union[QemuConfidentialVMConfiguration, QemuVMConfiguration, VMConfiguration]
+    vm_configuration: QemuConfidentialVMConfiguration | QemuVMConfiguration | VMConfiguration
     hypervisor: HypervisorType = HypervisorType.firecracker
 
 
diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index 1d9fe6360..cbbad03c6 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -8,7 +8,7 @@
 from multiprocessing import Process, set_start_method
 from os.path import exists, isfile
 from pathlib import Path
-from typing import Generic, Optional, TypeVar
+from typing import Generic, TypeVar
 
 from aiohttp import ClientResponseError
 from aleph_message.models import ExecutableContent, ItemHash
@@ -75,18 +75,18 @@ class HostVolume:
 @dataclass
 class BaseConfiguration:
     vm_hash: ItemHash
-    ip: Optional[str] = None
-    route: Optional[str] = None
+    ip: str | None = None
+    route: str | None = None
     dns_servers: list[str] = field(default_factory=list)
     volumes: list[Volume] = field(default_factory=list)
-    variables: Optional[dict[str, str]] = None
+    variables: dict[str, str] | None = None
 
 
 @dataclass
 class ConfigurationResponse:
     success: bool
-    error: Optional[str] = None
-    traceback: Optional[str] = None
+    error: str | None = None
+    traceback: str | None = None
 
 
 class AlephFirecrackerResources:
@@ -149,14 +149,14 @@ class AlephFirecrackerExecutable(Generic[ConfigurationType], AlephVmControllerIn
     enable_console: bool
     enable_networking: bool
     hardware_resources: MachineResources
-    tap_interface: Optional[TapInterface] = None
+    tap_interface: TapInterface | None = None
     fvm: MicroVM
-    vm_configuration: Optional[ConfigurationType]
-    guest_api_process: Optional[Process] = None
+    vm_configuration: ConfigurationType | None
+    guest_api_process: Process | None = None
     is_instance: bool
     persistent: bool
-    _firecracker_config: Optional[FirecrackerConfig] = None
-    controller_configuration: Optional[Configuration] = None
+    _firecracker_config: FirecrackerConfig | None = None
+    controller_configuration: Configuration | None = None
     support_snapshot: bool
 
     @property
@@ -169,9 +169,9 @@ def __init__(
         vm_hash: ItemHash,
         resources: AlephFirecrackerResources,
         enable_networking: bool = False,
-        enable_console: Optional[bool] = None,
-        hardware_resources: Optional[MachineResources] = None,
-        tap_interface: Optional[TapInterface] = None,
+        enable_console: bool | None = None,
+        hardware_resources: MachineResources | None = None,
+        tap_interface: TapInterface | None = None,
         persistent: bool = False,
         prepare_jailer: bool = True,
     ):
diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index 4f936b7bb..f8c33b075 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -4,7 +4,6 @@
 import logging
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Optional, Union
 
 import yaml
 from aleph_message.models import ItemHash
@@ -56,7 +55,7 @@ async def download_all(self):
 class AlephFirecrackerInstance(AlephFirecrackerExecutable):
     vm_configuration: BaseConfiguration
     resources: AlephInstanceResources
-    latest_snapshot: Optional[DiskVolumeSnapshot]
+    latest_snapshot: DiskVolumeSnapshot | None
     is_instance = True
     support_snapshot = False
 
@@ -66,9 +65,9 @@ def __init__(
         vm_hash: ItemHash,
         resources: AlephInstanceResources,
         enable_networking: bool = False,
-        enable_console: Optional[bool] = None,
-        hardware_resources: Optional[MachineResources] = None,
-        tap_interface: Optional[TapInterface] = None,
+        enable_console: bool | None = None,
+        hardware_resources: MachineResources | None = None,
+        tap_interface: TapInterface | None = None,
         prepare_jailer: bool = True,
     ):
         self.latest_snapshot = None
@@ -169,13 +168,13 @@ def _get_hostname(self) -> str:
     def _encode_user_data(self) -> bytes:
         """Creates user data configuration file for cloud-init tool"""
 
-        ssh_authorized_keys: Optional[list[str]]
+        ssh_authorized_keys: list[str] | None
         if settings.USE_DEVELOPER_SSH_KEYS:
             ssh_authorized_keys = settings.DEVELOPER_SSH_KEYS or []
         else:
             ssh_authorized_keys = self.resources.message_content.authorized_keys or []
 
-        config: dict[str, Union[str, bool, list[str]]] = {
+        config: dict[str, str | bool | list[str]] = {
             "hostname": self._get_hostname(),
             "disable_root": False,
             "ssh_pwauth": False,
diff --git a/src/aleph/vm/controllers/firecracker/snapshot_manager.py b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
index d5f81d1eb..6cf36711c 100644
--- a/src/aleph/vm/controllers/firecracker/snapshot_manager.py
+++ b/src/aleph/vm/controllers/firecracker/snapshot_manager.py
@@ -2,7 +2,6 @@
 import logging
 import threading
 from time import sleep
-from typing import Optional
 
 from aleph_message.models import ItemHash
 from schedule import Job, Scheduler
@@ -95,7 +94,7 @@ def run_in_thread(self) -> None:
         )
         job_thread.start()
 
-    async def start_for(self, vm: AlephFirecrackerExecutable, frequency: Optional[int] = None) -> None:
+    async def start_for(self, vm: AlephFirecrackerExecutable, frequency: int | None = None) -> None:
         if not vm.support_snapshot:
             msg = "Snapshots are not implemented for programs."
             raise NotImplementedError(msg)
diff --git a/src/aleph/vm/controllers/firecracker/snapshots.py b/src/aleph/vm/controllers/firecracker/snapshots.py
index f1bb24436..7188bfb6d 100644
--- a/src/aleph/vm/controllers/firecracker/snapshots.py
+++ b/src/aleph/vm/controllers/firecracker/snapshots.py
@@ -1,6 +1,5 @@
 import logging
 from pathlib import Path
-from typing import Optional
 
 from aleph_message.models import ItemHash
 
@@ -35,7 +34,7 @@ async def upload(self) -> ItemHash:
 
 
 class DiskVolumeSnapshot(DiskVolumeFile):
-    compressed: Optional[CompressedDiskVolumeSnapshot]
+    compressed: CompressedDiskVolumeSnapshot | None
 
     def delete(self) -> None:
         if self.compressed:
diff --git a/src/aleph/vm/controllers/interface.py b/src/aleph/vm/controllers/interface.py
index d5a290173..bff265a2b 100644
--- a/src/aleph/vm/controllers/interface.py
+++ b/src/aleph/vm/controllers/interface.py
@@ -2,8 +2,8 @@
 import logging
 from abc import ABC
 from asyncio.subprocess import Process
-from collections.abc import Coroutine
-from typing import Any, Callable, Optional
+from collections.abc import Callable, Coroutine
+from typing import Any
 
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import MachineResources
@@ -31,26 +31,26 @@ class AlephVmControllerInterface(ABC):
     hardware_resources: MachineResources
     support_snapshot: bool
     """Does this controller support snapshotting"""
-    guest_api_process: Optional[Process] = None
-    tap_interface: Optional[TapInterface] = None
+    guest_api_process: Process | None = None
+    tap_interface: TapInterface | None = None
     """Network interface used for this VM"""
 
-    def get_ip(self) -> Optional[str]:
+    def get_ip(self) -> str | None:
         if self.tap_interface:
             return self.tap_interface.guest_ip.with_prefixlen
         return None
 
-    def get_ip_route(self) -> Optional[str]:
+    def get_ip_route(self) -> str | None:
         if self.tap_interface:
             return str(self.tap_interface.host_ip).split("/", 1)[0]
         return None
 
-    def get_ipv6(self) -> Optional[str]:
+    def get_ipv6(self) -> str | None:
         if self.tap_interface:
             return self.tap_interface.guest_ipv6.with_prefixlen
         return None
 
-    def get_ipv6_gateway(self) -> Optional[str]:
+    def get_ipv6_gateway(self) -> str | None:
         if self.tap_interface:
             return str(self.tap_interface.host_ipv6.ip)
         return None
diff --git a/src/aleph/vm/controllers/qemu/cloudinit.py b/src/aleph/vm/controllers/qemu/cloudinit.py
index 686abac31..bcfa51c0f 100644
--- a/src/aleph/vm/controllers/qemu/cloudinit.py
+++ b/src/aleph/vm/controllers/qemu/cloudinit.py
@@ -17,7 +17,6 @@
 import json
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Union
 
 import yaml
 from aleph_message.models import ItemHash
@@ -35,7 +34,7 @@ def get_hostname_from_hash(vm_hash: ItemHash) -> str:
 
 def encode_user_data(hostname, ssh_authorized_keys) -> bytes:
     """Creates user data configuration file for cloud-init tool"""
-    config: dict[str, Union[str, bool, list[str]]] = {
+    config: dict[str, str | bool | list[str]] = {
         "hostname": hostname,
         "disable_root": False,
         "ssh_pwauth": False,
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index 3f4f5ba3d..3ca138903 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -5,7 +5,7 @@
 from asyncio import Task
 from asyncio.subprocess import Process
 from pathlib import Path
-from typing import Generic, Optional, TypeVar, Union
+from typing import Generic, TypeVar
 
 import psutil
 from aleph_message.models import ItemHash
@@ -47,9 +47,9 @@ async def download_all(self):
             self.download_volumes(),
         )
 
-    async def make_writable_volume(self, parent_image_path, volume: Union[PersistentVolume, RootfsVolume]):
+    async def make_writable_volume(self, parent_image_path, volume: PersistentVolume | RootfsVolume):
         """Create a new qcow2 image file based on the passed one, that we give to the VM to write onto"""
-        qemu_img_path: Optional[str] = shutil.which("qemu-img")
+        qemu_img_path: str | None = shutil.which("qemu-img")
         if not qemu_img_path:
             raise VmSetupError("qemu-img not found in PATH")
 
@@ -98,10 +98,10 @@ class AlephQemuInstance(Generic[ConfigurationType], CloudInitMixin, AlephVmContr
     resources: AlephQemuResources
     enable_networking: bool
     hardware_resources: MachineResources
-    tap_interface: Optional[TapInterface] = None
-    vm_configuration: Optional[ConfigurationType]
+    tap_interface: TapInterface | None = None
+    vm_configuration: ConfigurationType | None
     is_instance: bool
-    qemu_process: Optional[Process]
+    qemu_process: Process | None
     support_snapshot = False
     persistent = True
     controller_configuration: Configuration
@@ -119,7 +119,7 @@ def __init__(
         resources: AlephQemuResources,
         enable_networking: bool = False,
         hardware_resources: MachineResources = MachineResources(),
-        tap_interface: Optional[TapInterface] = None,
+        tap_interface: TapInterface | None = None,
     ):
         self.vm_id = vm_id
         self.vm_hash = vm_hash
@@ -253,7 +253,7 @@ async def start_guest_api(self):
     async def stop_guest_api(self):
         pass
 
-    print_task: Optional[Task] = None
+    print_task: Task | None = None
 
     async def teardown(self):
         if self.print_task:
diff --git a/src/aleph/vm/controllers/qemu_confidential/instance.py b/src/aleph/vm/controllers/qemu_confidential/instance.py
index 2b22044ec..f432cff69 100644
--- a/src/aleph/vm/controllers/qemu_confidential/instance.py
+++ b/src/aleph/vm/controllers/qemu_confidential/instance.py
@@ -2,8 +2,8 @@
 import logging
 import shutil
 from asyncio.subprocess import Process
+from collections.abc import Callable
 from pathlib import Path
-from typing import Callable, Optional
 
 from aleph_message.models import ItemHash
 from aleph_message.models.execution.environment import AMDSEVPolicy, MachineResources
@@ -50,10 +50,10 @@ class AlephQemuConfidentialInstance(AlephQemuInstance):
     enable_console: bool
     enable_networking: bool
     hardware_resources: MachineResources
-    tap_interface: Optional[TapInterface] = None
-    vm_configuration: Optional[ConfigurationType]
+    tap_interface: TapInterface | None = None
+    vm_configuration: ConfigurationType | None
     is_instance: bool
-    qemu_process: Optional[Process]
+    qemu_process: Process | None
     support_snapshot = False
     persistent = True
     _queue_cancellers: dict[asyncio.Queue, Callable] = {}
@@ -74,7 +74,7 @@ def __init__(
         enable_networking: bool = False,
         confidential_policy: int = AMDSEVPolicy.NO_DBG,
         hardware_resources: MachineResources = MachineResources(),
-        tap_interface: Optional[TapInterface] = None,
+        tap_interface: TapInterface | None = None,
     ):
         super().__init__(vm_id, vm_hash, resources, enable_networking, hardware_resources, tap_interface)
         self.confidential_policy = confidential_policy
diff --git a/src/aleph/vm/guest_api/__main__.py b/src/aleph/vm/guest_api/__main__.py
index 1d35997dd..8000d52bc 100644
--- a/src/aleph/vm/guest_api/__main__.py
+++ b/src/aleph/vm/guest_api/__main__.py
@@ -2,7 +2,6 @@
 import logging
 import re
 from pathlib import Path
-from typing import Optional
 
 import aiohttp
 import aioredis
@@ -20,7 +19,7 @@
 CACHE_EXPIRES_AFTER = 7 * 24 * 3600  # Seconds
 REDIS_ADDRESS = "redis://localhost"
 
-_redis: Optional[aioredis.Redis] = None
+_redis: aioredis.Redis | None = None
 
 
 async def get_redis(address: str = REDIS_ADDRESS) -> aioredis.Redis:
@@ -105,7 +104,7 @@ async def sign(request: web.Request):
 
 async def get_from_cache(request: web.Request):
     prefix: str = request.app["meta_vm_hash"]
-    key: Optional[str] = request.match_info.get("key")
+    key: str | None = request.match_info.get("key")
     if not (key and re.match(r"^\w+$", key)):
         return web.HTTPBadRequest(text="Invalid key")
 
@@ -119,7 +118,7 @@ async def get_from_cache(request: web.Request):
 
 async def put_in_cache(request: web.Request):
     prefix: str = request.app["meta_vm_hash"]
-    key: Optional[str] = request.match_info.get("key")
+    key: str | None = request.match_info.get("key")
     if not (key and re.match(r"^\w+$", key)):
         return web.HTTPBadRequest(text="Invalid key")
 
@@ -131,7 +130,7 @@ async def put_in_cache(request: web.Request):
 
 async def delete_from_cache(request: web.Request):
     prefix: str = request.app["meta_vm_hash"]
-    key: Optional[str] = request.match_info.get("key")
+    key: str | None = request.match_info.get("key")
     if not (key and re.match(r"^\w+$", key)):
         return web.HTTPBadRequest(text="Invalid key")
 
@@ -154,9 +153,9 @@ async def list_keys_from_cache(request: web.Request):
 
 def run_guest_api(
     unix_socket_path: Path,
-    vm_hash: Optional[str] = None,
-    sentry_dsn: Optional[str] = None,
-    server_name: Optional[str] = None,
+    vm_hash: str | None = None,
+    sentry_dsn: str | None = None,
+    server_name: str | None = None,
 ):
     # This function runs in a separate process, requiring to reinitialize the Sentry SDK
     if sentry_sdk and sentry_dsn:
diff --git a/src/aleph/vm/hypervisors/firecracker/config.py b/src/aleph/vm/hypervisors/firecracker/config.py
index 27aa0e8ea..b7e4fc77a 100644
--- a/src/aleph/vm/hypervisors/firecracker/config.py
+++ b/src/aleph/vm/hypervisors/firecracker/config.py
@@ -1,5 +1,4 @@
 from pathlib import Path
-from typing import Optional
 
 from pydantic import BaseModel, PositiveInt
 
@@ -52,8 +51,8 @@ class FirecrackerConfig(BaseModel):
     boot_source: BootSource
     drives: list[Drive]
     machine_config: MachineConfig
-    vsock: Optional[Vsock]
-    network_interfaces: Optional[list[NetworkInterface]]
+    vsock: Vsock | None
+    network_interfaces: list[NetworkInterface] | None
 
     class Config:
         allow_population_by_field_name = True
diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index d85b80071..5590315c6 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -14,7 +14,7 @@
 from pathlib import Path
 from pwd import getpwnam
 from tempfile import NamedTemporaryFile
-from typing import Any, Optional, TextIO
+from typing import Any
 
 import msgpack
 from aleph_message.models import ItemHash
@@ -83,16 +83,16 @@ class MicroVM:
     vm_id: int
     use_jailer: bool
     firecracker_bin_path: Path
-    jailer_bin_path: Optional[Path]
-    proc: Optional[asyncio.subprocess.Process] = None
-    stdout_task: Optional[Task] = None
-    stderr_task: Optional[Task] = None
-    config_file_path: Optional[Path] = None
+    jailer_bin_path: Path | None
+    proc: asyncio.subprocess.Process | None = None
+    stdout_task: Task | None = None
+    stderr_task: Task | None = None
+    config_file_path: Path | None = None
     drives: list[Drive]
     init_timeout: float
-    runtime_config: Optional[RuntimeConfiguration]
-    mounted_rootfs: Optional[Path] = None
-    _unix_socket: Optional[Server] = None
+    runtime_config: RuntimeConfiguration | None
+    mounted_rootfs: Path | None = None
+    _unix_socket: Server | None = None
     enable_log: bool
 
     def __repr__(self):
@@ -131,7 +131,7 @@ def __init__(
         firecracker_bin_path: Path,
         jailer_base_directory: Path,
         use_jailer: bool = True,
-        jailer_bin_path: Optional[Path] = None,
+        jailer_bin_path: Path | None = None,
         init_timeout: float = 5.0,
         enable_log: bool = True,
     ):
diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index 53518eb0c..1d707c2a5 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -2,7 +2,7 @@
 from asyncio.subprocess import Process
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional, TextIO
+from typing import TextIO
 
 import qmp
 from systemd import journal
@@ -19,14 +19,14 @@ class HostVolume:
 
 class QemuVM:
     qemu_bin_path: str
-    cloud_init_drive_path: Optional[str]
+    cloud_init_drive_path: str | None
     image_path: str
     monitor_socket_path: Path
     qmp_socket_path: Path
     vcpu_count: int
     mem_size_mb: int
     interface_name: str
-    qemu_process: Optional[Process] = None
+    qemu_process: Process | None = None
     host_volumes: list[HostVolume]
 
     def __repr__(self) -> str:
@@ -129,7 +129,7 @@ async def start(
         )
         return proc
 
-    def _get_qmpclient(self) -> Optional[qmp.QEMUMonitorProtocol]:
+    def _get_qmpclient(self) -> qmp.QEMUMonitorProtocol | None:
         if not (self.qmp_socket_path and self.qmp_socket_path.exists()):
             return None
         client = qmp.QEMUMonitorProtocol(str(self.qmp_socket_path))
diff --git a/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
index 6b76f62f3..4b472e4ce 100644
--- a/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
@@ -8,7 +8,6 @@
 from systemd import journal
 
 from aleph.vm.controllers.configuration import QemuConfidentialVMConfiguration
-from aleph.vm.controllers.qemu.instance import logger
 from aleph.vm.hypervisors.qemu.qemuvm import QemuVM
 
 
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 42bc8c70e..27db9ec6b 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -2,10 +2,9 @@
 import logging
 import uuid
 from asyncio import Task
-from collections.abc import Coroutine
+from collections.abc import Callable, Coroutine
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Callable, Optional, Union
 
 from aleph_message.models import (
     ExecutableContent,
@@ -48,12 +47,12 @@
 @dataclass
 class VmExecutionTimes:
     defined_at: datetime
-    preparing_at: Optional[datetime] = None
-    prepared_at: Optional[datetime] = None
-    starting_at: Optional[datetime] = None
-    started_at: Optional[datetime] = None
-    stopping_at: Optional[datetime] = None
-    stopped_at: Optional[datetime] = None
+    preparing_at: datetime | None = None
+    prepared_at: datetime | None = None
+    starting_at: datetime | None = None
+    started_at: datetime | None = None
+    stopping_at: datetime | None = None
+    stopped_at: datetime | None = None
 
     def to_dict(self):
         return self.__dict__
@@ -70,8 +69,8 @@ class VmExecution:
     vm_hash: ItemHash
     original: ExecutableContent
     message: ExecutableContent
-    resources: Optional[AlephFirecrackerResources] = None
-    vm: Optional[Union[AlephFirecrackerExecutable, AlephQemuInstance]] = None
+    resources: AlephFirecrackerResources | None = None
+    vm: AlephFirecrackerExecutable | AlephQemuInstance | None = None
 
     times: VmExecutionTimes
 
@@ -80,11 +79,11 @@ class VmExecution:
     runs_done_event: asyncio.Event
     stop_pending_lock: asyncio.Lock
     stop_event: asyncio.Event
-    expire_task: Optional[asyncio.Task] = None
-    update_task: Optional[asyncio.Task] = None
+    expire_task: asyncio.Task | None = None
+    update_task: asyncio.Task | None = None
 
-    snapshot_manager: Optional[SnapshotManager]
-    systemd_manager: Optional[SystemDManager]
+    snapshot_manager: SnapshotManager | None
+    systemd_manager: SystemDManager | None
 
     persistent: bool = False
 
@@ -126,7 +125,7 @@ def becomes_ready(self) -> Callable[[], Coroutine]:
         return self.ready_event.wait
 
     @property
-    def vm_id(self) -> Optional[int]:
+    def vm_id(self) -> int | None:
         return self.vm.vm_id if self.vm else None
 
     @property
@@ -151,8 +150,8 @@ def __init__(
         vm_hash: ItemHash,
         message: ExecutableContent,
         original: ExecutableContent,
-        snapshot_manager: Optional[SnapshotManager],
-        systemd_manager: Optional[SystemDManager],
+        snapshot_manager: SnapshotManager | None,
+        systemd_manager: SystemDManager | None,
         persistent: bool,
     ):
         self.uuid = uuid.uuid1()  # uuid1() includes the hardware address and timestamp
@@ -176,7 +175,7 @@ def to_dict(self) -> dict:
             **self.__dict__,
         }
 
-    def to_json(self, indent: Optional[int] = None) -> str:
+    def to_json(self, indent: int | None = None) -> str:
         return dumps_for_json(self.to_dict(), indent=indent)
 
     async def prepare(self) -> None:
@@ -187,9 +186,9 @@ async def prepare(self) -> None:
                 return
 
             self.times.preparing_at = datetime.now(tz=timezone.utc)
-            resources: Union[
-                AlephProgramResources, AlephInstanceResources, AlephQemuResources, AlephQemuConfidentialInstance
-            ]
+            resources: (
+                AlephProgramResources | AlephInstanceResources | AlephQemuResources | AlephQemuConfidentialInstance
+            )
             if self.is_program:
                 resources = AlephProgramResources(self.message, namespace=self.vm_hash)
             elif self.is_instance:
@@ -213,7 +212,7 @@ async def prepare(self) -> None:
             self.resources = resources
 
     def create(
-        self, vm_id: int, tap_interface: Optional[TapInterface] = None, prepare: bool = True
+        self, vm_id: int, tap_interface: TapInterface | None = None, prepare: bool = True
     ) -> AlephVmControllerInterface:
         if not self.resources:
             msg = "Execution resources must be configured first"
@@ -296,7 +295,7 @@ async def wait_for_init(self):
         assert self.vm, "The VM attribute has to be set before calling wait_for_init()"
         await self.vm.wait_for_init()
 
-    def stop_after_timeout(self, timeout: float = 5.0) -> Optional[Task]:
+    def stop_after_timeout(self, timeout: float = 5.0) -> Task | None:
         if self.persistent:
             logger.debug("VM marked as long running. Ignoring timeout.")
             return None
@@ -439,7 +438,7 @@ async def record_usage(self):
         if settings.EXECUTION_LOG_ENABLED:
             await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json())
 
-    async def run_code(self, scope: Optional[dict] = None) -> bytes:
+    async def run_code(self, scope: dict | None = None) -> bytes:
         if not self.vm:
             msg = "The VM has not been created yet"
             raise ValueError(msg)
diff --git a/src/aleph/vm/network/hostnetwork.py b/src/aleph/vm/network/hostnetwork.py
index c2965e466..dee3869a6 100644
--- a/src/aleph/vm/network/hostnetwork.py
+++ b/src/aleph/vm/network/hostnetwork.py
@@ -1,7 +1,7 @@
 import logging
 from ipaddress import IPv6Network
 from pathlib import Path
-from typing import Optional, Protocol
+from typing import Protocol
 
 import pyroute2
 from aleph_message.models import ItemHash
@@ -105,8 +105,8 @@ def make_ipv6_allocator(
 
 
 class Network:
-    ipv4_forward_state_before_setup: Optional[int] = None
-    ipv6_forward_state_before_setup: Optional[int] = None
+    ipv4_forward_state_before_setup: int | None = None
+    ipv6_forward_state_before_setup: int | None = None
     external_interface: str
     ipv4_forwarding_enabled: bool
     ipv6_forwarding_enabled: bool
@@ -114,7 +114,7 @@ class Network:
     ipv4_address_pool: IPv4NetworkWithInterfaces = IPv4NetworkWithInterfaces("172.16.0.0/12")
     ipv6_address_pool: IPv6Network
     network_size: int
-    ndp_proxy: Optional[NdpProxy] = None
+    ndp_proxy: NdpProxy | None = None
 
     IPV6_SUBNET_PREFIX: int = 124
 
diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index b3fc14029..9171f25fe 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -3,7 +3,6 @@
 import logging
 import shutil
 from ipaddress import IPv4Interface, IPv6Interface, IPv6Network
-from typing import Optional, Union
 
 from pyroute2 import IPRoute, NetlinkError
 
@@ -44,7 +43,7 @@ def create_tap_interface(ipr: IPRoute, device_name: str):
             logger.error(f"Unknown exception while creating interface {device_name}: {error}")
 
 
-def add_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6Interface]):
+def add_ip_address(ipr: IPRoute, device_name: str, ip: IPv4Interface | IPv6Interface):
     """Add an IP address to the given interface. If the address already exists, a warning is logged and the function
     returns without error."""
     interface_index: list[int] = ipr.link_lookup(ifname=device_name)
@@ -61,7 +60,7 @@ def add_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6
         logger.error(f"Unknown exception while adding address {ip} to interface {device_name}: {e}")
 
 
-def delete_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6Interface]):
+def delete_ip_address(ipr: IPRoute, device_name: str, ip: IPv4Interface | IPv6Interface):
     """Delete an IP address to the given interface."""
     interface_index: list[int] = ipr.link_lookup(ifname=device_name)
     if not interface_index:
@@ -110,7 +109,7 @@ def __init__(
         device_name: str,
         ip_network: IPv4NetworkWithInterfaces,
         ipv6_network: IPv6Network,
-        ndp_proxy: Optional[NdpProxy],
+        ndp_proxy: NdpProxy | None,
     ):
         self.device_name: str = device_name
         self.ip_network: IPv4NetworkWithInterfaces = ip_network
diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py
index 2cedd8162..9a8db1df9 100644
--- a/src/aleph/vm/orchestrator/chain.py
+++ b/src/aleph/vm/orchestrator/chain.py
@@ -1,5 +1,4 @@
 import logging
-from typing import Dict, Optional, Union
 
 from aleph_message.models import Chain
 from pydantic import BaseModel, root_validator
@@ -14,13 +13,13 @@ class ChainInfo(BaseModel):
 
     chain_id: int
     rpc: str
-    standard_token: Optional[str] = None
-    super_token: Optional[str] = None
+    standard_token: str | None = None
+    super_token: str | None = None
     testnet: bool = False
     active: bool = True
 
     @property
-    def token(self) -> Optional[str]:
+    def token(self) -> str | None:
         return self.super_token or self.standard_token
 
     @root_validator(pre=True)
@@ -30,7 +29,7 @@ def check_tokens(cls, values):
         return values
 
 
-STREAM_CHAINS: Dict[Union[Chain, str], ChainInfo] = {
+STREAM_CHAINS: dict[Chain | str, ChainInfo] = {
     # TESTNETS
     "SEPOLIA": ChainInfo(
         chain_id=11155111,
@@ -63,5 +62,5 @@ def check_tokens(cls, values):
 def get_chain(chain: str) -> ChainInfo:
     try:
         return STREAM_CHAINS[chain]
-    except KeyError as error:
+    except KeyError:
         raise ValueError(f"Unknown chain id for chain {chain}")
diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index 65b290ba2..e0d11298a 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -5,9 +5,10 @@
 import os
 import sys
 import time
+from collections.abc import Callable
 from pathlib import Path
 from statistics import mean
-from typing import Callable, Optional, cast
+from typing import cast
 
 import alembic.command
 import alembic.config
@@ -238,7 +239,7 @@ async def fake_read() -> bytes:
     print("Event result", result)
 
 
-async def start_instance(item_hash: ItemHash, pubsub: Optional[PubSub], pool) -> VmExecution:
+async def start_instance(item_hash: ItemHash, pubsub: PubSub | None, pool) -> VmExecution:
     """Run an instance from an InstanceMessage."""
     return await start_persistent_vm(item_hash, pubsub, pool)
 
@@ -251,7 +252,7 @@ async def run_instances(instances: list[ItemHash]) -> None:
     # The main program uses a singleton pubsub instance in order to watch for updates.
     # We create another instance here since that singleton is not initialized yet.
     # Watching for updates on this instance will therefore not work.
-    pubsub: Optional[PubSub] = None
+    pubsub: PubSub | None = None
 
     await asyncio.gather(*[start_instance(instance_id, pubsub, pool) for instance_id in instances])
 
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 420754ecc..ca72b33bc 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -2,7 +2,6 @@
 import logging
 from collections.abc import Iterable
 from decimal import Decimal
-from typing import Optional
 
 import aiohttp
 from aleph_message.models import ItemHash, PaymentType
@@ -55,7 +54,7 @@ async def fetch_execution_flow_price(item_hash: ItemHash) -> Decimal:
 
         resp_data = await resp.json()
         required_flow: float = resp_data["required_tokens"]
-        payment_type: Optional[str] = resp_data["payment_type"]
+        payment_type: str | None = resp_data["payment_type"]
 
         if payment_type is None:
             raise ValueError("Payment type must be specified in the message")
@@ -75,7 +74,7 @@ async def fetch_execution_hold_price(item_hash: ItemHash) -> Decimal:
 
         resp_data = await resp.json()
         required_hold: float = resp_data["required_tokens"]
-        payment_type: Optional[str] = resp_data["payment_type"]
+        payment_type: str | None = resp_data["payment_type"]
 
         if payment_type not in (None, PaymentType.hold):
             raise ValueError(f"Payment type {payment_type} is not supported")
diff --git a/src/aleph/vm/orchestrator/pubsub.py b/src/aleph/vm/orchestrator/pubsub.py
index 465a5fdaf..6973a09db 100644
--- a/src/aleph/vm/orchestrator/pubsub.py
+++ b/src/aleph/vm/orchestrator/pubsub.py
@@ -7,7 +7,6 @@
 import logging
 import sys
 from collections.abc import Hashable
-from typing import Union
 
 from aleph_message.models import AlephMessage, ChainRef, ItemHash
 
@@ -59,6 +58,6 @@ async def msubscribe(self, *keys):
                 if self.subscribers.get(key) == set():
                     self.subscribers.pop(key)
 
-    async def publish(self, key: Union[ItemHash, str, ChainRef], value: AlephMessage):
+    async def publish(self, key: ItemHash | str | ChainRef, value: AlephMessage):
         for queue in self.subscribers.get(key, ()):
             await queue.put(value)
diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index 0f6ba0966..d4b9c8985 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -1,7 +1,6 @@
 import math
 from datetime import datetime, timezone
 from functools import lru_cache
-from typing import Optional
 
 import cpuinfo
 import psutil
@@ -157,8 +156,8 @@ class Allocation(BaseModel):
 
     persistent_vms: set[ItemHash] = Field(default_factory=set)
     instances: set[ItemHash] = Field(default_factory=set)
-    on_demand_vms: Optional[set[ItemHash]] = None
-    jobs: Optional[set[ItemHash]] = None
+    on_demand_vms: set[ItemHash] | None = None
+    jobs: set[ItemHash] | None = None
 
 
 class VMNotification(BaseModel):
diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index da29084dd..a2a2a824f 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -1,6 +1,6 @@
 import asyncio
 import logging
-from typing import Any, Optional
+from typing import Any
 
 import msgpack
 from aiohttp import web
@@ -99,7 +99,7 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques
     Execute the code corresponding to the 'code id' in the path.
     """
 
-    execution: Optional[VmExecution] = pool.get_running_vm(vm_hash=vm_hash)
+    execution: VmExecution | None = pool.get_running_vm(vm_hash=vm_hash)
 
     # Prevent execution issues if the execution resources are empty
     # TODO: Improve expiration process to avoid that kind of issues.
@@ -203,7 +203,7 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub, pool: VmPo
     Execute code in response to an event.
     """
 
-    execution: Optional[VmExecution] = pool.get_running_vm(vm_hash=vm_hash)
+    execution: VmExecution | None = pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
         execution = await create_vm_execution_or_raise_http_error(vm_hash=vm_hash, pool=pool)
@@ -248,8 +248,8 @@ async def run_code_on_event(vm_hash: ItemHash, event, pubsub: PubSub, pool: VmPo
             await execution.stop()
 
 
-async def start_persistent_vm(vm_hash: ItemHash, pubsub: Optional[PubSub], pool: VmPool) -> VmExecution:
-    execution: Optional[VmExecution] = pool.get_running_vm(vm_hash=vm_hash)
+async def start_persistent_vm(vm_hash: ItemHash, pubsub: PubSub | None, pool: VmPool) -> VmExecution:
+    execution: VmExecution | None = pool.get_running_vm(vm_hash=vm_hash)
 
     if not execution:
         logger.info(f"Starting persistent virtual machine with id: {vm_hash}")
@@ -269,7 +269,7 @@ async def start_persistent_vm(vm_hash: ItemHash, pubsub: Optional[PubSub], pool:
     return execution
 
 
-async def stop_persistent_vm(vm_hash: ItemHash, pool: VmPool) -> Optional[VmExecution]:
+async def stop_persistent_vm(vm_hash: ItemHash, pool: VmPool) -> VmExecution | None:
     logger.info(f"Stopping persistent VM {vm_hash}")
     execution = pool.get_running_vm(vm_hash)
 
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 91edf2dc1..118706370 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -8,10 +8,9 @@
 
 import asyncio
 import logging
-from collections.abc import Awaitable
+from collections.abc import Awaitable, Callable
 from pathlib import Path
 from secrets import token_urlsafe
-from typing import Callable
 
 from aiohttp import web
 from aiohttp_cors import ResourceOptions, setup
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index c99b0e38c..4bba01aa8 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -192,7 +192,7 @@ async def index(request: web.Request):
 
 
 @cors_allow_all
-async def status_check_fastapi(request: web.Request, vm_id: Optional[ItemHash] = None):
+async def status_check_fastapi(request: web.Request, vm_id: ItemHash | None = None):
     """Check that the FastAPI diagnostic VM runs correctly"""
 
     # Retro-compatibility mode ignores some of the newer checks. It is used to check the status of legacy VMs.
@@ -276,7 +276,7 @@ async def status_check_ipv6(request: web.Request):
 @cors_allow_all
 async def status_check_version(request: web.Request):
     """Check if the software is running a version equal or newer than the given one"""
-    reference_str: Optional[str] = request.query.get("reference")
+    reference_str: str | None = request.query.get("reference")
     if not reference_str:
         raise web.HTTPBadRequest(text="Query field '?reference=` must be specified")
     try:
diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index 1043ce994..dc2960030 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -3,15 +3,15 @@
 import functools
 import json
 import logging
-from collections.abc import Awaitable, Coroutine
-from typing import Any, Callable, Literal, Union
+from collections.abc import Awaitable, Callable, Coroutine
+from typing import Any, Literal
 
 import cryptography.exceptions
 import pydantic
 from aiohttp import web
 from eth_account import Account
 from eth_account.messages import encode_defunct
-from jwcrypto import jwk, jws
+from jwcrypto import jwk
 from jwcrypto.jwa import JWA
 from pydantic import BaseModel, ValidationError, root_validator, validator
 
@@ -98,7 +98,7 @@ def content(self) -> SignedPubKeyPayload:
 
 class SignedOperationPayload(BaseModel):
     time: datetime.datetime
-    method: Union[Literal["POST"], Literal["GET"]]
+    method: Literal["POST"] | Literal["GET"]
     domain: str
     path: str
     # body_sha256: str  # disabled since there is no body
@@ -166,8 +166,7 @@ def get_signed_pubkey(request: web.Request) -> SignedPubKeyHeader:
                 raise web.HTTPUnauthorized(reason="Token expired") from errors
             if str(err.exc) == "Invalid signature":
                 raise web.HTTPUnauthorized(reason="Invalid signature") from errors
-        else:
-            raise errors
+        raise errors
 
 
 def get_signed_operation(request: web.Request) -> SignedOperation:
diff --git a/src/aleph/vm/orchestrator/views/host_status.py b/src/aleph/vm/orchestrator/views/host_status.py
index 15c37dbe7..605de9b62 100644
--- a/src/aleph/vm/orchestrator/views/host_status.py
+++ b/src/aleph/vm/orchestrator/views/host_status.py
@@ -1,7 +1,7 @@
 import logging
 import socket
-from collections.abc import Awaitable
-from typing import Any, Callable, Optional
+from collections.abc import Awaitable, Callable
+from typing import Any
 
 import aiohttp
 
@@ -46,10 +46,10 @@ async def check_host_egress_ipv6() -> bool:
     return await check_ip_connectivity(settings.CONNECTIVITY_IPV6_URL)
 
 
-async def resolve_dns(hostname: str) -> tuple[Optional[str], Optional[str]]:
+async def resolve_dns(hostname: str) -> tuple[str | None, str | None]:
     """Resolve a hostname to an IPv4 and IPv6 address."""
-    ipv4: Optional[str] = None
-    ipv6: Optional[str] = None
+    ipv4: str | None = None
+    ipv6: str | None = None
 
     info = socket.getaddrinfo(hostname, 80, proto=socket.IPPROTO_TCP)
     if not info:
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 48e9c6b8c..3ecf500eb 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -5,7 +5,6 @@
 import logging
 from collections.abc import Iterable
 from datetime import datetime, timezone
-from typing import Optional
 
 from aleph_message.models import (
     Chain,
@@ -40,8 +39,8 @@ class VmPool:
     counter: int  # Used to provide distinct ids to network interfaces
     executions: dict[ItemHash, VmExecution]
     message_cache: dict[str, ExecutableMessage]
-    network: Optional[Network]
-    snapshot_manager: Optional[SnapshotManager] = None
+    network: Network | None
+    snapshot_manager: SnapshotManager | None = None
     systemd_manager: SystemDManager
     creation_lock: asyncio.Lock
 
@@ -168,11 +167,10 @@ def get_unique_vm_id(self) -> int:
             for i in range(settings.START_ID_INDEX, 255**2):
                 if i not in currently_used_vm_ids:
                     return i
-            else:
-                msg = "No available value for vm_id."
-                raise ValueError(msg)
+            msg = "No available value for vm_id."
+            raise ValueError(msg)
 
-    def get_running_vm(self, vm_hash: ItemHash) -> Optional[VmExecution]:
+    def get_running_vm(self, vm_hash: ItemHash) -> VmExecution | None:
         """Return a running VM or None. Disables the VM expiration task."""
         execution = self.executions.get(vm_hash)
         if execution and execution.is_running and not execution.is_stopping:
@@ -181,7 +179,7 @@ def get_running_vm(self, vm_hash: ItemHash) -> Optional[VmExecution]:
         else:
             return None
 
-    async def stop_vm(self, vm_hash: ItemHash) -> Optional[VmExecution]:
+    async def stop_vm(self, vm_hash: ItemHash) -> VmExecution | None:
         """Stop a VM."""
         execution = self.executions.get(vm_hash)
         if execution:
diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index 239a71586..f425103c5 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -14,7 +14,6 @@
 from pathlib import Path
 from shutil import copy2, make_archive
 from subprocess import CalledProcessError
-from typing import Union
 
 import aiohttp
 from aleph_message.models import (
@@ -142,7 +141,7 @@ async def get_latest_amend(item_hash: str) -> str:
             return result or item_hash
 
 
-async def get_message(ref: str) -> Union[ProgramMessage, InstanceMessage]:
+async def get_message(ref: str) -> ProgramMessage | InstanceMessage:
     if ref == settings.FAKE_INSTANCE_ID:
         logger.debug("Using the fake instance message since the ref matches")
         cache_path = settings.FAKE_INSTANCE_MESSAGE
@@ -256,7 +255,7 @@ async def create_ext4(path: Path, size_mib: int) -> bool:
     return True
 
 
-async def create_volume_file(volume: Union[PersistentVolume, RootfsVolume], namespace: str) -> Path:
+async def create_volume_file(volume: PersistentVolume | RootfsVolume, namespace: str) -> Path:
     volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
     # Assume that the main filesystem format is BTRFS
     path = settings.PERSISTENT_VOLUMES_DIR / namespace / f"{volume_name}.btrfs"
@@ -300,7 +299,7 @@ async def resize_and_tune_file_system(device_path: Path, mount_path: Path) -> No
     await run_in_subprocess(["umount", str(mount_path)])
 
 
-async def create_devmapper(volume: Union[PersistentVolume, RootfsVolume], namespace: str) -> Path:
+async def create_devmapper(volume: PersistentVolume | RootfsVolume, namespace: str) -> Path:
     """It creates a /dev/mapper/DEVICE inside the VM, that is an extended mapped device of the volume specified.
     We follow the steps described here: https://community.aleph.im/t/deploying-mutable-vm-instances-on-aleph/56/2
     """
diff --git a/src/aleph/vm/utils/__init__.py b/src/aleph/vm/utils/__init__.py
index 30b7ce90e..b2da90d63 100644
--- a/src/aleph/vm/utils/__init__.py
+++ b/src/aleph/vm/utils/__init__.py
@@ -5,12 +5,12 @@
 import logging
 import subprocess
 from base64 import b16encode, b32decode
-from collections.abc import Coroutine
+from collections.abc import Callable, Coroutine
 from dataclasses import asdict as dataclass_as_dict
 from dataclasses import is_dataclass
 from pathlib import Path
 from shutil import disk_usage
-from typing import Any, Callable, Optional
+from typing import Any, Optional
 
 import aiodns
 import msgpack
@@ -80,7 +80,7 @@ def to_json(o: Any):
         return str(o)
 
 
-def dumps_for_json(o: Any, indent: Optional[int] = None):
+def dumps_for_json(o: Any, indent: int | None = None):
     return json.dumps(o, default=to_json, indent=indent)
 
 
@@ -98,7 +98,7 @@ def create_task_log_exceptions(coro: Coroutine, *, name=None):
     return asyncio.create_task(run_and_log_exception(coro), name=name)
 
 
-async def run_in_subprocess(command: list[str], check: bool = True, stdin_input: Optional[bytes] = None) -> bytes:
+async def run_in_subprocess(command: list[str], check: bool = True, stdin_input: bytes | None = None) -> bytes:
     """Run the specified command in a subprocess, returns the stdout of the process."""
     command = [str(arg) for arg in command]
     logger.debug(f"command: {' '.join(command)}")
@@ -131,7 +131,7 @@ def is_command_available(command):
         return False
 
 
-def check_system_module(module_path: str) -> Optional[str]:
+def check_system_module(module_path: str) -> str | None:
     p = Path("/sys/module") / module_path
     if not p.exists():
         return None
diff --git a/src/aleph/vm/utils/logs.py b/src/aleph/vm/utils/logs.py
index a112cfc0a..ebba56616 100644
--- a/src/aleph/vm/utils/logs.py
+++ b/src/aleph/vm/utils/logs.py
@@ -1,7 +1,8 @@
 import asyncio
 import logging
+from collections.abc import Callable, Generator
 from datetime import datetime
-from typing import Callable, Generator, TypedDict
+from typing import TypedDict
 
 from systemd import journal
 
diff --git a/src/aleph/vm/version.py b/src/aleph/vm/version.py
index 2fdf24d17..ba4f34336 100644
--- a/src/aleph/vm/version.py
+++ b/src/aleph/vm/version.py
@@ -1,11 +1,10 @@
 import logging
 from subprocess import CalledProcessError, check_output
-from typing import Optional
 
 logger = logging.getLogger(__name__)
 
 
-def get_version_from_git() -> Optional[str]:
+def get_version_from_git() -> str | None:
     try:
         return check_output(("git", "describe", "--tags")).strip().decode()
     except FileNotFoundError:
@@ -16,7 +15,7 @@ def get_version_from_git() -> Optional[str]:
         return None
 
 
-def get_version_from_apt() -> Optional[str]:
+def get_version_from_apt() -> str | None:
     try:
         import apt
 
@@ -26,7 +25,7 @@ def get_version_from_apt() -> Optional[str]:
         return None
 
 
-def get_version() -> Optional[str]:
+def get_version() -> str | None:
     return get_version_from_git() or get_version_from_apt()
 
 

From 8f56dbe9c8491d9ada9b16192b46b2e6c7e4b291 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 3 Sep 2024 18:44:43 +0200
Subject: [PATCH 845/990] Fix: Ruff errored on exception raising

The message should be passed as a variable.

Obtained with `ruff check src --fix --unsafe-fixes`
---
 src/aleph/vm/conf.py                          |  3 ++-
 src/aleph/vm/controllers/qemu/client.py       |  3 ++-
 src/aleph/vm/controllers/qemu/instance.py     |  9 +++++---
 .../hypervisors/qemu_confidential/qemuvm.py   |  6 ++++--
 src/aleph/vm/models.py                        | 12 +++++++----
 src/aleph/vm/network/interfaces.py            | 12 +++++++----
 src/aleph/vm/orchestrator/chain.py            |  8 ++++---
 src/aleph/vm/orchestrator/payment.py          | 21 ++++++++++++-------
 .../vm/orchestrator/views/authentication.py   |  6 ++++--
 src/aleph/vm/utils/__init__.py                | 12 +++++++----
 10 files changed, 61 insertions(+), 31 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 9c86b6014..a1737b2b6 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -117,7 +117,8 @@ def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> list[st
         return list(resolvectl_dns_servers_ipv4(interface=network_interface))
 
     else:
-        assert False, "No DNS resolve defined, this should never happen."
+        msg = "No DNS resolve defined, this should never happen."
+        raise AssertionError(msg)
 
 
 class Settings(BaseSettings):
diff --git a/src/aleph/vm/controllers/qemu/client.py b/src/aleph/vm/controllers/qemu/client.py
index 936f65b5b..c98899d5b 100644
--- a/src/aleph/vm/controllers/qemu/client.py
+++ b/src/aleph/vm/controllers/qemu/client.py
@@ -16,7 +16,8 @@ class QemuVmClient:
     def __init__(self, vm):
         self.vm = vm
         if not (vm.qmp_socket_path and vm.qmp_socket_path.exists()):
-            raise Exception("VM is not running")
+            msg = "VM is not running"
+            raise Exception(msg)
         client = qmp.QEMUMonitorProtocol(str(vm.qmp_socket_path))
         client.connect()
 
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index 3ca138903..dd840e22b 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -51,7 +51,8 @@ async def make_writable_volume(self, parent_image_path, volume: PersistentVolume
         """Create a new qcow2 image file based on the passed one, that we give to the VM to write onto"""
         qemu_img_path: str | None = shutil.which("qemu-img")
         if not qemu_img_path:
-            raise VmSetupError("qemu-img not found in PATH")
+            msg = "qemu-img not found in PATH"
+            raise VmSetupError(msg)
 
         volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
 
@@ -60,9 +61,11 @@ async def make_writable_volume(self, parent_image_path, volume: PersistentVolume
         out = json.loads(out_json)
         parent_format = out.get("format", None)
         if parent_format is None:
-            raise VmSetupError(f"Failed to detect format for {volume}: {out_json}")
+            msg = f"Failed to detect format for {volume}: {out_json}"
+            raise VmSetupError(msg)
         if parent_format not in ("qcow2", "raw"):
-            raise VmSetupError(f"Format {parent_format} for {volume} unhandled by QEMU hypervisor")
+            msg = f"Format {parent_format} for {volume} unhandled by QEMU hypervisor"
+            raise VmSetupError(msg)
 
         dest_path = settings.PERSISTENT_VOLUMES_DIR / self.namespace / f"{volume_name}.qcow2"
         # Do not override if user asked for host persistance.
diff --git a/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
index 4b472e4ce..5e32e8990 100644
--- a/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
@@ -56,12 +56,14 @@ async def start(
         # TODO : ensure this is ok at launch
         sev_info = secure_encryption_info()
         if sev_info is None:
-            raise ValueError("Not running on an AMD SEV platform?")
+            msg = "Not running on an AMD SEV platform?"
+            raise ValueError(msg)
         godh = self.sev_dh_cert_file
         launch_blob = self.sev_session_file
 
         if not (godh.is_file() and launch_blob.is_file()):
-            raise FileNotFoundError("Missing guest owner certificates, cannot start the VM.`")
+            msg = "Missing guest owner certificates, cannot start the VM.`"
+            raise FileNotFoundError(msg)
         args = [
             self.qemu_bin_path,
             "-enable-kvm",
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 27db9ec6b..35de4076d 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -200,9 +200,11 @@ async def prepare(self) -> None:
                     else:
                         resources = AlephQemuResources(self.message, namespace=self.vm_hash)
                 else:
-                    raise ValueError(f"Unknown hypervisor type {self.hypervisor}")
+                    msg = f"Unknown hypervisor type {self.hypervisor}"
+                    raise ValueError(msg)
             else:
-                raise ValueError("Unknown executable message type")
+                msg = "Unknown executable message type"
+                raise ValueError(msg)
 
             if not resources:
                 msg = "Unknown executable message type"
@@ -265,9 +267,11 @@ def create(
                         tap_interface=tap_interface,
                     )
             else:
-                raise Exception("Unknown VM")
+                msg = "Unknown VM"
+                raise Exception(msg)
         else:
-            raise Exception("Unknown VM")
+            msg = "Unknown VM"
+            raise Exception(msg)
 
         return vm
 
diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index 9171f25fe..b90c84b4e 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -48,7 +48,8 @@ def add_ip_address(ipr: IPRoute, device_name: str, ip: IPv4Interface | IPv6Inter
     returns without error."""
     interface_index: list[int] = ipr.link_lookup(ifname=device_name)
     if not interface_index:
-        raise MissingInterfaceError(f"Interface {device_name} does not exist, can't add address {ip} to it.")
+        msg = f"Interface {device_name} does not exist, can't add address {ip} to it."
+        raise MissingInterfaceError(msg)
     try:
         ipr.addr("add", index=interface_index[0], address=str(ip.ip), mask=ip.network.prefixlen)
     except NetlinkError as e:
@@ -64,7 +65,8 @@ def delete_ip_address(ipr: IPRoute, device_name: str, ip: IPv4Interface | IPv6In
     """Delete an IP address to the given interface."""
     interface_index: list[int] = ipr.link_lookup(ifname=device_name)
     if not interface_index:
-        raise MissingInterfaceError(f"Interface {device_name} does not exist, can't delete address {ip} to it.")
+        msg = f"Interface {device_name} does not exist, can't delete address {ip} to it."
+        raise MissingInterfaceError(msg)
     try:
         ipr.addr("del", index=interface_index[0], address=str(ip.ip), mask=ip.network.prefixlen)
     except NetlinkError as e:
@@ -77,7 +79,8 @@ def set_link_up(ipr: IPRoute, device_name: str):
     """Set the given interface up."""
     interface_index: list[int] = ipr.link_lookup(ifname=device_name)
     if not interface_index:
-        raise MissingInterfaceError(f"Interface {device_name} does not exist, can't set it up.")
+        msg = f"Interface {device_name} does not exist, can't set it up."
+        raise MissingInterfaceError(msg)
     try:
         ipr.link("set", index=interface_index[0], state="up")
     except NetlinkError as e:
@@ -144,7 +147,8 @@ async def create(self):
 
         ip_command = shutil.which("ip")
         if not ip_command:
-            raise FileNotFoundError("ip command not found")
+            msg = "ip command not found"
+            raise FileNotFoundError(msg)
 
         ipv6_gateway = self.host_ipv6
 
diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py
index 9a8db1df9..b499ae8c0 100644
--- a/src/aleph/vm/orchestrator/chain.py
+++ b/src/aleph/vm/orchestrator/chain.py
@@ -23,9 +23,10 @@ def token(self) -> str | None:
         return self.super_token or self.standard_token
 
     @root_validator(pre=True)
-    def check_tokens(cls, values):
+    def check_tokens(self, values):
         if not values.get("standard_token") and not values.get("super_token"):
-            raise ValueError("At least one of standard_token or super_token must be provided.")
+            msg = "At least one of standard_token or super_token must be provided."
+            raise ValueError(msg)
         return values
 
 
@@ -63,4 +64,5 @@ def get_chain(chain: str) -> ChainInfo:
     try:
         return STREAM_CHAINS[chain]
     except KeyError:
-        raise ValueError(f"Unknown chain id for chain {chain}")
+        msg = f"Unknown chain id for chain {chain}"
+        raise ValueError(msg)
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index ca72b33bc..7194f873a 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -57,9 +57,11 @@ async def fetch_execution_flow_price(item_hash: ItemHash) -> Decimal:
         payment_type: str | None = resp_data["payment_type"]
 
         if payment_type is None:
-            raise ValueError("Payment type must be specified in the message")
+            msg = "Payment type must be specified in the message"
+            raise ValueError(msg)
         elif payment_type != PaymentType.superfluid:
-            raise ValueError(f"Payment type {payment_type} is not supported")
+            msg = f"Payment type {payment_type} is not supported"
+            raise ValueError(msg)
 
         return Decimal(required_flow)
 
@@ -77,7 +79,8 @@ async def fetch_execution_hold_price(item_hash: ItemHash) -> Decimal:
         payment_type: str | None = resp_data["payment_type"]
 
         if payment_type not in (None, PaymentType.hold):
-            raise ValueError(f"Payment type {payment_type} is not supported")
+            msg = f"Payment type {payment_type} is not supported"
+            raise ValueError(msg)
 
         return Decimal(required_hold)
 
@@ -99,24 +102,28 @@ async def get_stream(sender: str, receiver: str, chain: str) -> Decimal:
     """
     chain_info: ChainInfo = get_chain(chain=chain)
     if not chain_info.active:
-        raise InvalidChainError(f"Chain : {chain} is not active for superfluid")
+        msg = f"Chain : {chain} is not active for superfluid"
+        raise InvalidChainError(msg)
 
     superfluid_instance = CFA_V1(chain_info.rpc, chain_info.chain_id)
 
     try:
         super_token: HexAddress = to_normalized_address(chain_info.super_token)
     except ValueError as error:
-        raise InvalidAddressError(f"Invalid token address '{chain_info.super_token}' - {error.args}") from error
+        msg = f"Invalid token address '{chain_info.super_token}' - {error.args}"
+        raise InvalidAddressError(msg) from error
 
     try:
         sender_address: HexAddress = to_normalized_address(sender)
     except ValueError as error:
-        raise InvalidAddressError(f"Invalid sender address '{sender}' - {error.args}") from error
+        msg = f"Invalid sender address '{sender}' - {error.args}"
+        raise InvalidAddressError(msg) from error
 
     try:
         receiver_address: HexAddress = to_normalized_address(receiver)
     except ValueError as error:
-        raise InvalidAddressError(f"Invalid receiver address '{receiver}' - {error.args}") from error
+        msg = f"Invalid receiver address '{receiver}' - {error.args}"
+        raise InvalidAddressError(msg) from error
 
     # Run the network request in a background thread and wait for it to complete.
     loop = asyncio.get_event_loop()
diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index dc2960030..d4b24bc20 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -109,9 +109,11 @@ def time_is_current(cls, v: datetime.datetime) -> datetime.datetime:
         max_past = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(minutes=2)
         max_future = datetime.datetime.now(tz=datetime.timezone.utc) + datetime.timedelta(minutes=2)
         if v < max_past:
-            raise ValueError("Time is too far in the past")
+            msg = "Time is too far in the past"
+            raise ValueError(msg)
         if v > max_future:
-            raise ValueError("Time is too far in the future")
+            msg = "Time is too far in the future"
+            raise ValueError(msg)
         return v
 
 
diff --git a/src/aleph/vm/utils/__init__.py b/src/aleph/vm/utils/__init__.py
index b2da90d63..507a41e0e 100644
--- a/src/aleph/vm/utils/__init__.py
+++ b/src/aleph/vm/utils/__init__.py
@@ -206,7 +206,8 @@ async def get_path_size(path: Path) -> int:
     elif path.is_file():
         return path.stat().st_size
     else:
-        raise ValueError(f"Unknown path type for {path}")
+        msg = f"Unknown path type for {path}"
+        raise ValueError(msg)
 
 
 async def get_block_device_size(device: str) -> int:
@@ -226,11 +227,13 @@ def to_normalized_address(value: str) -> HexAddress:
     try:
         hex_address = hexstr_if_str(to_hex, value).lower()
     except AttributeError:
-        raise TypeError(f"Value must be any string, instead got type {type(value)}")
+        msg = f"Value must be any string, instead got type {type(value)}"
+        raise TypeError(msg)
     if is_address(hex_address):
         return HexAddress(HexStr(hex_address))
     else:
-        raise ValueError(f"Unknown format {value}, attempted to normalize to {hex_address}")
+        msg = f"Unknown format {value}, attempted to normalize to {hex_address}"
+        raise ValueError(msg)
 
 
 def md5sum(file_path: Path) -> str:
@@ -241,7 +244,8 @@ def md5sum(file_path: Path) -> str:
 def file_hashes_differ(source: Path, destination: Path, checksum: Callable[[Path], str] = md5sum) -> bool:
     """Check if the MD5 hash of two files differ."""
     if not source.exists():
-        raise FileNotFoundError(f"Source file does not exist: {source}")
+        msg = f"Source file does not exist: {source}"
+        raise FileNotFoundError(msg)
 
     if not destination.exists():
         return True

From 7ecb84e9cdb99149bd6a1887bbbfaf965e97244f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 3 Sep 2024 18:49:56 +0200
Subject: [PATCH 846/990] Fix: Python < 3.10 is no supported anymore

---
 src/aleph/vm/hypervisors/firecracker/microvm.py | 10 ++--------
 src/aleph/vm/orchestrator/pubsub.py             |  6 +-----
 2 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 5590315c6..2c75f9132 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -296,10 +296,7 @@ def enable_kernel(self, kernel_image_path: Path) -> Path:
             jailer_kernel_image_path = f"/opt/{kernel_filename}"
 
             try:
-                if sys.version_info >= (3, 10):
-                    Path(f"{self.jailer_path}{jailer_kernel_image_path}").hardlink_to(kernel_image_path)
-                else:
-                    kernel_image_path.link_to(f"{self.jailer_path}{jailer_kernel_image_path}")
+                Path(f"{self.jailer_path}{jailer_kernel_image_path}").hardlink_to(kernel_image_path)
             except FileExistsError:
                 logger.debug(f"File {jailer_kernel_image_path} already exists")
 
@@ -379,10 +376,7 @@ def enable_drive(self, drive_path: Path, read_only: bool = True) -> Drive:
             jailer_path_on_host = f"/opt/{drive_filename}"
 
             try:
-                if sys.version_info >= (3, 10):
-                    Path(f"{self.jailer_path}/{jailer_path_on_host}").hardlink_to(drive_path)
-                else:
-                    drive_path.link_to(f"{self.jailer_path}/{jailer_path_on_host}")
+                Path(f"{self.jailer_path}/{jailer_path_on_host}").hardlink_to(drive_path)
             except FileExistsError:
                 logger.debug(f"File {jailer_path_on_host} already exists")
             drive_path = Path(jailer_path_on_host)
diff --git a/src/aleph/vm/orchestrator/pubsub.py b/src/aleph/vm/orchestrator/pubsub.py
index 6973a09db..f1c784a96 100644
--- a/src/aleph/vm/orchestrator/pubsub.py
+++ b/src/aleph/vm/orchestrator/pubsub.py
@@ -14,11 +14,7 @@
 
 
 class PubSub:
-    if sys.version_info >= (3, 9):
-        subscribers: dict[Hashable, set[asyncio.Queue[set]]]
-    else:
-        # Support for Python 3.8 (Ubuntu 20.04)
-        subscribers: dict[Hashable, set[asyncio.Queue]]
+    subscribers: dict[Hashable, set[asyncio.Queue[set]]]
 
     def __init__(self):
         self.subscribers = {}

From 590daa1ae0d3d61293bb4ad92802c95747a13479 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 3 Sep 2024 18:51:17 +0200
Subject: [PATCH 847/990] Fix: Type annotations could be improved

---
 src/aleph/vm/storage.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index f425103c5..7e289dca2 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -161,7 +161,7 @@ async def get_message(ref: str) -> ProgramMessage | InstanceMessage:
             msg = fix_message_validation(msg)
 
         result = parse_message(message_dict=msg)
-        assert isinstance(result, (InstanceMessage, ProgramMessage)), "Parsed message is not executable"
+        assert isinstance(result, InstanceMessage | ProgramMessage), "Parsed message is not executable"
         return result
 
 
@@ -369,7 +369,7 @@ async def get_volume_path(volume: MachineVolume, namespace: str) -> Path:
     if isinstance(volume, ImmutableVolume):
         ref = volume.ref
         return await get_existing_file(ref)
-    elif isinstance(volume, (PersistentVolume, RootfsVolume)):
+    elif isinstance(volume, PersistentVolume | RootfsVolume):
         volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
         if volume.persistence != VolumePersistence.host:
             msg = "Only 'host' persistence is supported"

From 48086a16df3596069dcc02898d04a69e0a162edc Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 3 Sep 2024 18:52:12 +0200
Subject: [PATCH 848/990] Fix: Generator loop -> 'yield from'

---
 src/aleph/vm/utils/logs.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/aleph/vm/utils/logs.py b/src/aleph/vm/utils/logs.py
index ebba56616..1bf0dc449 100644
--- a/src/aleph/vm/utils/logs.py
+++ b/src/aleph/vm/utils/logs.py
@@ -79,5 +79,4 @@ def get_past_vm_logs(stdout_identifier, stderr_identifier) -> Generator[EntryDic
     r.add_match(SYSLOG_IDENTIFIER=stderr_identifier)
 
     r.seek_head()
-    for entry in r:
-        yield entry
+    yield from r

From 1beef1e2a740ca034267cb3f1b4e7a1a446e49db Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 4 Sep 2024 15:52:45 +0200
Subject: [PATCH 849/990] more ruffs fixes

---
 src/aleph/vm/hypervisors/firecracker/microvm.py | 1 -
 src/aleph/vm/orchestrator/pubsub.py             | 1 -
 2 files changed, 2 deletions(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 2c75f9132..176454718 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -5,7 +5,6 @@
 import os.path
 import shutil
 import string
-import sys
 import traceback
 from asyncio import Task
 from asyncio.base_events import Server
diff --git a/src/aleph/vm/orchestrator/pubsub.py b/src/aleph/vm/orchestrator/pubsub.py
index f1c784a96..b12a76bc8 100644
--- a/src/aleph/vm/orchestrator/pubsub.py
+++ b/src/aleph/vm/orchestrator/pubsub.py
@@ -5,7 +5,6 @@
 
 import asyncio
 import logging
-import sys
 from collections.abc import Hashable
 
 from aleph_message.models import AlephMessage, ChainRef, ItemHash

From de27ed0268d5c1f108d3dc418041ae1e1b45df4f Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Wed, 4 Sep 2024 16:05:56 +0200
Subject: [PATCH 850/990] Fix: Useless comma could be removed

---
 tests/supervisor/views/test_run_code.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/supervisor/views/test_run_code.py b/tests/supervisor/views/test_run_code.py
index 639a8f7bf..0c42b5243 100644
--- a/tests/supervisor/views/test_run_code.py
+++ b/tests/supervisor/views/test_run_code.py
@@ -21,7 +21,7 @@ async def test_run_code_from_invalid_path(aiohttp_client):
 
     app = web.Application()
 
-    app.router.add_route("*", "/vm/{ref}{suffix:.*}", run_code_from_path),
+    app.router.add_route("*", "/vm/{ref}{suffix:.*}", run_code_from_path)
     client = await aiohttp_client(app)
 
     invalid_hash_request: web.Request = make_mocked_request(

From acc2302863f2adeedd83ae9c321675459a93176a Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 4 Sep 2024 16:37:48 +0200
Subject: [PATCH 851/990] Fix Pydantic error

---
 src/aleph/vm/orchestrator/chain.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py
index b499ae8c0..7321aa458 100644
--- a/src/aleph/vm/orchestrator/chain.py
+++ b/src/aleph/vm/orchestrator/chain.py
@@ -23,7 +23,7 @@ def token(self) -> str | None:
         return self.super_token or self.standard_token
 
     @root_validator(pre=True)
-    def check_tokens(self, values):
+    def check_tokens(cls, values):
         if not values.get("standard_token") and not values.get("super_token"):
             msg = "At least one of standard_token or super_token must be provided."
             raise ValueError(msg)

From 073eb83a4dbd1b1a757d83d686f3129a7cf3c310 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 6 Sep 2024 11:44:09 +0200
Subject: [PATCH 852/990] Problem: Non deterministic teardown

There was a teardown() inside a __del__
which was triggered by the garbage collection
This resulted in an unclear lifecycle and strange log error since the teardown was already triggered before
and made for strange error when running tests

Solution: remove it
---
 src/aleph/vm/hypervisors/firecracker/microvm.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 176454718..3cf3e3087 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -503,13 +503,3 @@ async def teardown(self):
             self.config_file_path.unlink(missing_ok=True)
         if Path(self.namespace_path).exists():
             system(f"rm -fr {self.namespace_path}")
-
-    def __del__(self):
-        try:
-            loop = asyncio.get_running_loop()
-            loop.create_task(self.teardown())
-        except RuntimeError as error:
-            if error.args == ("no running event loop",):
-                return
-            else:
-                raise

From 7ee7d04a5bae1c45069a528fed2711354e83c90d Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 10 Sep 2024 15:18:07 +0200
Subject: [PATCH 853/990] Problem: Test failing not clearly

The execution and instances tests were failing if the runtime or dis image were not properly set up but it was not clear to the user why

Solution:
Use xfail to display an user to the message on how to set up properly

> tests/supervisor/test_execution.py::test_create_execution XFAIL (Test
Runtime not setup. run `cd runtimes/aleph-debian-12-python && sudo
./create_disk_image.sh`)
---
 tests/supervisor/test_execution.py     | 3 +++
 tests/supervisor/test_instance.py      | 2 ++
 tests/supervisor/test_qemu_instance.py | 4 ++++
 3 files changed, 9 insertions(+)

diff --git a/tests/supervisor/test_execution.py b/tests/supervisor/test_execution.py
index 7f64b5a8f..40c6fd71b 100644
--- a/tests/supervisor/test_execution.py
+++ b/tests/supervisor/test_execution.py
@@ -23,6 +23,9 @@ async def test_create_execution(mocker):
     mocker.patch("aleph.vm.controllers.firecracker.executable.settings", new=mock_settings)
     mocker.patch("aleph.vm.controllers.firecracker.program.settings", new=mock_settings)
 
+    if not mock_settings.FAKE_DATA_RUNTIME.exists():
+        pytest.xfail("Test Runtime not setup. run `cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh`")
+
     mock_settings.FAKE_DATA_PROGRAM = mock_settings.BENCHMARK_FAKE_DATA_PROGRAM
     mock_settings.ALLOW_VM_NETWORKING = False
     mock_settings.USE_JAILER = False
diff --git a/tests/supervisor/test_instance.py b/tests/supervisor/test_instance.py
index 54c26aee7..6a3ffa509 100644
--- a/tests/supervisor/test_instance.py
+++ b/tests/supervisor/test_instance.py
@@ -63,6 +63,8 @@ async def test_create_instance():
     # Ensure that the settings are correct and required files present.
     settings.setup()
     settings.check()
+    if not settings.FAKE_DATA_RUNTIME.exists():
+        pytest.xfail("Test Runtime not setup. run `cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh`")
 
     # The database is required for the metrics and is currently not optional.
     engine = metrics.setup_engine()
diff --git a/tests/supervisor/test_qemu_instance.py b/tests/supervisor/test_qemu_instance.py
index 9834e253c..6da59a625 100644
--- a/tests/supervisor/test_qemu_instance.py
+++ b/tests/supervisor/test_qemu_instance.py
@@ -56,6 +56,8 @@ async def test_create_qemu_instance():
     settings.ENABLE_CONFIDENTIAL_COMPUTING = False
     settings.ALLOW_VM_NETWORKING = False
     settings.USE_JAILER = False
+    if not settings.FAKE_INSTANCE_BASE.exists():
+        pytest.xfail("Test Runtime not setup. run `cd runtimes/instance-rootfs && sudo ./create-debian-12-disk.sh`")
 
     logging.basicConfig(level=logging.DEBUG)
 
@@ -117,6 +119,8 @@ async def test_create_qemu_instance_online():
     # Ensure that the settings are correct and required files present.
     settings.setup()
     settings.check()
+    if not settings.FAKE_INSTANCE_BASE.exists():
+        pytest.xfail("Test Runtime not setup. run `cd runtimes/instance-rootfs && sudo ./create-debian-12-disk.sh`")
 
     # The database is required for the metrics and is currently not optional.
     engine = metrics.setup_engine()

From eb2863ddcf5d64c2c69d27bf1e9f266bdbd29dfa Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 13 Sep 2024 18:36:34 +0200
Subject: [PATCH 854/990] Problem: QEMUVM killed before shutdown command (#698)

* Problem: QEMUVM killed before shutdown command

When running the controller as a systemd service (the normal usecase in prod) and stopping the service
the QEMU process was always killed BEFORE the shutdown command could be sent
so the VM could not properly clean up

As an additional symptom this error appeared and confused dev and user
```
Sep 05 12:24:03 testing-hetzner python3[2468548]: 2024-09-05 12:24:03,187 | ERROR | Task exception was never retrieved
Sep 05 12:24:03 testing-hetzner python3[2468548]: future:  exception=QMPCapabilitiesError()>
Sep 05 12:24:03 testing-hetzner python3[2468548]: Traceback (most recent call last):
Sep 05 12:24:03 testing-hetzner python3[2468548]:   File "/opt/aleph-vm/aleph/vm/hypervisors/qemu/qemuvm.py", line 151, in stop
Sep 05 12:24:03 testing-hetzner python3[2468548]:     self.send_shutdown_message()
Sep 05 12:24:03 testing-hetzner python3[2468548]:   File "/opt/aleph-vm/aleph/vm/hypervisors/qemu/qemuvm.py", line 141, in send_shutdown_message
Sep 05 12:24:03 testing-hetzner python3[2468548]:     client = self._get_qmpclient()
Sep 05 12:24:03 testing-hetzner python3[2468548]:              ^^^^^^^^^^^^^^^^^^^^^
Sep 05 12:24:03 testing-hetzner python3[2468548]:   File "/opt/aleph-vm/aleph/vm/hypervisors/qemu/qemuvm.py", line 136, in _get_qmpclient
Sep 05 12:24:03 testing-hetzner python3[2468548]:     client.connect()
Sep 05 12:24:03 testing-hetzner python3[2468548]:   File "/opt/aleph-vm/qmp.py", line 162, in connect
Sep 05 12:24:03 testing-hetzner python3[2468548]:     return self.__negotiate_capabilities()
Sep 05 12:24:03 testing-hetzner python3[2468548]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Sep 05 12:24:03 testing-hetzner python3[2468548]:   File "/opt/aleph-vm/qmp.py", line 88, in __negotiate_capabilities
Sep 05 12:24:03 testing-hetzner python3[2468548]:     raise QMPCapabilitiesError
Sep 05 12:24:03 testing-hetzner python3[2468548]: qmp.QMPCapabilitiesError
Sep 05 12:24:03 testing-hetzner python3[2468548]: 2024-09-05 12:24:03,285 | WARNING | Process terminated with 0
```

Solution: Use mixed kill mode in Systemd, which will at first only send
the term signal to the main process, and give the VM time to properly
cleanup and shutdown.

Note that some time the "shutdown" error is not acted upon so stoping
the process is still necessary. It seems to happend when the boot is not
completed yet.

So a fallback kill is done after a timeout.

---------

Co-authored-by: Hugo Herter 
---
 .../etc/systemd/system/aleph-vm-controller@.service         | 6 ++++++
 src/aleph/vm/controllers/__main__.py                        | 1 +
 2 files changed, 7 insertions(+)

diff --git a/packaging/aleph-vm/etc/systemd/system/aleph-vm-controller@.service b/packaging/aleph-vm/etc/systemd/system/aleph-vm-controller@.service
index c817aad1e..7bbfc67d8 100644
--- a/packaging/aleph-vm/etc/systemd/system/aleph-vm-controller@.service
+++ b/packaging/aleph-vm/etc/systemd/system/aleph-vm-controller@.service
@@ -11,6 +11,12 @@ WorkingDirectory=/opt/aleph-vm
 Environment=PYTHONPATH=/opt/aleph-vm/:$PYTHONPATH
 ExecStart=/usr/bin/python3 -m aleph.vm.controllers --config=/var/lib/aleph/vm/%i-controller.json
 Restart=on-failure
+# KillMode=Mixed is used so initially only the Python controller process receives the SIGTERM signal.
+# The controller catches it and sends a QEMU command to shut down the Guest VM, allowing it to clean up
+# properly and avoid disk corruption.
+# After 30s (TimeoutStopSec), if the process is still running, both the controller and subprocesses receive SIGKILL.
+KillMode=mixed
+TimeoutStopSec=30
 
 [Install]
 WantedBy=multi-user.target
diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index 19701c4bf..519270b48 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -90,6 +90,7 @@ async def handle_persistent_vm(config: Configuration, execution: MicroVM | QemuV
 
     def callback():
         """Callback for the signal handler to stop the VM and cleanup properly on SIGTERM."""
+        logger.debug("Received SIGTERM")
         loop.create_task(execution.stop())
 
     loop.add_signal_handler(signal.SIGTERM, callback)

From fefb30d264420448ecef88bcdfe455a7ac417b76 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Fri, 13 Sep 2024 12:20:32 +0200
Subject: [PATCH 855/990] Fix: dictionary changed size during iteration

Iterating over the keys of a dictionary is a lazy
operation that uses a generator under the hood.

An error was reported on Sentry due to
"RuntimeError: dictionary changed size during iteration".

This fixes the issue by transforming the generator
into a list, which is the standard way to approach
this issue in Python.
---
 src/aleph/vm/orchestrator/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 3f468785d..99c814a82 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -150,7 +150,7 @@ async def monitor_payments(app: web.Application):
         await asyncio.sleep(settings.PAYMENT_MONITOR_INTERVAL)
 
         # Check if the executions continues existing or are forgotten before checking the payment
-        for vm_hash in pool.executions.keys():
+        for vm_hash in list(pool.executions.keys()):
             message_status = await get_message_status(vm_hash)
             if message_status != MessageStatus.PROCESSED:
                 logger.debug(f"Stopping {vm_hash} execution due to {message_status} message status")

From 75cc948bf6e33c61b3741b0a59e5bc03eda541a2 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 17 Sep 2024 11:22:39 +0200
Subject: [PATCH 856/990] Problem: Test test_websocket_logs_invalid_auth fail
 on Python 3.12.3

All python version didn\'t return the same error
```
E         - {"status": "failed", "reason": "string indices must be integers"}
E         + {"status": "failed", "reason": "string indices must be integers, not \'str\'"}
```

Solution: Force a error message.
Should also make the message a bit clearer
---
 src/aleph/vm/orchestrator/views/authentication.py | 9 +++++++++
 tests/supervisor/views/test_operator.py           | 4 +++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index d4b24bc20..6bc9a6c04 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -1,3 +1,10 @@
+"""Functions for authentications
+
+See /doc/operator_auth.md for the explaination of how the operator authentication works.
+
+Can be enabled on an endpoint using the @require_jwk_authentication decorator
+"""
+
 # Keep datetime import as is as it allow patching in test
 import datetime
 import functools
@@ -216,6 +223,8 @@ async def authenticate_jwk(request: web.Request) -> str:
 
 async def authenticate_websocket_message(message) -> str:
     """Authenticate a websocket message since JS cannot configure headers on WebSockets."""
+    if not isinstance(message, dict):
+        raise Exception("Invalid format for auth packet, see /doc/operator_auth.md")
     signed_pubkey = SignedPubKeyHeader.parse_obj(message["X-SignedPubKey"])
     signed_operation = SignedOperation.parse_obj(message["X-SignedOperation"])
     if signed_operation.content.domain != settings.DOMAIN_NAME:
diff --git a/tests/supervisor/views/test_operator.py b/tests/supervisor/views/test_operator.py
index 0b0c4cf13..150a33002 100644
--- a/tests/supervisor/views/test_operator.py
+++ b/tests/supervisor/views/test_operator.py
@@ -350,7 +350,9 @@ async def test_websocket_logs_invalid_auth(aiohttp_client, mocker):
     response = await websocket.receive()
     # Subject to change in the future, for now the connexion si broken and closed
     assert response.type == aiohttp.WSMsgType.TEXT
-    assert response.data == '{"status": "failed", "reason": "string indices must be integers"}'
+    assert (
+        response.data == '{"status": "failed", "reason": "Invalid format for auth packet, see /doc/operator_auth.md"}'
+    )
     response = await websocket.receive()
     assert response.type == aiohttp.WSMsgType.CLOSE
     assert websocket.closed

From 3e76ed62c3a39400756b82ec58892efe75c201c8 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Tue, 17 Sep 2024 11:23:03 +0200
Subject: [PATCH 857/990] Doc: No documentation on operator API auth

The custom authentication protocol used to access
the operator API (logs, reboot, ... of a VM) was
not documented.
---
 doc/operator_auth.md | 171 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 171 insertions(+)
 create mode 100644 doc/operator_auth.md

diff --git a/doc/operator_auth.md b/doc/operator_auth.md
new file mode 100644
index 000000000..b2e5dd630
--- /dev/null
+++ b/doc/operator_auth.md
@@ -0,0 +1,171 @@
+ Authentication protocol for VM owner
+=======================================
+
+This custom protocol allows a user (owner of a VM) to securely authenticate to a CRN, using their Ethereum wallet.
+This scheme was designed in a way that's convenient to be integrated in the console web page.
+
+It allows the user to control their VM. e.g : stop reboot, view their log, etc…
+
+## Motivations
+
+This protocol ensures secure authentication between a blockchain wallet owner and an aleph.im compute node.
+
+Signing operations is typically gated by prompts requiring manual approval for each operation.
+With hardware wallets, users are prompted both by the software on their device and the hardware wallet itself.
+
+## Overview
+
+The client generates a [JSON Web Key](https://www.rfc-editor.org/rfc/rfc7517) (JWK) key pair and signs the public key with their Ethereum account. The signed public key is sent
+in the `X-SignedPubKey` header. The client also signs the operation payload with the private JWK, sending it in the
+`X-SignedOperation` header. The server verifies both the public key and payload signatures, ensuring the request's
+integrity and authenticity. If validation fails (e.g., expired key or invalid signature), the server returns a 401
+Unauthorized error.
+
+Support for Solana wallets is planned in the near future.
+
+## Authentication Method for HTTP Endpoints
+
+Two custom headers are added to each authenticated request:
+
+* X-SignedPubKey: This contains the public key and its associated metadata (such as the sender’s address and expiration
+  date), along with a signature that ensures its authenticity.
+* X-SignedOperation: This includes the payload of the operation and its cryptographic signature, ensuring that the
+  operation itself has not been tampered with.
+
+### 1. Generate and Sign Public Key
+
+A new JWK is generated using elliptic curve cryptography (EC, P-256).
+
+The use of a temporary JWK key allows the user to delegate limited control to the console without needing to sign every
+individual request with their Ethereum wallet. This is crucial for improving the user experience, as constantly signing
+each operation would be cumbersome and inefficient. By generating a temporary key, the user can provide permission for a
+set period of time (until the key expires), enabling the console to perform actions like stopping or rebooting the VM on
+their behalf. This maintains security while streamlining interactions with the console, as the server verifies each
+operation using the temporary key without requiring ongoing involvement from the user's wallet.
+
+The generated public key is converted into a JSON structure with additional metadata:
+* `pubkey`: The public key information.
+* `alg`: The signing algorithm, ECDSA.
+* `domain`: The domain for which the key is valid.
+* `address`: The Ethereum address of the sender, binding the public key to this identity.
+* `expires`: The expiration time of the key.
+
+Example
+```json
+{
+  "pubkey": {
+    "crv": "P-256",
+    "kty": "EC",
+    "x": "hbslLmhG3h2RwuzBYNVeQ7WCbU-tUzMjSpCFO2i5-tA",
+    "y": "KI4FJARKwyYcRy6xz1J9lu8OItV87Fw91eThe2hnnuc"
+  },
+  "alg": "ECDSA",
+  "domain": "localhost",
+  "address": "0x8Dd070629F107e7946dD68BDcb8ABE8475F47B0E",
+  "expires": "2010-12-26T17:05:55Z"
+}
+```
+
+This public key is signed using the Ethereum account to ensure its authenticity. The resulting signature is
+combined with the public key into a payload and sent as the `X-SignedPubKey` header.
+
+### 2. Sign Operation Payload
+
+#### Operation Payload Format
+
+The operation payload is a JSON object that encapsulates the details of an API request. It ensures that the request's
+integrity can be verified through signing. Below are the fields included:
+
+- **`time`**: (string, ISO 8601 format) The timestamp for when the operation is valid, including the timezone is mandatory (`Z`
+  indicates UTC). This helps prevent replay attacks (capturing the packet and replying it multiple time). e.g. `"2010-12-25T17:05:55Z"`
+- **`method`**: (string) The HTTP method used for the operation (e.g., `GET`, `POST`).
+- **`path`**: (string) The endpoint path of the request (e.g., `/`).
+- **`domain`**: (string) The domain associated with the request. This ensures the request is valid for the intended
+  CRN. (e.g., `localhost`).
+
+Example
+
+```json
+{
+  "time": "2010-12-25T17:05:55Z",
+  "method": "GET",
+  "path": "/",
+  "domain": "localhost"
+}
+```
+
+It is sent serialized as a hex string.
+
+#### Signature
+This payload is serialized in JSON, signed, and sent in the `X-SignedOperation` header to ensure the integrity and authenticity
+of the request.
+
+* The operation payload (containing details such as time, method, path, and domain) is serialized and converted into a byte array.
+* The JWK (private key) is used to sign this operation payload, ensuring its integrity. This signature is then included in the X-SignedOperation header.
+
+
+###  3. Include authentication Headers
+These two headers are to be added to the HTTP Request:
+
+1. **`X-SignedPubKey` Header:**
+    - This header contains the public key payload and the signature of the public key generated by the Ethereum account.
+
+   Example:
+   ```json
+   {
+      "payload": "",
+      "signature": ""
+   }
+   ```
+
+2. **`X-SignedOperation` Header:**
+    - This header contains the operation payload and the signature of the operation payload generated using the private
+      JWK.
+
+   Example:
+   ```json
+   {
+      "payload": "",
+      "signature": ""
+   }
+   ```
+
+### Expiration and Validation
+
+- The public key has an expiration date, ensuring that keys are not used indefinitely.
+- Both the public key and the operation signature are validated for authenticity and integrity at the server side.
+- Requests failing verification or expired keys are rejected with `401 Unauthorized` status, providing an error message
+  indicating the reason.
+
+### WebSocket Authentication Protocol
+
+In the WebSocket variant of the authentication protocol, the client establishes a connection and authenticates through
+an initial message that includes their Ethereum-signed identity, ensuring secure communication.
+
+Due to web browsers not allowing custom HTTP headers in WebSocket connections, 
+the two header are sent in one json packet, under the `auth` key.
+
+Example authentication packet 
+```json
+{
+    "auth": {
+        "X-SignedPubKey": {
+            "payload": "7b227075626b6579223a207b22637276223a2022502d323536222c20226b7479223a20224543222c202278223a20223962446f34754949686b735a5272677a31477972325050656d4334364e735f4730577144364d4d6a774673222c202279223a20226f48343342786c7854334f3065733336685967713143372d61325a535a71456d5f6b56356e636c79667a59227d2c2022616c67223a20224543445341222c2022646f6d61696e223a20226c6f63616c686f7374222c202261646472657373223a2022307862413236623135333539314434363230666432413734304130463165463730644164363532336230222c202265787069726573223a2022323031302d31322d32365431373a30353a35355a227d",
+            "signature": "0xea99ef5f1a10f2d103f94dce4f8650730315246e6d15cf9e5862c11adfd6482703cd1ec684a4f3dffb36ae5c4a57b08a47108fe55e3b2454e45f6e63342e0f471b"
+        },
+        "X-SignedOperation": {
+            "payload": "7b2274696d65223a2022323031302d31322d32355431373a30353a35355a222c20226d6574686f64223a2022474554222c202270617468223a20222f222c2022646f6d61696e223a20226c6f63616c686f7374227d",
+            "signature": "6f737654cd00e4d4155d387509978e7a9a4d27f5b59c9492ac1dec7b09f9aecc58c9365526bbddd6211b65f40f4956c50ab26f395f7170ce1698c11e28e25d3a"
+        }
+    }
+}
+```
+
+If the  authentication  succeed the server will answer with
+```json
+{
+  "status": "connected"
+}
+```
+
+In case of failed auth the server will respond with await `{"status": "failed", "reason": "string describing the reason"})` and close the connexion  

From 10fc2c71238aae8e2ed79b70624dc156231bd06d Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 3 May 2024 13:28:25 +0200
Subject: [PATCH 858/990] CI: Calculate the droplet ip only once

---
 .github/workflows/test-on-droplets-matrix.yml | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index 84b3c1e15..e3883a89f 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -105,15 +105,16 @@ jobs:
           cd packaging && make ${{ matrix.os_config.package_build_command }} && cd ..
           ls packaging/target
 
+      - name: Get droplet ip and export it in env
+        run: |
+          echo "DROPLET_IPV4=$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)" >>  "$GITHUB_ENV"
+
       - name: Wait for the system to setup and boot
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done
         timeout-minutes: 3
-
       - name: Install Aleph-VM on the Droplet
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
 
           # Wait a few seconds for DigitalOcean to setup the Droplet using apt, which conflicts with our comands:
@@ -138,16 +139,12 @@ jobs:
         if: always()
         continue-on-error: true
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi${{ matrix.check_vm.query_params }}"
 
       - name: Test Aleph-VM on the Droplet again restarting the server first
         if: steps.test-aleph-vm.outcome == 'failure'
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-
           # If the first execution fails, restart supervisor and try again
           ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor"
           sleep 5
@@ -156,7 +153,6 @@ jobs:
 
       - name: Schedule an instance on the Droplet by faking a call from the scheduler
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           curl --retry 5 --max-time 10 --fail -X POST -H "Content-Type: application/json" \
             -H "X-Auth-Signature: test" \
             -d '{"persistent_vms": [], "instances": ["${{ matrix.check_vm.item_hash }}"]}' \
@@ -178,7 +174,6 @@ jobs:
       - name: Export aleph logs
         if: always()
         run: |
-          export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh root@${DROPLET_IPV4} "journalctl -u aleph-vm-supervisor"
 
       - name: Cleanup

From 8cf10e4eaf4f1709054e4bba78c3dfdc6b2c8dd4 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 4 Sep 2024 12:21:59 +0200
Subject: [PATCH 859/990] Do not copy the configuration on the server, instead
 ssh it on the server

---
 .github/workflows/test-on-droplets-matrix.yml | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index e3883a89f..ed08db61d 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -98,10 +98,6 @@ jobs:
 
       - name: Build Package
         run: |
-          echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> packaging/aleph-vm/etc/aleph-vm/supervisor.env
-          echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> packaging/aleph-vm/etc/aleph-vm/supervisor.env
-          echo ALEPH_VM_CHECK_FASTAPI_VM_ID=${{ matrix.check_vm.item_hash }} >> packaging/aleph-vm/etc/aleph-vm/supervisor.env
-          echo ALEPH_VM_SENTRY_DSN=${{ secrets.SENTRY_DSN }} >> packaging/aleph-vm/etc/aleph-vm/supervisor.env
           cd packaging && make ${{ matrix.os_config.package_build_command }} && cd ..
           ls packaging/target
 
@@ -117,6 +113,14 @@ jobs:
         run: |
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
 
+          # Configuration
+          echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> supervisor.env
+          echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> supervisor.env
+          echo ALEPH_VM_CHECK_FASTAPI_VM_ID=${{ matrix.check_vm.item_hash }} >> supervisor.env
+          echo ALEPH_VM_SENTRY_DSN=${{ secrets.SENTRY_DSN }} >> supervisor.env
+          ssh  root@${DROPLET_IPV4} mkdir -p /etc/aleph-vm/
+          scp supervisor.env root@${DROPLET_IPV4}:/etc/aleph-vm/supervisor.env
+
           # Wait a few seconds for DigitalOcean to setup the Droplet using apt, which conflicts with our comands:
           sleep 5
 
@@ -129,7 +133,8 @@ jobs:
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
 
           scp packaging/target/${{ matrix.os_config.package_name }} root@${DROPLET_IPV4}:/opt
-          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y /opt/${{ matrix.os_config.package_name }}"
+          # "--force-confold" keeps existing config files during package install/upgrade, avoiding prompts.
+          ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60  -o Dpkg::Options::="--force-confold" install -y /opt/${{ matrix.os_config.package_name }}"
 
           # Allow some time for IPFS Kubo to start
           sleep 5

From c8ded1b95de18c96b7a721db611c5502b1aeefe7 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 20 Sep 2024 12:10:46 +0200
Subject: [PATCH 860/990] Debug Error Unknown exception while deleting address

Add more debug info to help investigate the error
Jira: ALEPH-115

This convert the logging.error to a logging.exception
Which mean the traceback will be included in Sentry as well as more debug information
---
 src/aleph/vm/network/interfaces.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index b90c84b4e..f401b507f 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -70,9 +70,9 @@ def delete_ip_address(ipr: IPRoute, device_name: str, ip: IPv4Interface | IPv6In
     try:
         ipr.addr("del", index=interface_index[0], address=str(ip.ip), mask=ip.network.prefixlen)
     except NetlinkError as e:
-        logger.error(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
+        logger.exception(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
     except OSError as e:
-        logger.error(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
+        logger.exception(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
 
 
 def set_link_up(ipr: IPRoute, device_name: str):

From 5518086ac869143ab40928cac51831c8510f4007 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 18 Sep 2024 16:04:08 +0200
Subject: [PATCH 861/990] Problem: error org.freedesktop.systemd1.NoSuchUnit:
 Unit aleph-vm-controller@...

Jira issue: ALEPH-113

This happened because we tried to check if the service was running
before checking if it was enabled. It resulted in no proper error
but was creating an error message in the logs

Solution: Check if service is enabled before checking if it is active

How to test:
Launch instances. Restart the aleph-vm process, stop them. That error
message should not display
---
 src/aleph/vm/systemd.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/systemd.py b/src/aleph/vm/systemd.py
index 001c4671d..fec117164 100644
--- a/src/aleph/vm/systemd.py
+++ b/src/aleph/vm/systemd.py
@@ -60,7 +60,10 @@ def is_service_enabled(self, service: str) -> bool:
 
     def is_service_active(self, service: str) -> bool:
         try:
-            systemd_service = self.bus.get_object("org.freedesktop.systemd1", object_path=self.manager.GetUnit(service))
+            if not self.is_service_enabled(service):
+                return False
+            unit_path = self.manager.GetUnit(service)
+            systemd_service = self.bus.get_object("org.freedesktop.systemd1", object_path=unit_path)
             unit = dbus.Interface(systemd_service, "org.freedesktop.systemd1.Unit")
             unit_properties = dbus.Interface(unit, "org.freedesktop.DBus.Properties")
             active_state = unit_properties.Get("org.freedesktop.systemd1.Unit", "ActiveState")

From f5654c93fe6adfaa9ccf9d7b244956f9522bc5fe Mon Sep 17 00:00:00 2001
From: Laurent Peuch 
Date: Wed, 25 Sep 2024 00:20:13 +0200
Subject: [PATCH 862/990] feat: add return type to aleph.vm.utils.to_json

---
 src/aleph/vm/utils/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/utils/__init__.py b/src/aleph/vm/utils/__init__.py
index 507a41e0e..15a57819a 100644
--- a/src/aleph/vm/utils/__init__.py
+++ b/src/aleph/vm/utils/__init__.py
@@ -69,7 +69,7 @@ async def get_ref_from_dns(domain):
     return record[0].text
 
 
-def to_json(o: Any):
+def to_json(o: Any) -> dict | str:
     if hasattr(o, "to_dict"):  # default method
         return o.to_dict()
     elif hasattr(o, "dict"):  # Pydantic

From 861199c46c3f00da2c3ed024561796ac282d3f25 Mon Sep 17 00:00:00 2001
From: Laurent Peuch 
Date: Wed, 25 Sep 2024 23:40:59 +0200
Subject: [PATCH 863/990] feat: add pyproject-fmt

---
 pyproject.toml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index 33457e454..c8888cf3e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -116,6 +116,7 @@ dependencies = [
   "mypy==1.8.0",
   "ruff==0.4.6",
   "isort==5.13.2",
+  "pyproject-fmt==2.2.1",
 ]
 [tool.hatch.envs.lint.scripts]
 typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}"
@@ -123,11 +124,13 @@ style = [
 #  "ruff {args:.}",
   "black --check --diff {args:.}",
   "isort --check-only --profile black {args:.}",
+  "pyproject-fmt --check pyproject.toml",
 ]
 fmt = [
   "black {args:.}",
 #  "ruff --fix {args:.}",
   "isort --profile black {args:.}",
+  "pyproject-fmt pyproject.toml",
   "style",
 ]
 all = [

From 3c807fabd25819bb97b65e9f710a190abedc91c9 Mon Sep 17 00:00:00 2001
From: Laurent Peuch 
Date: Wed, 25 Sep 2024 23:43:14 +0200
Subject: [PATCH 864/990] fix: run pyproject-fmt

---
 pyproject.toml | 171 ++++++++++++++++++++++++-------------------------
 1 file changed, 83 insertions(+), 88 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index c8888cf3e..83ea1315c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,18 +1,18 @@
 [build-system]
-requires = ["hatchling", "hatch-vcs"]
 build-backend = "hatchling.build"
 
+requires = [ "hatch-vcs", "hatchling" ]
+
 [project]
 name = "aleph-vm"
-dynamic = ["version"]
 description = "Aleph.im VM execution engine"
 readme = "README.md"
-requires-python = ">=3.10"
-license = {file = "LICENSE"}
-keywords = []
+keywords = [  ]
+license = { file = "LICENSE" }
 authors = [
-  { name="Hugo Herter", email="git@hugoherter.com" },
+  { name = "Hugo Herter", email = "git@hugoherter.com" },
 ]
+requires-python = ">=3.10"
 classifiers = [
   "Development Status :: 4 - Beta",
   "Environment :: Console",
@@ -20,62 +20,62 @@ classifiers = [
   "Intended Audience :: Information Technology",
   "License :: OSI Approved :: MIT License",
   "Operating System :: POSIX :: Linux",
-  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Topic :: System :: Distributed Computing",
 ]
+dynamic = [ "version" ]
 dependencies = [
-  "pydantic[dotenv]~=1.10.13",
+  "aiodns==3.1",
   "aiohttp==3.9.5",
-  "aiodns==3.1.0",
-  "setproctitle==1.3.3",
-  "pyyaml==6.0.1",
+  "aiohttp-cors~=0.7.0",
+  "aioredis==1.3.1",
+  "aiosqlite==0.19",
+  "alembic==1.13.1",
   "aleph-message==0.4.9",
+  "aleph-superfluid~=0.2.1",
+  "dbus-python==1.3.2",
   "eth-account~=0.10",
-  "sentry-sdk==1.31.0",
-  "aioredis==1.3.1",
-  "psutil==5.9.5",
-  "py-cpuinfo==9.0.0",
-  "schedule==1.2.1",
-  "nftables @ git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py",
+  "jsonschema==4.19.1",
+  "jwcrypto==1.5.6",
   "msgpack==1.0.7",
+  "nftables @ git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py",
   "packaging==23.2",
-  "jsonschema==4.19.1",
-  "qmp==1.1.0",
-  "dbus-python==1.3.2",
-  "systemd-python==235",
-  "systemd-python==235",
-  "aleph-superfluid~=0.2.1",
-  "sqlalchemy[asyncio]>=2.0",
-  "aiosqlite==0.19.0",
-  "alembic==1.13.1",
-  "aiohttp_cors~=0.7.0",
+  "psutil==5.9.5",
+  "py-cpuinfo==9",
+  "pydantic[dotenv]~=1.10.13",
   "pyroute2==0.7.12",
-  "jwcrypto==1.5.6",
-  "python-cpuid==0.1.0"
+  "python-cpuid==0.1",
+  "pyyaml==6.0.1",
+  "qmp==1.1",
+  "schedule==1.2.1",
+  "sentry-sdk==1.31",
+  "setproctitle==1.3.3",
+  "sqlalchemy[asyncio]>=2",
+  "systemd-python==235",
 ]
 
-[project.urls]
-Documentation = "https://docs.aleph.im/nodes/compute/"
-Issues = "https://github.com/aleph-im/aleph-vm/issues"
-Source = "https://github.com/aleph-im/aleph-vm"
-Discussions = "https://community.aleph.im/"
-
-[project.scripts]
-aleph-vm = "aleph.vm.orchestrator.cli:main"
+urls.Discussions = "https://community.aleph.im/"
+urls.Documentation = "https://docs.aleph.im/nodes/compute/"
+urls.Issues = "https://github.com/aleph-im/aleph-vm/issues"
+urls.Source = "https://github.com/aleph-im/aleph-vm"
+scripts.aleph-vm = "aleph.vm.orchestrator.cli:main"
 
 [tool.hatch.version]
 source = "vcs"
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/aleph"]
+packages = [ "src/aleph" ]
 
 [tool.hatch.metadata]
 allow-direct-references = true
 
 [tool.hatch.envs.default]
-platforms = ["linux"]
+platforms = [ "linux" ]
 dependencies = [
-#  "git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py",
+  #  "git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py",
 ]
 
 [tool.hatch.envs.default.scripts]
@@ -87,7 +87,7 @@ check = "aleph-vm controller run {args:--help}"
 type = "virtual"
 system-packages = true
 dependencies = [
-  "eth_typing==4.3.1", # Temp fix for bug in CI with 5.0.0
+  "eth_typing==4.3.1",      # Temp fix for bug in CI with 5.0.0
   "pytest==8.2.1",
   "pytest-cov==5.0.0",
   "pytest-mock==3.14.0",
@@ -107,7 +107,7 @@ cov = [
 ]
 
 [[tool.hatch.envs.all.matrix]]
-python = ["3.10", "3.11", "3.12"]
+python = [ "3.10", "3.11", "3.12" ]
 
 [tool.hatch.envs.lint]
 detached = true
@@ -121,14 +121,14 @@ dependencies = [
 [tool.hatch.envs.lint.scripts]
 typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}"
 style = [
-#  "ruff {args:.}",
+  #  "ruff {args:.}",
   "black --check --diff {args:.}",
   "isort --check-only --profile black {args:.}",
   "pyproject-fmt --check pyproject.toml",
 ]
 fmt = [
   "black {args:.}",
-#  "ruff --fix {args:.}",
+  #  "ruff --fix {args:.}",
   "isort --profile black {args:.}",
   "pyproject-fmt pyproject.toml",
   "style",
@@ -138,36 +138,15 @@ all = [
   "typing",
 ]
 
-[tool.pytest.ini_options]
-pythonpath = [
-  "src"
-]
-testpaths = [
-  "tests"
-]
-norecursedirs = [
-  "runtimes/aleph-debian-11-python/rootfs/",
-  "runtimes/aleph-debian-12-python/rootfs/",
-]
-
 [tool.black]
-target-version = ["py310"]
+target-version = [ "py310" ]
 line-length = 120
 #skip-string-normalization = true
 
-[tool.mypy]
-python_version = "3.10"
-install_types = true
-non_interactive = true
-ignore_missing_imports = true
-explicit_package_bases = true
-check_untyped_defs = true
-
 [tool.ruff]
 target-version = "py310"
 line-length = 120
-[tool.ruff.lint]
-select = [
+lint.select = [
   "A",
   "ARG",
   "B",
@@ -194,34 +173,42 @@ select = [
   "W",
   "YTT",
 ]
-ignore = [
-#  # Allow non-abstract empty methods in abstract base classes
-#  "B027",
-#  # Allow boolean positional values in function calls, like `dict.get(... True)`
-#  "FBT003",
-#  # Ignore checks for possible passwords
-#  "S105", "S106", "S107",
-#  # Ignore complexity
-#  "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
+lint.ignore = [
+  #  # Allow non-abstract empty methods in abstract base classes
+  #  "B027",
+  #  # Allow boolean positional values in function calls, like `dict.get(... True)`
+  #  "FBT003",
+  #  # Ignore checks for possible passwords
+  #  "S105", "S106", "S107",
+  #  # Ignore complexity
+  #  "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
   # Allow the use of assert statements
-  "S101"
+  "S101",
 ]
+# Tests can use magic values, assertions, and relative imports
+lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ]
+#[tool.ruff.flake8-tidy-imports]
+#ban-relative-imports = "all"
 #unfixable = [
 #  # Don't touch unused imports
 #  "F401",
 #]
+lint.isort = [ "aleph.vm" ]
 
-isort.known-first-party = ["aleph.vm"]
-
-#[tool.ruff.flake8-tidy-imports]
-#ban-relative-imports = "all"
-
-[tool.ruff.lint.per-file-ignores]
-# Tests can use magic values, assertions, and relative imports
-"tests/**/*" = ["PLR2004", "S101", "TID252"]
+[tool.pytest.ini_options]
+pythonpath = [
+  "src",
+]
+testpaths = [
+  "tests",
+]
+norecursedirs = [
+  "runtimes/aleph-debian-11-python/rootfs/",
+  "runtimes/aleph-debian-12-python/rootfs/",
+]
 
 [tool.coverage.run]
-source_pkgs = ["aleph.vm", "tests"]
+source_pkgs = [ "aleph.vm", "tests" ]
 branch = true
 parallel = true
 omit = [
@@ -229,8 +216,8 @@ omit = [
 ]
 
 [tool.coverage.paths]
-aleph_vm = ["src/aleph/vm", "*/aleph-vm/src/aleph/vm"]
-tests = ["tests", "*/aleph-vm/tests"]
+aleph_vm = [ "src/aleph/vm", "*/aleph-vm/src/aleph/vm" ]
+tests = [ "tests", "*/aleph-vm/tests" ]
 
 [tool.coverage.report]
 exclude_lines = [
@@ -238,3 +225,11 @@ exclude_lines = [
   "if __name__ == .__main__.:",
   "if TYPE_CHECKING:",
 ]
+
+[tool.mypy]
+python_version = "3.10"
+install_types = true
+non_interactive = true
+ignore_missing_imports = true
+explicit_package_bases = true
+check_untyped_defs = true

From 9015baf641795845cec80c4021b79f740d86ee8a Mon Sep 17 00:00:00 2001
From: Laurent Peuch 
Date: Wed, 25 Sep 2024 23:29:30 +0200
Subject: [PATCH 865/990] feat: add yamlfix

---
 pyproject.toml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index 83ea1315c..7b9a638af 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -116,6 +116,7 @@ dependencies = [
   "mypy==1.8.0",
   "ruff==0.4.6",
   "isort==5.13.2",
+  "yamlfix==1.16.1",
   "pyproject-fmt==2.2.1",
 ]
 [tool.hatch.envs.lint.scripts]
@@ -124,12 +125,14 @@ style = [
   #  "ruff {args:.}",
   "black --check --diff {args:.}",
   "isort --check-only --profile black {args:.}",
+  "yamlfix --check .",
   "pyproject-fmt --check pyproject.toml",
 ]
 fmt = [
   "black {args:.}",
   #  "ruff --fix {args:.}",
   "isort --profile black {args:.}",
+  "yamlfix .",
   "pyproject-fmt pyproject.toml",
   "style",
 ]
@@ -233,3 +236,9 @@ non_interactive = true
 ignore_missing_imports = true
 explicit_package_bases = true
 check_untyped_defs = true
+
+[tool.yamlfix]
+sequence_style = "keep_style"
+preserve_quotes = true
+whitelines = 1
+section_whitelines = 2

From 15bd5f5b38a81ed6fae10e3d5baef1875e1d9729 Mon Sep 17 00:00:00 2001
From: Laurent Peuch 
Date: Thu, 3 Oct 2024 16:35:53 +0200
Subject: [PATCH 866/990] fix: run yamlfix

---
 .github/workflows/build-deb-package.yml       |  5 ++--
 .github/workflows/codeql-analysis.yml         | 29 ++++++++++---------
 .github/workflows/deploy-main-on-staging.yml  |  6 ++--
 .github/workflows/pr-rating.yml               |  4 +++
 .github/workflows/test-build-examples.yml     |  6 ++--
 .../workflows/test-new-runtime-examples.yml   | 13 +++++----
 .github/workflows/test-on-droplets-matrix.yml | 12 ++++----
 .github/workflows/test-using-pytest.yml       | 11 ++++---
 8 files changed, 49 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml
index 544b523d5..624117f55 100644
--- a/.github/workflows/build-deb-package.yml
+++ b/.github/workflows/build-deb-package.yml
@@ -1,6 +1,7 @@
+---
 name: "Build Packages"
-on:
-  push
+on: push
+
 
 jobs:
   build_deb:
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index d928b37c0..fda8e50e3 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -1,3 +1,4 @@
+---
 # For most projects, this workflow file will not need changing; you simply need
 # to commit it to your repository.
 #
@@ -11,15 +12,17 @@
 #
 name: "CodeQL"
 
+
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ main ]
+    branches: [main]
   schedule:
     - cron: '15 16 * * 0'
 
+
 jobs:
   analyze:
     name: Analyze
@@ -32,20 +35,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'python' ]
+        language: ['python']
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
         # Learn more:
         # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
-      with:
-        languages: ${{ matrix.language }}
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
@@ -53,8 +56,8 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -67,5 +70,5 @@ jobs:
     #   make bootstrap
     #   make release
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
diff --git a/.github/workflows/deploy-main-on-staging.yml b/.github/workflows/deploy-main-on-staging.yml
index 51a9f43c3..76d4a3343 100644
--- a/.github/workflows/deploy-main-on-staging.yml
+++ b/.github/workflows/deploy-main-on-staging.yml
@@ -1,11 +1,14 @@
+---
 # This workflow automatically deploys main on staging
 name: "Deploy `main` automatically on staging"
 
+
 on:
   push:
     branches:
       - main
 
+
 jobs:
   deploy_staging_servers:
     name: "Deploying on ${{ matrix.staging_servers.hostname }}"
@@ -51,7 +54,7 @@ jobs:
           STAGING_SSH_PRIVATE_KEY: ${{ secrets.STAGING_SSH_PRIVATE_KEY }}
 
       - name: Install Aleph-VM on the Staging servers
-        run: |
+        run: |-
           echo ${{ matrix.staging_servers.host_keys }} | base64 --decode > ~/.ssh/known_hosts
 
           # Wait for /var/lib/apt/lists/lock to be unlocked on the remote host via SSH.
@@ -59,4 +62,3 @@ jobs:
 
           scp packaging/target/${{ matrix.staging_servers.artifact_name }} root@${{ matrix.staging_servers.hostname }}:/opt
           ssh root@${{ matrix.staging_servers.hostname }} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y --allow-downgrades /opt/${{ matrix.staging_servers.artifact_name }}"
-
diff --git a/.github/workflows/pr-rating.yml b/.github/workflows/pr-rating.yml
index 5878fd146..e1f4e2cb6 100644
--- a/.github/workflows/pr-rating.yml
+++ b/.github/workflows/pr-rating.yml
@@ -1,12 +1,16 @@
+---
 name: Test PR Difficulty Rating Action
 
+
 permissions:
   pull-requests: write
 
+
 on:
   pull_request:
     types: [opened, reopened, ready_for_review]
 
+
 jobs:
   difficulty-rating:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/test-build-examples.yml b/.github/workflows/test-build-examples.yml
index 2cf702c89..dc4c76ade 100644
--- a/.github/workflows/test-build-examples.yml
+++ b/.github/workflows/test-build-examples.yml
@@ -1,8 +1,8 @@
-
+---
 
 name: "Build Examples"
-on:
-  push
+on: push
+
 
 jobs:
   build_pip:
diff --git a/.github/workflows/test-new-runtime-examples.yml b/.github/workflows/test-new-runtime-examples.yml
index f48ae2ac4..25c65302d 100644
--- a/.github/workflows/test-new-runtime-examples.yml
+++ b/.github/workflows/test-new-runtime-examples.yml
@@ -1,6 +1,7 @@
+---
 name: "Test new runtime and examples"
-on:
-  push
+on: push
+
 
 jobs:
   run_debian_12:
@@ -74,12 +75,12 @@ jobs:
         run: |
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
-          
+
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 update"
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 upgrade -y"
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y docker.io apparmor-profiles"
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
-          
+
           scp packaging/target/aleph-vm.debian-12.deb root@${DROPLET_IPV4}:/opt
           scp -pr ./examples root@${DROPLET_IPV4}:/opt/
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt -o DPkg::Lock::Timeout=60 install -y /opt/aleph-vm.debian-12.deb"
@@ -91,7 +92,7 @@ jobs:
       - name: Test Aleph-VM on the Droplet
         run: |
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          
+
           sleep 3
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
@@ -104,5 +105,5 @@ jobs:
 
       - name: Cleanup
         if: always()
-        run: |
+        run: |-
           doctl compute droplet delete -f aleph-vm-ci-runtime
diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index ed08db61d..dce875bf3 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -1,3 +1,4 @@
+---
 # These are end-to-end tests running on ephemeral DigitalOcean "Droplet" virtual machines
 # with the different operating systems that are supported.
 #
@@ -18,11 +19,14 @@ on:
       - "ready_for_review"
   workflow_dispatch:
 
+
 jobs:
   run_on_droplet:
-    name: "Test Droplet with ${{ matrix.os_config.os_name }}-${{ matrix.check_vm.alias }}"
+    name: "Test Droplet with ${{ matrix.os_config.os_name }}-${{ matrix.check_vm.alias\
+      \ }}"
     runs-on: ubuntu-latest
-    concurrency: "${{ matrix.os_config.concurrency_group }}-${{ matrix.check_vm.alias }}"
+    concurrency: "${{ matrix.os_config.concurrency_group }}-${{ matrix.check_vm.alias\
+      \ }}"
     timeout-minutes: 10
 
     strategy:
@@ -169,13 +173,11 @@ jobs:
           curl -X GET -H "Content-Type: application/json" \
             "http://${DROPLET_IPV4}:4020/about/usage/system"
 
-
       - name: Run the sevctl command to ensure it's properly packaged and working
         run: |
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh root@${DROPLET_IPV4} "/opt/sevctl --version"
 
-
       - name: Export aleph logs
         if: always()
         run: |
@@ -183,7 +185,7 @@ jobs:
 
       - name: Cleanup
         if: always()
-        run: |
+        run: |-
           DROPLET_IDS=$(doctl compute droplet list --format "ID,Name" --no-header | grep "aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }}" | awk '{print $1}')
 
           for DROPLET_ID in $DROPLET_IDS; do
diff --git a/.github/workflows/test-using-pytest.yml b/.github/workflows/test-using-pytest.yml
index 0ca8ec032..067f8255e 100644
--- a/.github/workflows/test-using-pytest.yml
+++ b/.github/workflows/test-using-pytest.yml
@@ -1,7 +1,8 @@
+---
 name: "py.test and linting"
 
-on:
-  push
+on: push
+
 
 jobs:
   tests-python:
@@ -80,13 +81,12 @@ jobs:
           sudo hatch run testing:cov
 
       - name: Output modules used and their version
-        if:  always()
+        if: always()
         run: |
           # re-install hatch in case previous job failed and hatch didn't get installed
           sudo python3 -m pip install hatch hatch-vcs coverage
           sudo hatch -e testing run pip freeze
 
-
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4.0.1
         with:
@@ -108,6 +108,5 @@ jobs:
           sudo apt-get install -y shellcheck
 
       - name: Run Shellcheck on all shell scripts
-        run: |
+        run: |-
           find ./ -type f -name "*.sh" -exec shellcheck {} \;
-

From de4a0ff82fe34cbe198d67df01ee00a8b2f59483 Mon Sep 17 00:00:00 2001
From: Laurent Peuch 
Date: Wed, 25 Sep 2024 23:36:26 +0200
Subject: [PATCH 867/990] chore: rename hatch lint env to linting

This is to uniformise its name with other projects env names
---
 .github/workflows/test-using-pytest.yml | 4 ++--
 pyproject.toml                          | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test-using-pytest.yml b/.github/workflows/test-using-pytest.yml
index 067f8255e..a0b1ec229 100644
--- a/.github/workflows/test-using-pytest.yml
+++ b/.github/workflows/test-using-pytest.yml
@@ -34,11 +34,11 @@ jobs:
 
       - name: Test style wth ruff, black and isort
         run: |
-          hatch run lint:style
+          hatch run linting:style
 
       - name: Test typing with Mypy
         run: |
-          hatch run lint:typing
+          hatch run linting:typing
 
       - name: Install required system packages for installing and running tests
         run: |
diff --git a/pyproject.toml b/pyproject.toml
index 7b9a638af..0fe050f00 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -109,7 +109,7 @@ cov = [
 [[tool.hatch.envs.all.matrix]]
 python = [ "3.10", "3.11", "3.12" ]
 
-[tool.hatch.envs.lint]
+[tool.hatch.envs.linting]
 detached = true
 dependencies = [
   "black==24.3.0",
@@ -119,7 +119,7 @@ dependencies = [
   "yamlfix==1.16.1",
   "pyproject-fmt==2.2.1",
 ]
-[tool.hatch.envs.lint.scripts]
+[tool.hatch.envs.linting.scripts]
 typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}"
 style = [
   #  "ruff {args:.}",

From 15ecb92cf66f4d0c064b2a48e72f86eef426c13c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 8 Oct 2024 13:51:36 +0200
Subject: [PATCH 868/990] Problem: Solana wallet  couln't be used to control
 the VM (#700)

* Improve documentation for auth decorator

* Problem: Solana wallet  couln't be used to control the VM

* Add new dep to debian package

* Fix mypy error not related to PR

Had do ignore since mypy expect a DataClassIstance that don't exist

* Document solana support for operator endpoints

* mod: Move signature checking for all chain in a function
---
 doc/operator_auth.md                          | 108 ++++++++++--------
 packaging/Makefile                            |   2 +-
 pyproject.toml                                |   3 +
 .../vm/orchestrator/views/authentication.py   |  57 +++++++--
 src/aleph/vm/utils/__init__.py                |   2 +-
 tests/supervisor/test_authentication.py       |  74 ++++++++++++
 6 files changed, 188 insertions(+), 58 deletions(-)

diff --git a/doc/operator_auth.md b/doc/operator_auth.md
index b2e5dd630..b37b3e397 100644
--- a/doc/operator_auth.md
+++ b/doc/operator_auth.md
@@ -1,56 +1,59 @@
  Authentication protocol for VM owner
 =======================================
 
-This custom protocol allows a user (owner of a VM) to securely authenticate to a CRN, using their Ethereum wallet.
-This scheme was designed in a way that's convenient to be integrated in the console web page.
+This custom protocol allows a user (owner of a VM) to securely authenticate to a CRN, using their Ethereum or Solana
+wallet. This scheme was designed in a way that's convenient to be integrated into the console web page.
 
-It allows the user to control their VM. e.g : stop reboot, view their log, etc…
+It allows the user to control their VM. e.g: stop, reboot, view their log, etc.
 
 ## Motivations
 
 This protocol ensures secure authentication between a blockchain wallet owner and an aleph.im compute node.
 
-Signing operations is typically gated by prompts requiring manual approval for each operation.
-With hardware wallets, users are prompted both by the software on their device and the hardware wallet itself.
+Signing operations are typically gated by prompts requiring manual approval for each operation. With hardware wallets,
+users are prompted both by the software on their device and the hardware wallet itself.
 
 ## Overview
 
-The client generates a [JSON Web Key](https://www.rfc-editor.org/rfc/rfc7517) (JWK) key pair and signs the public key with their Ethereum account. The signed public key is sent
-in the `X-SignedPubKey` header. The client also signs the operation payload with the private JWK, sending it in the
-`X-SignedOperation` header. The server verifies both the public key and payload signatures, ensuring the request's
-integrity and authenticity. If validation fails (e.g., expired key or invalid signature), the server returns a 401
-Unauthorized error.
+The client generates a [JSON Web Key](https://www.rfc-editor.org/rfc/rfc7517) (JWK) key pair and signs the public key
+with their Ethereum or Solana account. The signed public key is sent in the `X-SignedPubKey` header. The client also
+signs the operation payload with the private JWK, sending it in the `X-SignedOperation` header. The server verifies both
+the public key and payload signatures, ensuring the request's integrity and authenticity. If validation fails (e.g.,
+expired key or invalid signature), the server returns a 401 Unauthorized error.
 
-Support for Solana wallets is planned in the near future.
 
 ## Authentication Method for HTTP Endpoints
 
 Two custom headers are added to each authenticated request:
 
-* X-SignedPubKey: This contains the public key and its associated metadata (such as the sender’s address and expiration
-  date), along with a signature that ensures its authenticity.
-* X-SignedOperation: This includes the payload of the operation and its cryptographic signature, ensuring that the
+- **X-SignedPubKey**: This contains the public key and its associated metadata (such as the sender’s address, chain, and
+  expiration date), along with a signature that ensures its authenticity.
+- **X-SignedOperation**: This includes the payload of the operation and its cryptographic signature, ensuring that the
   operation itself has not been tampered with.
 
-### 1. Generate and Sign Public Key
+### 1. Generate an ephemeral keys and Sign Public Key
 
-A new JWK is generated using elliptic curve cryptography (EC, P-256).
+An ephemeral key pair (as JWK) is generated using elliptic curve cryptography (EC, P-256).
 
 The use of a temporary JWK key allows the user to delegate limited control to the console without needing to sign every
-individual request with their Ethereum wallet. This is crucial for improving the user experience, as constantly signing
-each operation would be cumbersome and inefficient. By generating a temporary key, the user can provide permission for a
-set period of time (until the key expires), enabling the console to perform actions like stopping or rebooting the VM on
-their behalf. This maintains security while streamlining interactions with the console, as the server verifies each
-operation using the temporary key without requiring ongoing involvement from the user's wallet.
+individual request with their Ethereum or Solana wallet. This is crucial for improving the user experience, as
+constantly signing each operation would be cumbersome and inefficient. By generating a temporary key, the user can
+provide permission for a set period of time (until the key expires), enabling the console to perform actions like
+stopping or rebooting the VM on their behalf. This maintains security while streamlining interactions with the console,
+as the server verifies each operation using the temporary key without requiring ongoing involvement from the user's
+wallet.
 
 The generated public key is converted into a JSON structure with additional metadata:
-* `pubkey`: The public key information.
-* `alg`: The signing algorithm, ECDSA.
-* `domain`: The domain for which the key is valid.
-* `address`: The Ethereum address of the sender, binding the public key to this identity.
-* `expires`: The expiration time of the key.
 
-Example
+- **`pubkey`**: The public key information.
+- **`alg`**: The signing algorithm, ECDSA.
+- **`domain`**: The domain for which the key is valid.
+- **`address`**: The wallet address of the sender, binding the temporary key to this identity.
+- **`chain`**: Indicates the blockchain used for signing (`ETH` or `SOL`). Defaults to `ETH`.
+- **`expires`**: The expiration time of the key.
+
+Example:
+
 ```json
 {
   "pubkey": {
@@ -62,12 +65,13 @@ Example
   "alg": "ECDSA",
   "domain": "localhost",
   "address": "0x8Dd070629F107e7946dD68BDcb8ABE8475F47B0E",
+  "chain": "ETH",
   "expires": "2010-12-26T17:05:55Z"
 }
 ```
 
-This public key is signed using the Ethereum account to ensure its authenticity. The resulting signature is
-combined with the public key into a payload and sent as the `X-SignedPubKey` header.
+This public key is signed using either the Ethereum or Solana account, depending on the `chain` parameter. The resulting
+signature is combined with the public key into a payload and sent as the `X-SignedPubKey` header.
 
 ### 2. Sign Operation Payload
 
@@ -83,7 +87,7 @@ integrity can be verified through signing. Below are the fields included:
 - **`domain`**: (string) The domain associated with the request. This ensures the request is valid for the intended
   CRN. (e.g., `localhost`).
 
-Example
+Example:
 
 ```json
 {
@@ -97,55 +101,61 @@ Example
 It is sent serialized as a hex string.
 
 #### Signature
-This payload is serialized in JSON, signed, and sent in the `X-SignedOperation` header to ensure the integrity and authenticity
-of the request.
 
-* The operation payload (containing details such as time, method, path, and domain) is serialized and converted into a byte array.
-* The JWK (private key) is used to sign this operation payload, ensuring its integrity. This signature is then included in the X-SignedOperation header.
 
+- The operation payload (containing details such as time, method, path, and domain) is JSON serialized and converted into a
+  hex string.
+- The ephemeral key  (private key) is used to sign this operation payload, ensuring its integrity. This signature is then included
+  in the `X-SignedOperation` header.
+
+### 3. Include Authentication Headers
 
-###  3. Include authentication Headers
-These two headers are to be added to the HTTP Request:
+These two headers are to be added to the HTTP request:
 
-1. **`X-SignedPubKey` Header:**
-    - This header contains the public key payload and the signature of the public key generated by the Ethereum account.
+1. **`X-SignedPubKey` Header**:
+    - This header contains the public key payload and the signature of the public key generated by the Ethereum or
+      Solana account.
 
    Example:
+
    ```json
    {
-      "payload": "",
-      "signature": ""
+     "payload": "",
+     "signature": ""
    }
    ```
 
-2. **`X-SignedOperation` Header:**
+2. **`X-SignedOperation` Header**:
     - This header contains the operation payload and the signature of the operation payload generated using the private
       JWK.
 
    Example:
+
    ```json
    {
-      "payload": "",
-      "signature": ""
+     "payload": "",
+     "signature": ""
    }
    ```
 
 ### Expiration and Validation
 
 - The public key has an expiration date, ensuring that keys are not used indefinitely.
-- Both the public key and the operation signature are validated for authenticity and integrity at the server side.
+- Both the public key and the operation signature are validated for authenticity and integrity at the server side,
+  taking into account the specified blockchain (Ethereum or Solana).
 - Requests failing verification or expired keys are rejected with `401 Unauthorized` status, providing an error message
   indicating the reason.
 
-### WebSocket Authentication Protocol
+## WebSocket Authentication Protocol
 
 In the WebSocket variant of the authentication protocol, the client establishes a connection and authenticates through
-an initial message that includes their Ethereum-signed identity, ensuring secure communication.
+an initial message that includes their Ethereum or Solana-signed identity, ensuring secure communication.
+
+Due to web browsers not allowing custom HTTP headers in WebSocket connections, the two headers are sent in one JSON
+packet, under the `auth` key.
 
-Due to web browsers not allowing custom HTTP headers in WebSocket connections, 
-the two header are sent in one json packet, under the `auth` key.
+Example authentication packet:
 
-Example authentication packet 
 ```json
 {
     "auth": {
diff --git a/packaging/Makefile b/packaging/Makefile
index f22788c02..8ed7861ac 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0'
+	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo  target/bin/sevctl
diff --git a/pyproject.toml b/pyproject.toml
index 0fe050f00..5c4efc54a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -27,6 +27,8 @@ classifiers = [
   "Topic :: System :: Distributed Computing",
 ]
 dynamic = [ "version" ]
+
+# Upon adding or updating dependencies, update `packaging/Makefile` for the Debian package
 dependencies = [
   "aiodns==3.1",
   "aiohttp==3.9.5",
@@ -53,6 +55,7 @@ dependencies = [
   "schedule==1.2.1",
   "sentry-sdk==1.31",
   "setproctitle==1.3.3",
+  "solathon==1.0.2",
   "sqlalchemy[asyncio]>=2",
   "systemd-python==235",
 ]
diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index 6bc9a6c04..419662072 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -1,6 +1,6 @@
 """Functions for authentications
 
-See /doc/operator_auth.md for the explaination of how the operator authentication works.
+See /doc/operator_auth.md for the explanation of how the operator authentication works.
 
 Can be enabled on an endpoint using the @require_jwk_authentication decorator
 """
@@ -16,11 +16,14 @@
 import cryptography.exceptions
 import pydantic
 from aiohttp import web
+from aleph_message.models import Chain
 from eth_account import Account
 from eth_account.messages import encode_defunct
 from jwcrypto import jwk
 from jwcrypto.jwa import JWA
+from nacl.exceptions import BadSignatureError
 from pydantic import BaseModel, ValidationError, root_validator, validator
+from solathon.utils import verify_signature
 
 from aleph.vm.conf import settings
 
@@ -37,7 +40,7 @@ def is_token_still_valid(datestr: str):
     return expiry_datetime > current_datetime
 
 
-def verify_wallet_signature(signature, message, address):
+def verify_eth_wallet_signature(signature, message, address):
     """
     Verifies a signature issued by a wallet
     """
@@ -46,6 +49,21 @@ def verify_wallet_signature(signature, message, address):
     return computed_address.lower() == address.lower()
 
 
+def check_wallet_signature_or_raise(address, chain, payload, signature):
+    if chain == Chain.SOL:
+        try:
+            verify_signature(address, signature, payload.hex())
+        except BadSignatureError:
+            msg = "Invalid signature"
+            raise ValueError(msg)
+    elif chain == "ETH":
+        if not verify_eth_wallet_signature(signature, payload.hex(), address):
+            msg = "Invalid signature"
+            raise ValueError(msg)
+    else:
+        raise ValueError("Unsupported chain")
+
+
 class SignedPubKeyPayload(BaseModel):
     """This payload is signed by the wallet of the user to authorize an ephemeral key to act on his behalf."""
 
@@ -55,6 +73,12 @@ class SignedPubKeyPayload(BaseModel):
     # alg: Literal["ECDSA"]
     address: str
     expires: str
+    chain: Chain = Chain.ETH
+
+    def check_chain(self, v: Chain):
+        if v not in (Chain.ETH, Chain.SOL):
+            raise ValueError("Chain not supported")
+        return v
 
     @property
     def json_web_key(self) -> jwk.JWK:
@@ -89,12 +113,10 @@ def check_expiry(cls, values) -> dict[str, bytes]:
     @root_validator(pre=False, skip_on_failure=True)
     def check_signature(cls, values) -> dict[str, bytes]:
         """Check that the signature is valid"""
-        signature: bytes = values["signature"]
+        signature: list = values["signature"]
         payload: bytes = values["payload"]
         content = SignedPubKeyPayload.parse_raw(payload)
-        if not verify_wallet_signature(signature, payload.hex(), content.address):
-            msg = "Invalid signature"
-            raise ValueError(msg)
+        check_wallet_signature_or_raise(content.address, content.chain, payload, signature)
         return values
 
     @property
@@ -208,6 +230,7 @@ def verify_signed_operation(signed_operation: SignedOperation, signed_pubkey: Si
 async def authenticate_jwk(request: web.Request) -> str:
     """Authenticate a request using the X-SignedPubKey and X-SignedOperation headers."""
     signed_pubkey = get_signed_pubkey(request)
+
     signed_operation = get_signed_operation(request)
     if signed_operation.content.domain != settings.DOMAIN_NAME:
         logger.debug(f"Invalid domain '{signed_operation.content.domain}' != '{settings.DOMAIN_NAME}'")
@@ -236,6 +259,26 @@ async def authenticate_websocket_message(message) -> str:
 def require_jwk_authentication(
     handler: Callable[[web.Request, str], Coroutine[Any, Any, web.StreamResponse]]
 ) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
+    """A decorator to enforce JWK-based authentication for HTTP requests.
+
+    The decorator ensures that the incoming request includes valid authentication headers
+    (as per the VM owner authentication protocol) and provides the authenticated wallet address (`authenticated_sender`)
+    to the handler. The handler can then use this address to verify access to the requested resource.
+
+    Args:
+        handler (Callable[[web.Request, str], Coroutine[Any, Any, web.StreamResponse]]):
+            The request handler function that will receive the `authenticated_sender` (the authenticated wallet address)
+            as an additional argument.
+
+    Returns:
+        Callable[[web.Request], Awaitable[web.StreamResponse]]:
+            A wrapped handler that verifies the authentication and passes the wallet address to the handler.
+
+    Note:
+        Refer to the "Authentication protocol for VM owner" documentation for detailed information on the authentication
+        headers and validation process.
+    """
+
     @functools.wraps(handler)
     async def wrapper(request):
         try:
@@ -247,7 +290,7 @@ async def wrapper(request):
             logging.exception(e)
             raise
 
-        # authenticated_sender is the authenticted wallet address of the requester (as a string)
+        # authenticated_sender is the authenticate wallet address of the requester (as a string)
         response = await handler(request, authenticated_sender)
         return response
 
diff --git a/src/aleph/vm/utils/__init__.py b/src/aleph/vm/utils/__init__.py
index 15a57819a..d8eecad95 100644
--- a/src/aleph/vm/utils/__init__.py
+++ b/src/aleph/vm/utils/__init__.py
@@ -75,7 +75,7 @@ def to_json(o: Any) -> dict | str:
     elif hasattr(o, "dict"):  # Pydantic
         return o.dict()
     elif is_dataclass(o):
-        return dataclass_as_dict(o)
+        return dataclass_as_dict(o)  # type: ignore
     else:
         return str(o)
 
diff --git a/tests/supervisor/test_authentication.py b/tests/supervisor/test_authentication.py
index 77ba154d7..6fb3d0811 100644
--- a/tests/supervisor/test_authentication.py
+++ b/tests/supervisor/test_authentication.py
@@ -1,8 +1,10 @@
+import datetime
 import json
 from typing import Any
 
 import eth_account.messages
 import pytest
+import solathon
 from aiohttp import web
 from eth_account.datastructures import SignedMessage
 from jwcrypto import jwk, jws
@@ -238,6 +240,78 @@ async def view(request, authenticated_sender):
     assert "ok" == r
 
 
+async def generate_sol_signer_and_signed_headers_for_operation(
+    patch_datetime_now, operation_payload: dict
+) -> tuple[solathon.Keypair, dict]:
+    """Generate a temporary eth_account for testing and sign the operation with it"""
+
+    kp = solathon.Keypair()
+    key = jwk.JWK.generate(
+        kty="EC",
+        crv="P-256",
+        # key_ops=["verify"],
+    )
+
+    pubkey = {
+        "pubkey": json.loads(key.export_public()),
+        "alg": "ECDSA",
+        "domain": "localhost",
+        "address": str(kp.public_key),
+        "expires": (patch_datetime_now.FAKE_TIME + datetime.timedelta(days=1)).isoformat() + "Z",
+        "chain": "SOL",
+    }
+    pubkey_payload = json.dumps(pubkey).encode("utf-8").hex()
+    import nacl.signing
+
+    signed_message: nacl.signing.SignedMessage = kp.sign(pubkey_payload)
+    pubkey_signature = to_0x_hex(signed_message.signature)
+    pubkey_signature_header = json.dumps(
+        {
+            "payload": pubkey_payload,
+            "signature": pubkey_signature,
+        }
+    )
+    payload_as_bytes = json.dumps(operation_payload).encode("utf-8")
+    from jwcrypto.jwa import JWA
+
+    payload_signature = JWA.signing_alg("ES256").sign(key, payload_as_bytes)
+    headers = {
+        "X-SignedPubKey": pubkey_signature_header,
+        "X-SignedOperation": json.dumps(
+            {
+                "payload": payload_as_bytes.hex(),
+                "signature": payload_signature.hex(),
+            }
+        ),
+    }
+    return kp, headers
+
+
+@pytest.mark.asyncio
+async def test_require_jwk_authentication_good_key_solana(aiohttp_client, patch_datetime_now):
+    """An HTTP request to a view decorated by `@require_jwk_authentication`
+    auth correctly a temporary key signed by a wallet and an operation signed by that key"""
+
+    app = web.Application()
+    payload = {"time": "2010-12-25T17:05:55Z", "method": "GET", "path": "/", "domain": "localhost"}
+
+    signer_account, headers = await generate_sol_signer_and_signed_headers_for_operation(patch_datetime_now, payload)
+
+    @require_jwk_authentication
+    async def view(request, authenticated_sender):
+        assert authenticated_sender == str(signer_account.public_key)
+        return web.Response(text="ok")
+
+    app.router.add_get("", view)
+    client = await aiohttp_client(app)
+
+    resp = await client.get("/", headers=headers)
+    assert resp.status == 200, await resp.text()
+
+    r = await resp.text()
+    assert "ok" == r
+
+
 @pytest.fixture
 def valid_jwk_headers(mocker):
     mocker.patch("aleph.vm.orchestrator.views.authentication.is_token_still_valid", lambda timestamp: True)

From 0aeb39ef71fc386ec2d867b595a3e9e8017e2daa Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 13 Sep 2024 12:08:37 +0200
Subject: [PATCH 869/990] mod Simplify
 test_operator_confidential_initialize_already_running

---
 tests/supervisor/views/test_operator.py | 45 +++++++++++++------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/tests/supervisor/views/test_operator.py b/tests/supervisor/views/test_operator.py
index 150a33002..911eaeb48 100644
--- a/tests/supervisor/views/test_operator.py
+++ b/tests/supervisor/views/test_operator.py
@@ -61,7 +61,7 @@ def __init__(self):
 
 
 @pytest.mark.asyncio
-async def test_operator_confidential_initialize_already_running(aiohttp_client):
+async def test_operator_confidential_initialize_already_running(aiohttp_client, mocker):
     """Test that the confidential initialize endpoint rejects if the VM is already running. Auth needed"""
 
     settings.ENABLE_QEMU_SUPPORT = True
@@ -71,30 +71,31 @@ async def test_operator_confidential_initialize_already_running(aiohttp_client):
     vm_hash = ItemHash(settings.FAKE_INSTANCE_ID)
     instance_message = await get_message(ref=vm_hash)
 
-    class FakeExecution:
-        message = instance_message.content
-        is_running: bool = True
-        is_confidential: bool = False
-
-    class FakeVmPool:
-        executions: dict[ItemHash, FakeExecution] = {}
-
-        def __init__(self):
-            self.executions[vm_hash] = FakeExecution()
+    fake_vm_pool = mocker.Mock(
+        executions={
+            vm_hash: mocker.Mock(
+                vm_hash=vm_hash,
+                message=instance_message.content,
+                is_confidential=False,
+                is_running=True,
+            ),
+        },
+    )
 
-    with mock.patch(
+    # Disable auth
+    mocker.patch(
         "aleph.vm.orchestrator.views.authentication.authenticate_jwk",
         return_value=instance_message.sender,
-    ):
-        app = setup_webapp()
-        app["vm_pool"] = FakeVmPool()
-        client = await aiohttp_client(app)
-        response = await client.post(
-            f"/control/machine/{vm_hash}/confidential/initialize",
-            json={"persistent_vms": []},
-        )
-        assert response.status == 403
-        assert await response.text() == f"VM with ref {vm_hash} already running"
+    )
+    app = setup_webapp()
+    app["vm_pool"] = fake_vm_pool
+    client = await aiohttp_client(app)
+    response = await client.post(
+        f"/control/machine/{vm_hash}/confidential/initialize",
+        json={"persistent_vms": []},
+    )
+    assert response.status == 403
+    assert await response.text() == f"VM with ref {vm_hash} already running"
 
 
 @pytest.mark.asyncio

From 90bae310c7d6b255572d7fe5df65d15da8d0b7c0 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 1 Oct 2024 15:02:26 +0200
Subject: [PATCH 870/990] Endpoint /confidential/initialize return json error
 now and proper https status

return json errors for missing field
---
 src/aleph/vm/orchestrator/views/operator.py | 22 +++++++--
 tests/supervisor/views/test_operator.py     | 52 +++++++++++++++++++--
 2 files changed, 66 insertions(+), 8 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 48fe7d036..af0e98f45 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -1,6 +1,7 @@
 import json
 import logging
 from datetime import timedelta
+from http import HTTPStatus
 
 import aiohttp.web_exceptions
 import pydantic
@@ -187,10 +188,15 @@ async def operate_confidential_initialize(request: web.Request, authenticated_se
         return web.Response(status=403, body="Unauthorized sender")
 
     if execution.is_running:
-        return web.Response(status=403, body=f"VM with ref {vm_hash} already running")
-
+        return web.json_response(
+            {"code": "vm_running", "description": "Operation not allowed, instance already running"},
+            status=HTTPStatus.BAD_REQUEST,
+        )
     if not execution.is_confidential:
-        return web.Response(status=403, body=f"Operation not allowed for VM {vm_hash} because it isn't confidential")
+        return web.json_response(
+            {"code": "not_confidential", "description": "Instance is not a confidential instance"},
+            status=HTTPStatus.BAD_REQUEST,
+        )
 
     post = await request.post()
 
@@ -199,14 +205,20 @@ async def operate_confidential_initialize(request: web.Request, authenticated_se
 
     session_file_content = post.get("session")
     if not session_file_content:
-        return web.Response(status=403, body=f"Session file required for VM with ref {vm_hash}")
+        return web.json_response(
+            {"code": "field_missing", "description": "Session field is missing"},
+            status=HTTPStatus.BAD_REQUEST,
+        )
 
     session_file_path = vm_session_path / "vm_session.b64"
     session_file_path.write_bytes(session_file_content.file.read())
 
     godh_file_content = post.get("godh")
     if not godh_file_content:
-        return web.Response(status=403, body=f"GODH file required for VM with ref {vm_hash}")
+        return web.json_response(
+            {"code": "field_missing", "description": "godh field is missing. Please provide a GODH file"},
+            status=HTTPStatus.BAD_REQUEST,
+        )
 
     godh_file_path = vm_session_path / "vm_godh.b64"
     godh_file_path.write_bytes(godh_file_content.file.read())
diff --git a/tests/supervisor/views/test_operator.py b/tests/supervisor/views/test_operator.py
index 911eaeb48..b8e370de7 100644
--- a/tests/supervisor/views/test_operator.py
+++ b/tests/supervisor/views/test_operator.py
@@ -89,13 +89,59 @@ async def test_operator_confidential_initialize_already_running(aiohttp_client,
     )
     app = setup_webapp()
     app["vm_pool"] = fake_vm_pool
-    client = await aiohttp_client(app)
+    client: TestClient = await aiohttp_client(app)
     response = await client.post(
         f"/control/machine/{vm_hash}/confidential/initialize",
         json={"persistent_vms": []},
     )
-    assert response.status == 403
-    assert await response.text() == f"VM with ref {vm_hash} already running"
+    assert response.status == 400
+    assert response.content_type == "application/json"
+    assert await response.json() == {
+        "code": "vm_running",
+        "description": "Operation not allowed, instance already running",
+    }
+
+
+@pytest.mark.asyncio
+async def test_operator_confidential_initialize_not_confidential(aiohttp_client, mocker):
+    """Test that the confidential initialize endpoint rejects if the VM is not confidential"""
+
+    settings.ENABLE_QEMU_SUPPORT = True
+    settings.ENABLE_CONFIDENTIAL_COMPUTING = True
+    settings.setup()
+
+    vm_hash = ItemHash(settings.FAKE_INSTANCE_ID)
+    instance_message = await get_message(ref=vm_hash)
+
+    fake_vm_pool = mocker.Mock(
+        executions={
+            vm_hash: mocker.Mock(
+                vm_hash=vm_hash,
+                message=instance_message.content,
+                is_confidential=False,
+                is_running=False,
+            ),
+        },
+    )
+
+    # Disable auth
+    mocker.patch(
+        "aleph.vm.orchestrator.views.authentication.authenticate_jwk",
+        return_value=instance_message.sender,
+    )
+    app = setup_webapp()
+    app["vm_pool"] = fake_vm_pool
+    client: TestClient = await aiohttp_client(app)
+    response = await client.post(
+        f"/control/machine/{vm_hash}/confidential/initialize",
+        json={"persistent_vms": []},
+    )
+    assert response.status == 400
+    assert response.content_type == "application/json"
+    assert await response.json() == {
+        "code": "not_confidential",
+        "description": "Instance is not a confidential instance",
+    }
 
 
 @pytest.mark.asyncio

From 49d91d28d9e71e3b723f85f6dd4c3107fdd462dd Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 9 Oct 2024 12:43:22 +0200
Subject: [PATCH 871/990] Display message when CoCo disk creation abort (#709)

Fix ALEPH-211
---
 .../example_confidential_image/build_debian_image.sh   | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/examples/example_confidential_image/build_debian_image.sh b/examples/example_confidential_image/build_debian_image.sh
index 3be89a77e..c7a928dff 100644
--- a/examples/example_confidential_image/build_debian_image.sh
+++ b/examples/example_confidential_image/build_debian_image.sh
@@ -18,7 +18,6 @@ cleanup() {
     return
   fi
   CLEANUP_DONE=true
-  echo "Cleaning up..."
   if mountpoint -q "${MOUNT_POINT}"; then
     sudo umount --recursive "${MOUNT_POINT}" || echo "Failed to unmount ${MOUNT_POINT}"
   fi
@@ -43,6 +42,15 @@ cleanup() {
 # - TERM (SIGTERM): Signal 15, sent by the kill command to request the process to terminate gracefully.
 trap cleanup EXIT HUP INT QUIT PIPE TERM
 
+error_handler() {
+	echo ""
+	echo "An error occured while building the image and the process was not completed properly."
+	echo "Please check the log, fix any error if required and restart the script."
+	echo "For more help see https://docs.aleph.im/computing/confidential/encrypted-disk/"
+}
+
+trap error_handler ERR
+
 usage() {
   cat <&2
 Usage:

From 2dfb42b5dcbe8b7a07ab6a979693b112f4fbf353 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 8 Oct 2024 14:18:25 +0200
Subject: [PATCH 872/990] Fix: Error fetching message from vm-connector for
 start_watch_for_messages_task

Jira ticket : ALEPH-111

Problem:
When aleph-vm was starting if the vm-connector couldn't connect to the
pyaleph API , the startup crashed and systemd restarted it in a loop.

Solution:
Stop registering the sample program at startup, so the  pyaleph api
don't need to be reached at startup. This code was left as a demo but
it didn't really do anything  at the moment.

We have just commented the code if we want to reuse it in the future.

Note:
A more complete and long analysis of the problem, along with propsed
solution is on the JIRA ticket

https://aleph-im.atlassian.net/browse/ALEPH-111?focusedCommentId=10008
---
 src/aleph/vm/orchestrator/tasks.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 99c814a82..c7062d931 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -123,12 +123,12 @@ async def start_watch_for_messages_task(app: web.Application):
 
     # Register an hardcoded initial program
     # TODO: Register all programs with subscriptions
-    sample_message, _ = await load_updated_message(
-        ref=ItemHash("cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e")
-    )
-    if isinstance(sample_message, ProgramMessage):
-        assert sample_message.content.on.message, sample_message
-        reactor.register(sample_message)
+    # sample_message, _ = await load_updated_message(
+    #     ref=ItemHash("cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e")
+    # )
+    # if isinstance(sample_message, ProgramMessage):
+    #     assert sample_message.content.on.message, sample_message
+    #     reactor.register(sample_message)
 
     app["pubsub"] = pubsub
     app["reactor"] = reactor

From 123a4d82ad35bf9c060dc49cf23ad1020487540c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 17 Oct 2024 17:44:12 +0200
Subject: [PATCH 873/990] =?UTF-8?q?FirecrackerVM=20drive=20not=20working?=
 =?UTF-8?q?=20if=20/var/lib=20and=20/var/cache=20on=20two=20sep=E2=80=A6?=
 =?UTF-8?q?=20(#711)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

FirecrackerVM drive not working if /var/lib and /var/cache on two separate partions

Jira Ticket ALEPH-238

Similar issue to https://github.com/aleph-im/aleph-vm/pull/682
That was merged inside https://github.com/aleph-im/aleph-vm/pull/686

We have fixed a variation of this alread but this one triggered for additional volumes only

Explanation:
The prepare step for jailer is failing because it attempt create a hardlink to a file between the CACHE and EXECUTION dir which is not allowed between separate partition

Solution: Make a hardlink
Similiarly to the previous resolution, we cannot make a symlink as it
is not accessible inside the jailer enclave
---
 src/aleph/vm/hypervisors/firecracker/microvm.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 3cf3e3087..7a8fe787e 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -365,7 +365,7 @@ def compute_device_name(index: int) -> str:
     def enable_drive(self, drive_path: Path, read_only: bool = True) -> Drive:
         """Make a volume available to the VM.
 
-        Creates a symlink to the volume file if jailer is in use.
+        Creates a hardlink or a copy to the volume file if jailer is in use.
         """
         index = len(self.drives)
         device_name = self.compute_device_name(index)
@@ -376,6 +376,11 @@ def enable_drive(self, drive_path: Path, read_only: bool = True) -> Drive:
 
             try:
                 Path(f"{self.jailer_path}/{jailer_path_on_host}").hardlink_to(drive_path)
+            except OSError as err:
+                if err.errno == errno.EXDEV:
+                    # Invalid cross-device link: cannot make hard link between partition.
+                    # In this case, copy the file instead:
+                    shutil.copyfile(drive_path, f"{self.jailer_path}/{jailer_path_on_host}")
             except FileExistsError:
                 logger.debug(f"File {jailer_path_on_host} already exists")
             drive_path = Path(jailer_path_on_host)

From 7a2d6cceff8adbf696a76dea3027977b48163b75 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 22 Oct 2024 14:31:06 +0200
Subject: [PATCH 874/990] Problem: Sentry reporting didn't have release
 information

Solution : Add the release number to sentry init according
to the Sentry documentation
https://docs.sentry.io/platforms/python/configuration/releases/
---
 src/aleph/vm/orchestrator/cli.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index e0d11298a..ddcf8910d 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -20,7 +20,7 @@
 from aleph.vm.conf import ALLOW_DEVELOPER_SSH_KEYS, make_db_url, settings
 from aleph.vm.models import VmExecution
 from aleph.vm.pool import VmPool
-from aleph.vm.version import get_version_from_apt, get_version_from_git
+from aleph.vm.version import __version__, get_version_from_apt, get_version_from_git
 
 from . import metrics, supervisor
 from .pubsub import PubSub
@@ -325,6 +325,7 @@ def main():
                 # of transactions for performance monitoring.
                 # We recommend adjusting this value in production.
                 traces_sample_rate=1.0,
+                release=__version__,
             )
             sentry_sdk.set_context(
                 "version",

From 5a7a0eccb91894693b23950c2290ddb204c39cb5 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 29 Oct 2024 12:12:46 +0100
Subject: [PATCH 875/990] Problem: sevctl installation broken on Ubuntu

sevctl 0.4.3 was refusing to compile on Ubuntu 22.04 and 24.02
because of some deps. This happened in the Github CI

This prevented the build of package and the testing inside the CI

Solution: Update sevctl to 0.6.0

Error logs
        Compiling structopt-derive v0.4.18
    error[E0658]: `c".."` literals are experimental
      --> /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/kvm-ioctls-0.19.0/src/ioctls/system.rs:99:24
       |
    99 |         let kvm_path = c"/dev/kvm";
       |                        ^^^^^^^^^^^
       |
       = note: see issue #105723  for more information

    error[E0658]: `c".."` literals are experimental
       --> /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/kvm-ioctls-0.19.0/src/ioctls/system.rs:744:24
        |
    744 |         let kvm_path = c"/dev/kvm";
        |                        ^^^^^^^^^^^
        |
        = note: see issue #105723  for more information

    For more information about this error, try `rustc --explain E0658`.
    The following warnings were emitted during compilation:

    warning: kvm-ioctls@0.19.0: cargo:rustc-check-cfg requires -Zcheck-cfg flag

    error: could not compile `kvm-ioctls` (lib) due to 2 previous errors
    warning: build failed, waiting for other jobs to finish...
    error: failed to compile `sevctl v0.4.3 (https://github.com/virtee/sevctl.git?rev=c41c9172be013d6f10b9e1d7286fcb021805d5a5#c41c9172)`, intermediate artifacts can be found at `/tmp/cargo-installXhERbT`.
    To reuse those artifacts with a future compilation, set the environment variable `CARGO_TARGET_DIR` to that path.
    make: *** [Makefile:51: target/bin/sevctl] Error 101
    make: *** [Makefile:89: all-podman-ubuntu-2204] Error 2
---
 packaging/Makefile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 8ed7861ac..468ed7225 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -47,8 +47,7 @@ download-ipfs-kubo: target-dir build-dir
 	curl -fsSL https://github.com/ipfs/kubo/releases/download/v0.23.0/kubo_v0.23.0_linux-amd64.tar.gz | tar -xz --directory ./target/kubo
 
 target/bin/sevctl:
-	# Release 0.4.3 matches revision c41c9172be013d6f10b9e1d7286fcb021805d5a5
-	cargo install --git https://github.com/virtee/sevctl.git --rev c41c9172be013d6f10b9e1d7286fcb021805d5a5 --target x86_64-unknown-linux-gnu --root ./target
+	cargo install --git https://github.com/virtee/sevctl.git --rev v0.6.0 --target x86_64-unknown-linux-gnu --root ./target
 	./target/bin/sevctl -V
 
 version:

From 7741ac8ae3256aa45b429a0809f74bbb5220152a Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 29 Oct 2024 12:23:22 +0100
Subject: [PATCH 876/990] remove progess bar for pip3 in CI don't make sense in
 log

---
 packaging/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 468ed7225..0d1c4dcb9 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2'
+	pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo  target/bin/sevctl

From 089ccef903648db0479b1203c9f6c13d0eeea62a Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 31 Oct 2024 14:48:19 +0100
Subject: [PATCH 877/990] Make vm_id assignment more robust (#714)

Remove the counter way to assign a vm_id as it didn't work reliably

Jira ticket: ALEPH-272

That method was broken when persitent instances were loaded at start up.
Since the "new" feature that allow persistent instance across aleph-vm reboot
if one was started then aleph-vm was stopped and restarted the counter method
could reassign the ip and break the existing vm's.

Secundary reason was that the feature wasn't working properly with the default
settings, as `2**available_bits` returned 1. So that code path was only used if
the node owner tweaked some undocumented settings making it hard to identify and
debug in prod nodes.
---
 src/aleph/vm/pool.py | 35 ++++++++++-------------------------
 1 file changed, 10 insertions(+), 25 deletions(-)

diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 3ecf500eb..025bfe45c 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -28,15 +28,13 @@
 
 
 class VmPool:
-    """Pool of VMs already started and used to decrease response time.
+    """Pool of existing VMs
+
+    For function VM we keep the VM a while after they  have run, so we can reuse them  and thus decrease response time.
     After running, a VM is saved for future reuse from the same function during a
     configurable duration.
-
-    The counter is used by the VMs to set their tap interface name and the corresponding
-    IPv4 subnet.
     """
 
-    counter: int  # Used to provide distinct ids to network interfaces
     executions: dict[ItemHash, VmExecution]
     message_cache: dict[str, ExecutableMessage]
     network: Network | None
@@ -45,7 +43,6 @@ class VmPool:
     creation_lock: asyncio.Lock
 
     def __init__(self, loop: asyncio.AbstractEventLoop):
-        self.counter = settings.START_ID_INDEX
         self.executions = {}
         self.message_cache = {}
 
@@ -150,25 +147,13 @@ def get_unique_vm_id(self) -> int:
         This identifier is used to name the network interface and in the IPv4 range
         dedicated to the VM.
         """
-        _, network_range = settings.IPV4_ADDRESS_POOL.split("/")
-        available_bits = int(network_range) - settings.IPV4_NETWORK_PREFIX_LENGTH
-        self.counter += 1
-        if self.counter < 2**available_bits:
-            # In common cases, use the counter itself as the vm_id. This makes it
-            # easier to debug.
-            return self.counter
-        else:
-            # The value of the counter is too high and some functions such as the
-            # IPv4 range dedicated to the VM do not support such high values.
-            #
-            # We therefore recycle vm_id values from executions that are not running
-            # anymore.
-            currently_used_vm_ids = {execution.vm_id for execution in self.executions.values()}
-            for i in range(settings.START_ID_INDEX, 255**2):
-                if i not in currently_used_vm_ids:
-                    return i
-            msg = "No available value for vm_id."
-            raise ValueError(msg)
+        # Take the first id that is not already taken
+        currently_used_vm_ids = {execution.vm_id for execution in self.executions.values()}
+        for i in range(settings.START_ID_INDEX, 255**2):
+            if i not in currently_used_vm_ids:
+                return i
+        msg = "No available value for vm_id."
+        raise ValueError(msg)
 
     def get_running_vm(self, vm_hash: ItemHash) -> VmExecution | None:
         """Return a running VM or None. Disables the VM expiration task."""

From 5acbdef9bf0e8937a2ebe09d65dd8fda8fc77e17 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Mon, 4 Nov 2024 17:00:30 +0100
Subject: [PATCH 878/990] Implement new EVM chains (#717)

* Feature: Implement new EVM chains.

* FIX: Update Makefile with new dependency.

* Fix: Updated to proper released package version of aleph_message dependency.

---------

Co-authored-by: Andres D. Molins 
---
 packaging/Makefile | 4 ++--
 pyproject.toml     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 0d1c4dcb9..cc217ce3f 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,7 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2'
+	pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.5.0' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo  target/bin/sevctl
@@ -144,6 +144,6 @@ repository-noble:
 	cd ./repositories/noble && reprepro -Vb . includedeb noble ../../target/aleph-vm.ubuntu-24.04.deb && cd ..
 
 repositories: repository-bookworm repository-jammy repository-noble
- 
+
 all-podman: all-podman-debian-12 all-podman-ubuntu-2204 all-podman-ubuntu-2404 repositories
 
diff --git a/pyproject.toml b/pyproject.toml
index 5c4efc54a..50c314c2f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -36,7 +36,7 @@ dependencies = [
   "aioredis==1.3.1",
   "aiosqlite==0.19",
   "alembic==1.13.1",
-  "aleph-message==0.4.9",
+  "aleph-message==0.5",
   "aleph-superfluid~=0.2.1",
   "dbus-python==1.3.2",
   "eth-account~=0.10",

From b113406a2e79ac04c00c0fbc1556ac590457a9cb Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 5 Nov 2024 15:30:46 +0100
Subject: [PATCH 879/990] Feature: allow IPv6 DNS (#455)

* Feature: allow IPv6 DNS

Problem IPv6 DNS were automatically filtered when detected from resolvectl

Solution: Nameservers are now split into ipv4 and ipv6 and can be passed to the VM accordingly
At the moment we pass them if the ipv6 parameter is present on the tap interface but we need a
more robust detection method


* Display proper env conf
---
 src/aleph/vm/conf.py                          | 36 +++++++++++--------
 .../vm/controllers/firecracker/instance.py    |  7 +++-
 .../supervisor/test_resolvectl_dns_servers.py |  8 +----
 3 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index a1737b2b6..b68ff9e88 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -74,17 +74,6 @@ def resolvectl_dns_servers(interface: str) -> Iterable[str]:
         yield server.strip()
 
 
-def resolvectl_dns_servers_ipv4(interface: str) -> Iterable[str]:
-    """
-    Use resolvectl to list available IPv4 DNS servers.
-    VMs only support IPv4 networking for now, we must exclude IPv6 DNS from their config.
-    """
-    for server in resolvectl_dns_servers(interface):
-        ip_addr = ipaddress.ip_address(server)
-        if isinstance(ip_addr, ipaddress.IPv4Address):
-            yield server
-
-
 def get_default_interface() -> str | None:
     """Returns the default network interface"""
     with open("/proc/net/route") as f:
@@ -102,7 +91,7 @@ def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> list[st
         # Use a try-except approach since resolvectl can be present but disabled and raise the following
         # "Failed to get global data: Unit dbus-org.freedesktop.resolve1.service not found."
         try:
-            return list(resolvectl_dns_servers_ipv4(interface=network_interface))
+            return list(resolvectl_dns_servers(interface=network_interface))
         except (FileNotFoundError, CalledProcessError) as error:
             if Path("/etc/resolv.conf").exists():
                 return list(etc_resolv_conf_dns_servers())
@@ -114,7 +103,7 @@ def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> list[st
         return list(etc_resolv_conf_dns_servers())
 
     elif dns_resolver == DnsResolver.resolvectl:
-        return list(resolvectl_dns_servers_ipv4(interface=network_interface))
+        return list(resolvectl_dns_servers(interface=network_interface))
 
     else:
         msg = "No DNS resolve defined, this should never happen."
@@ -180,8 +169,13 @@ class Settings(BaseSettings):
         description="Use the Neighbor Discovery Protocol Proxy to respond to Router Solicitation for instances on IPv6",
     )
 
-    DNS_RESOLUTION: DnsResolver | None = DnsResolver.detect
+    DNS_RESOLUTION: DnsResolver | None = Field(
+        default=DnsResolver.detect,
+        description="Method used to resolve the dns server if DNS_NAMESERVERS is not present.",
+    )
     DNS_NAMESERVERS: list[str] | None = None
+    DNS_NAMESERVERS_IPV4: list[str] | None
+    DNS_NAMESERVERS_IPV6: list[str] | None
 
     FIRECRACKER_PATH = Path("/opt/firecracker/firecracker")
     JAILER_PATH = Path("/opt/firecracker/jailer")
@@ -439,6 +433,18 @@ def setup(self):
                 network_interface=self.NETWORK_INTERFACE,
             )
 
+        if not self.DNS_NAMESERVERS_IPV4:
+            self.DNS_NAMESERVERS_IPV4 = []
+        if not self.DNS_NAMESERVERS_IPV6:
+            self.DNS_NAMESERVERS_IPV6 = []
+        if self.DNS_NAMESERVERS:
+            for server in self.DNS_NAMESERVERS:
+                ip_addr = ipaddress.ip_address(server)
+                if isinstance(ip_addr, ipaddress.IPv4Address):
+                    self.DNS_NAMESERVERS_IPV4.append(server)
+                if isinstance(ip_addr, ipaddress.IPv6Address):
+                    self.DNS_NAMESERVERS_IPV6.append(server)
+
         if not settings.ENABLE_QEMU_SUPPORT:
             # If QEmu is not supported, ignore the setting and use Firecracker by default
             settings.INSTANCE_DEFAULT_HYPERVISOR = HypervisorType.firecracker
@@ -456,7 +462,7 @@ def display(self) -> str:
             else:
                 attributes[attr] = getattr(self, attr)
 
-        return "\n".join(f"{attribute:<27} = {value}" for attribute, value in attributes.items())
+        return "\n".join(f"{self.Config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items())
 
     def __init__(
         self,
diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py
index f8c33b075..da423ef73 100644
--- a/src/aleph/vm/controllers/firecracker/instance.py
+++ b/src/aleph/vm/controllers/firecracker/instance.py
@@ -198,6 +198,11 @@ def _create_network_file(self) -> bytes:
         ipv6 = self.get_ipv6()
         ipv6_gateway = self.get_ipv6_gateway()
 
+        nameservers_ip = []
+        if ip:
+            nameservers_ip = settings.DNS_NAMESERVERS_IPV4
+        if ipv6:
+            nameservers_ip += settings.DNS_NAMESERVERS_IPV6
         network = {
             "ethernets": {
                 "eth0": {
@@ -207,7 +212,7 @@ def _create_network_file(self) -> bytes:
                     "gateway4": route,
                     "gateway6": ipv6_gateway,
                     "nameservers": {
-                        "addresses": settings.DNS_NAMESERVERS,
+                        "addresses": nameservers_ip,
                     },
                 },
             },
diff --git a/tests/supervisor/test_resolvectl_dns_servers.py b/tests/supervisor/test_resolvectl_dns_servers.py
index 0daaf03c4..0af9b6fb8 100644
--- a/tests/supervisor/test_resolvectl_dns_servers.py
+++ b/tests/supervisor/test_resolvectl_dns_servers.py
@@ -2,7 +2,7 @@
 import os
 from unittest import mock
 
-from aleph.vm.conf import resolvectl_dns_servers, resolvectl_dns_servers_ipv4
+from aleph.vm.conf import resolvectl_dns_servers
 
 os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False"
 
@@ -17,9 +17,6 @@ def test_resolvectl():
         dns_servers = set(resolvectl_dns_servers("eth0"))
         assert dns_servers == servers
 
-        dns_servers_ipv4 = set(resolvectl_dns_servers_ipv4("eth0"))
-        assert dns_servers_ipv4 == servers
-
 
 def test_resolvectl_ipv6():
     with mock.patch(
@@ -31,6 +28,3 @@ def test_resolvectl_ipv6():
 
         dns_servers = set(resolvectl_dns_servers("eth0"))
         assert dns_servers == ipv4_servers | ipv6_servers
-
-        dns_servers_ipv4 = set(resolvectl_dns_servers_ipv4("eth0"))
-        assert dns_servers_ipv4 == ipv4_servers

From 662c0c00b50b1feef83a849038f0c6abd922c773 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 6 Nov 2024 14:54:42 +0100
Subject: [PATCH 880/990] Problem: IGNORE_TRACEBACK_FROM_DIAGNOSTICS broken
 (#713)

Symptom:
The CustomError from the diagnostics VM was printed even if if IGNORE_TRACEBACK_FROM_DIAGNOSTICS was set to True (the default)

Analysis:
This was caused by the refactoring of the fastapi_example/main.py file
done in fe9235ac658915eea20d5371ae45cedabe1f7b17
Which changed the output used to detect the error to catch

Solution:
Fix detection string
---
 src/aleph/vm/orchestrator/run.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index a2a2a824f..9c2a8b298 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -150,9 +150,11 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques
 
             # The Diagnostics VM checks for the proper handling of exceptions.
             # This fills the logs with noisy stack traces, so we ignore this specific error.
-            ignored_error = 'raise CustomError("Whoops")'
+            ignored_errors = ['raise CustomError("Whoops")', "main.CustomError: Whoops"]
 
-            if settings.IGNORE_TRACEBACK_FROM_DIAGNOSTICS and ignored_error in result["traceback"]:
+            if settings.IGNORE_TRACEBACK_FROM_DIAGNOSTICS and any(
+                ignored_error in result["traceback"] for ignored_error in ignored_errors
+            ):
                 logger.debug('Ignored traceback from CustomError("Whoops")')
             else:
                 logger.warning(result["traceback"])

From 7461a4958c123bc8a8890d18b39374998d55d9f9 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 8 Nov 2024 15:17:09 +0100
Subject: [PATCH 881/990] Problem: error Too many open files (#720)

Jira ticket: ALEPH-298

some CRN failed on any action with error OSError: [Errno 24] Too many open files:

Solution:
Properly close stream to journald when the VM is stopped
---
 .../vm/hypervisors/firecracker/microvm.py     | 41 +++++++++++++------
 src/aleph/vm/hypervisors/qemu/qemuvm.py       | 17 +++++---
 2 files changed, 40 insertions(+), 18 deletions(-)

diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py
index 7a8fe787e..d357fb6e0 100644
--- a/src/aleph/vm/hypervisors/firecracker/microvm.py
+++ b/src/aleph/vm/hypervisors/firecracker/microvm.py
@@ -13,7 +13,7 @@
 from pathlib import Path
 from pwd import getpwnam
 from tempfile import NamedTemporaryFile
-from typing import Any
+from typing import Any, BinaryIO
 
 import msgpack
 from aleph_message.models import ItemHash
@@ -93,6 +93,8 @@ class MicroVM:
     mounted_rootfs: Path | None = None
     _unix_socket: Server | None = None
     enable_log: bool
+    journal_stdout: BinaryIO | int | None = None
+    journal_stderr: BinaryIO | int | None = None
 
     def __repr__(self):
         return f""
@@ -219,19 +221,19 @@ async def start_firecracker(self, config_path: Path) -> asyncio.subprocess.Proce
             str(config_path),
         )
         if self.enable_log:
-            journal_stdout = journal.stream(self._journal_stdout_name)
-            journal_stderr = journal.stream(self._journal_stderr_name)
+            self.journal_stdout = journal.stream(self._journal_stdout_name)
+            self.journal_stderr = journal.stream(self._journal_stderr_name)
         else:
-            journal_stdout = asyncio.subprocess.DEVNULL
-            journal_stderr = asyncio.subprocess.DEVNULL
+            self.journal_stdout = asyncio.subprocess.DEVNULL
+            self.journal_stderr = asyncio.subprocess.DEVNULL
 
         logger.debug(" ".join(options))
 
         self.proc = await asyncio.create_subprocess_exec(
             *options,
             stdin=asyncio.subprocess.PIPE,
-            stdout=journal_stdout,
-            stderr=journal_stderr,
+            stdout=self.journal_stdout,
+            stderr=self.journal_stderr,
         )
         return self.proc
 
@@ -252,11 +254,11 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces
 
         self.config_file_path = config_path
         if self.enable_log:
-            journal_stdout = journal.stream(self._journal_stdout_name)
-            journal_stderr = journal.stream(self._journal_stderr_name)
+            self.journal_stdout = journal.stream(self._journal_stdout_name)
+            self.journal_stderr = journal.stream(self._journal_stderr_name)
         else:
-            journal_stdout = asyncio.subprocess.DEVNULL
-            journal_stderr = asyncio.subprocess.DEVNULL
+            self.journal_stdout = asyncio.subprocess.DEVNULL
+            self.journal_stderr = asyncio.subprocess.DEVNULL
 
         options = (
             str(self.jailer_bin_path),
@@ -280,8 +282,8 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces
         self.proc = await asyncio.create_subprocess_exec(
             *options,
             stdin=asyncio.subprocess.PIPE,
-            stdout=journal_stdout,
-            stderr=journal_stderr,
+            stdout=self.journal_stdout,
+            stderr=self.journal_stderr,
         )
         return self.proc
 
@@ -480,6 +482,19 @@ async def teardown(self):
         if self.stderr_task:
             self.stderr_task.cancel()
 
+        if (
+            self.journal_stdout
+            and self.journal_stdout != asyncio.subprocess.DEVNULL
+            and hasattr(self.journal_stdout, "close")
+        ):
+            self.journal_stdout.close()
+        if (
+            self.journal_stderr
+            and self.journal_stderr != asyncio.subprocess.DEVNULL
+            and hasattr(self.journal_stderr, "close")
+        ):
+            self.journal_stderr.close()
+
         # Clean mounted block devices
         if self.mounted_rootfs:
             logger.debug("Waiting for one second for the VM to shutdown")
diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index 1d707c2a5..5949fbdc4 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -2,7 +2,7 @@
 from asyncio.subprocess import Process
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TextIO
+from typing import BinaryIO, TextIO
 
 import qmp
 from systemd import journal
@@ -28,6 +28,8 @@ class QemuVM:
     interface_name: str
     qemu_process: Process | None = None
     host_volumes: list[HostVolume]
+    journal_stdout: TextIO | None
+    journal_stderr: TextIO | None
 
     def __repr__(self) -> str:
         if self.qemu_process:
@@ -72,8 +74,8 @@ async def start(
         #  qemu-system-x86_64 -enable-kvm -m 2048 -net nic,model=virtio
         # -net tap,ifname=tap0,script=no,downscript=no -drive file=alpine.qcow2,media=disk,if=virtio -nographic
 
-        journal_stdout: TextIO = journal.stream(self._journal_stdout_name)
-        journal_stderr: TextIO = journal.stream(self._journal_stderr_name)
+        self.journal_stdout: BinaryIO = journal.stream(self._journal_stdout_name)
+        self.journal_stderr: BinaryIO = journal.stream(self._journal_stderr_name)
         # hardware_resources.published ports -> not implemented at the moment
         # hardware_resources.seconds -> only for microvm
         args = [
@@ -120,8 +122,8 @@ async def start(
         self.qemu_process = proc = await asyncio.create_subprocess_exec(
             *args,
             stdin=asyncio.subprocess.DEVNULL,
-            stdout=journal_stdout,
-            stderr=journal_stderr,
+            stdout=self.journal_stdout,
+            stderr=self.journal_stderr,
         )
 
         print(
@@ -149,3 +151,8 @@ def send_shutdown_message(self):
     async def stop(self):
         """Stop the VM."""
         self.send_shutdown_message()
+
+        if self.journal_stdout and self.journal_stdout != asyncio.subprocess.DEVNULL:
+            self.journal_stdout.close()
+        if self.journal_stderr and self.journal_stderr != asyncio.subprocess.DEVNULL:
+            self.journal_stderr.close()

From 7ee5384edabd0326a5db1f5c14803730c0561d63 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 14 Nov 2024 16:40:41 +0100
Subject: [PATCH 882/990] Update PULL_REQUEST_TEMPLATE.md for dependencies
 check (#722)

Add a check for dependencies update
---
 .github/PULL_REQUEST_TEMPLATE.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index bcf764608..ff965a1de 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -10,6 +10,7 @@ Related ClickUp, GitHub or Jira tickets : ALEPH-XXX
 - [ ] New classes and functions contain docstrings explaining what they provide.
 - [ ] All new code is covered by relevant tests.
 - [ ] Documentation has been updated regarding these changes.
+- [ ] Dependencies update in the project.toml have been mirrored in the Debian package build script `packaging/Makefile`
 
 ## Changes
 

From 927ce06705d8a980926d2e2100aee4e4a62f1da7 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 20 Nov 2024 08:45:27 +0100
Subject: [PATCH 883/990] CI: docker: Error response from daemon:
 toomanyrequests (#725)

CI returned error:
docker: Error response from daemon: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit.
DAMN

Solution: :Use GitHub container repository instead of hub.docker.io
(default one for docker)

olethanh pushed the image using:
```bash
podman pull docker.io/alephim/vm-connector:alpha
podman tag alephim/vm-connector:alpha ghcr.io/aleph-im/vm-connector:alpha
podman push ghcr.io/aleph-im/vm-connector:alpha
```

Auth to github firt https://docs.github.com/fr/packages/working-with-a-github-packages-registry/working-with-the-container-registry
---
 .github/workflows/test-on-droplets-matrix.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
index dce875bf3..380e91341 100644
--- a/.github/workflows/test-on-droplets-matrix.yml
+++ b/.github/workflows/test-on-droplets-matrix.yml
@@ -134,7 +134,8 @@ jobs:
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 update"
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 upgrade -y"
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y docker.io apparmor-profiles"
-          ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
+          ssh root@${DROPLET_IPV4} "docker pull ghcr.io/aleph-im/vm-connector:alpha"
+          ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector ghcr.io/aleph-im/vm-connector:alpha"
 
           scp packaging/target/${{ matrix.os_config.package_name }} root@${DROPLET_IPV4}:/opt
           # "--force-confold" keeps existing config files during package install/upgrade, avoiding prompts.

From af429a468a32ffc88b0026a5ce0efddfffac1f5c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 23 Oct 2024 16:15:17 +0200
Subject: [PATCH 884/990] Problem: Database logs were displayed twice

---
 src/aleph/vm/orchestrator/cli.py     | 2 +-
 src/aleph/vm/orchestrator/metrics.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index ddcf8910d..c75f6989d 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -282,7 +282,7 @@ def run_db_migrations(connection):
 
 
 async def run_async_db_migrations():
-    async_engine = create_async_engine(make_db_url(), echo=True)
+    async_engine = create_async_engine(make_db_url(), echo=False)
     async with async_engine.begin() as conn:
         await conn.run_sync(run_db_migrations)
 
diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index 3b8cdf9f3..672225212 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -38,7 +38,7 @@
 
 def setup_engine():
     global AsyncSessionMaker
-    engine = create_async_engine(make_db_url(), echo=True)
+    engine = create_async_engine(make_db_url(), echo=False)
     AsyncSessionMaker = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
     return engine
 

From 2b0c894c5b5e30d6a3dde0b78b41741c54d657c2 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 23 Oct 2024 16:15:40 +0200
Subject: [PATCH 885/990] Problem: database message were too spammy

---
 src/aleph/vm/orchestrator/cli.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index c75f6989d..10f7ff6aa 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -300,6 +300,9 @@ def main():
         format=log_format,
     )
 
+    logging.getLogger("aiosqlite").setLevel(logging.WARNING)
+    logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
+
     settings.update(
         USE_JAILER=args.use_jailer,
         PRINT_SYSTEM_LOGS=args.system_logs,

From 915e08b50e55be6ea8d203bd0079f926ffb8a090 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 23 Oct 2024 16:16:31 +0200
Subject: [PATCH 886/990] Reduce logs

---
 src/aleph/vm/conf.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index b68ff9e88..230aa5bfd 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -396,8 +396,6 @@ def setup(self):
         STREAM_CHAINS[Chain.AVAX].rpc = str(self.RPC_AVAX)
         STREAM_CHAINS[Chain.BASE].rpc = str(self.RPC_BASE)
 
-        logger.info(STREAM_CHAINS)
-
         os.makedirs(self.MESSAGE_CACHE, exist_ok=True)
         os.makedirs(self.CODE_CACHE, exist_ok=True)
         os.makedirs(self.RUNTIME_CACHE, exist_ok=True)

From 6969959f78262f406ca05fc932fd3eed9736f08d Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 23 Oct 2024 16:18:37 +0200
Subject: [PATCH 887/990] Log formatting with more information

---
 src/aleph/vm/orchestrator/cli.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index 10f7ff6aa..a4c56cb38 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -293,8 +293,10 @@ def main():
     log_format = (
         "%(relativeCreated)4f | %(levelname)s | %(message)s"
         if args.profile
-        else "%(asctime)s | %(levelname)s | %(message)s"
+        else "%(asctime)s | %(levelname)s %(name)s:%(lineno)s | %(message)s"
     )
+    # log_format = "[%(asctime)s] p%(process)s {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s"
+
     logging.basicConfig(
         level=args.loglevel,
         format=log_format,

From 3aaca16f744f416ec39c8e6c0b686dd01d7df11c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 29 Oct 2024 09:47:19 +0100
Subject: [PATCH 888/990] Problem: Logs didn't show which VM was used

Solution:
Add ContextManager that allow showing which VM is currently acted uppon,
making log easier far more useful
---
 src/aleph/vm/orchestrator/cli.py            |  6 +--
 src/aleph/vm/orchestrator/custom_logs.py    | 58 +++++++++++++++++++++
 src/aleph/vm/orchestrator/views/__init__.py |  5 +-
 3 files changed, 64 insertions(+), 5 deletions(-)
 create mode 100644 src/aleph/vm/orchestrator/custom_logs.py

diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index a4c56cb38..db5959b5d 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -23,6 +23,7 @@
 from aleph.vm.version import __version__, get_version_from_apt, get_version_from_git
 
 from . import metrics, supervisor
+from .custom_logs import setup_handlers
 from .pubsub import PubSub
 from .run import run_code_on_event, run_code_on_request, start_persistent_vm
 
@@ -297,10 +298,7 @@ def main():
     )
     # log_format = "[%(asctime)s] p%(process)s {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s"
 
-    logging.basicConfig(
-        level=args.loglevel,
-        format=log_format,
-    )
+    setup_handlers(args, log_format)
 
     logging.getLogger("aiosqlite").setLevel(logging.WARNING)
     logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
diff --git a/src/aleph/vm/orchestrator/custom_logs.py b/src/aleph/vm/orchestrator/custom_logs.py
new file mode 100644
index 000000000..5809b4136
--- /dev/null
+++ b/src/aleph/vm/orchestrator/custom_logs.py
@@ -0,0 +1,58 @@
+import contextlib
+import logging
+from contextvars import ContextVar
+
+from aleph_message.models import ItemHash
+
+from aleph.vm.models import VmExecution
+
+ctx_current_execution: ContextVar[VmExecution | None] = ContextVar("current_execution")
+ctx_current_execution_hash: ContextVar[ItemHash | None] = ContextVar("current_execution_hash")
+
+
+@contextlib.contextmanager
+def set_vm_for_logging(vm_hash):
+    token = ctx_current_execution_hash.set(vm_hash)
+    try:
+        yield
+    finally:
+        ctx_current_execution_hash.reset(token)
+
+
+class InjectingFilter(logging.Filter):
+    """
+    A filter which injects context-specific information into logs
+    """
+
+    def filter(self, record):
+
+        vm_hash = ctx_current_execution_hash.get(None)
+        if not vm_hash:
+            vm_execution: VmExecution | None = ctx_current_execution.get(None)
+            if vm_execution:
+                vm_hash = vm_execution.vm_hash
+
+        if not vm_hash:
+            return False
+
+        record.vm_hash = vm_hash
+        return True
+
+
+def setup_handlers(args, log_format):
+    # Set up two custom handler, one that will add the VM information if present and the other print if not
+    execution_handler = logging.StreamHandler()
+    execution_handler.addFilter(InjectingFilter())
+    execution_handler.setFormatter(
+        logging.Formatter("%(asctime)s | %(levelname)s %(name)s:%(lineno)s | {%(vm_hash)s} %(message)s ")
+    )
+    non_execution_handler = logging.StreamHandler()
+    non_execution_handler.addFilter(lambda x: ctx_current_execution_hash.get(None) is None)
+    non_execution_handler.setFormatter(
+        logging.Formatter("%(asctime)s | %(levelname)s %(name)s:%(lineno)s | %(message)s ")
+    )
+    logging.basicConfig(
+        level=args.loglevel,
+        format=log_format,
+        handlers=[non_execution_handler, execution_handler],
+    )
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 4bba01aa8..1793a4678 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -1,4 +1,5 @@
 import binascii
+import contextlib
 import logging
 from decimal import Decimal
 from hashlib import sha256
@@ -25,6 +26,7 @@
 from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInitError
 from aleph.vm.orchestrator import payment, status
 from aleph.vm.orchestrator.chain import STREAM_CHAINS, ChainInfo
+from aleph.vm.orchestrator.custom_logs import set_vm_for_logging
 from aleph.vm.orchestrator.messages import try_get_message
 from aleph.vm.orchestrator.metrics import get_execution_records
 from aleph.vm.orchestrator.payment import (
@@ -75,7 +77,8 @@ async def run_code_from_path(request: web.Request) -> web.Response:
         ) from e
 
     pool: VmPool = request.app["vm_pool"]
-    return await run_code_on_request(message_ref, path, pool, request)
+    with set_vm_for_logging(vm_hash=message_ref):
+        return await run_code_on_request(message_ref, path, pool, request)
 
 
 async def run_code_from_hostname(request: web.Request) -> web.Response:

From 8cd6b3dd7d3c089a6cb6b05dc9e772d6620cdf6d Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 29 Oct 2024 10:27:14 +0100
Subject: [PATCH 889/990] More log context on more endpoint

---
 src/aleph/vm/orchestrator/views/__init__.py |   3 +-
 src/aleph/vm/orchestrator/views/operator.py | 339 ++++++++++----------
 2 files changed, 176 insertions(+), 166 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 1793a4678..4c9b3866d 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -115,7 +115,8 @@ async def run_code_from_hostname(request: web.Request) -> web.Response:
                 return HTTPNotFound(reason="Invalid message reference")
 
     pool = request.app["vm_pool"]
-    return await run_code_on_request(message_ref, path, pool, request)
+    with set_vm_for_logging(vm_hash=message_ref):
+        return await run_code_on_request(message_ref, path, pool, request)
 
 
 def authenticate_request(request: web.Request) -> None:
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index af0e98f45..72218f3ea 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -15,6 +15,7 @@
 from aleph.vm.conf import settings
 from aleph.vm.controllers.qemu.client import QemuVmClient
 from aleph.vm.models import VmExecution
+from aleph.vm.orchestrator.custom_logs import set_vm_for_logging
 from aleph.vm.orchestrator.run import create_vm_execution_or_raise_http_error
 from aleph.vm.orchestrator.views.authentication import (
     authenticate_websocket_message,
@@ -63,36 +64,37 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
     allow Javascript to set headers in WebSocket requests.
     """
     vm_hash = get_itemhash_or_400(request.match_info)
-    pool: VmPool = request.app["vm_pool"]
-    execution = get_execution_or_404(vm_hash, pool=pool)
+    with set_vm_for_logging(vm_hash=vm_hash):
+        pool: VmPool = request.app["vm_pool"]
+        execution = get_execution_or_404(vm_hash, pool=pool)
 
-    if execution.vm is None:
-        raise web.HTTPBadRequest(body=f"VM {vm_hash} is not running")
-    queue = None
-    try:
-        ws = web.WebSocketResponse()
-        logger.info(f"starting websocket: {request.path}")
-        await ws.prepare(request)
+        if execution.vm is None:
+            raise web.HTTPBadRequest(body=f"VM {vm_hash} is not running")
+        queue = None
         try:
-            await authenticate_websocket_for_vm_or_403(execution, vm_hash, ws)
-            await ws.send_json({"status": "connected"})
+            ws = web.WebSocketResponse()
+            logger.info(f"starting websocket: {request.path}")
+            await ws.prepare(request)
+            try:
+                await authenticate_websocket_for_vm_or_403(execution, vm_hash, ws)
+                await ws.send_json({"status": "connected"})
 
-            queue = execution.vm.get_log_queue()
+                queue = execution.vm.get_log_queue()
 
-            while True:
-                log_type, message = await queue.get()
-                assert log_type in ("stdout", "stderr")
-                logger.debug(message)
+                while True:
+                    log_type, message = await queue.get()
+                    assert log_type in ("stdout", "stderr")
+                    logger.debug(message)
 
-                await ws.send_json({"type": log_type, "message": message})
+                    await ws.send_json({"type": log_type, "message": message})
 
-        finally:
-            await ws.close()
-            logger.info(f"connection  {ws} closed")
+            finally:
+                await ws.close()
+                logger.info(f"connection  {ws} closed")
 
-    finally:
-        if queue:
-            execution.vm.unregister_queue(queue)
+        finally:
+            if queue:
+                execution.vm.unregister_queue(queue)
 
 
 @cors_allow_all
@@ -100,20 +102,21 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
 async def operate_logs(request: web.Request, authenticated_sender: str) -> web.StreamResponse:
     """Logs of a VM (not streaming)"""
     vm_hash = get_itemhash_or_400(request.match_info)
-    pool: VmPool = request.app["vm_pool"]
-    execution = get_execution_or_404(vm_hash, pool=pool)
-    if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=403, body="Unauthorized sender")
+    with set_vm_for_logging(vm_hash=vm_hash):
+        pool: VmPool = request.app["vm_pool"]
+        execution = get_execution_or_404(vm_hash, pool=pool)
+        if not is_sender_authorized(authenticated_sender, execution.message):
+            return web.Response(status=403, body="Unauthorized sender")
 
-    response = web.StreamResponse()
-    response.headers["Content-Type"] = "text/plain"
-    await response.prepare(request)
+        response = web.StreamResponse()
+        response.headers["Content-Type"] = "text/plain"
+        await response.prepare(request)
 
-    for entry in execution.vm.past_logs():
-        msg = f'{entry["__REALTIME_TIMESTAMP"].isoformat()}> {entry["MESSAGE"]}'
-        await response.write(msg.encode())
-    await response.write_eof()
-    return response
+        for entry in execution.vm.past_logs():
+            msg = f'{entry["__REALTIME_TIMESTAMP"].isoformat()}> {entry["MESSAGE"]}'
+            await response.write(msg.encode())
+        await response.write_eof()
+        return response
 
 
 async def authenticate_websocket_for_vm_or_403(execution: VmExecution, vm_hash: ItemHash, ws: web.WebSocketResponse):
@@ -154,24 +157,25 @@ async def operate_expire(request: web.Request, authenticated_sender: str) -> web
 
     A timeout may be specified to delay the action."""
     vm_hash = get_itemhash_or_400(request.match_info)
-    try:
-        timeout = float(ItemHash(request.match_info["timeout"]))
-    except (KeyError, ValueError) as error:
-        raise web.HTTPBadRequest(body="Invalid timeout duration") from error
-    if not 0 < timeout < timedelta(days=10).total_seconds():
-        return web.HTTPBadRequest(body="Invalid timeout duration")
+    with set_vm_for_logging(vm_hash=vm_hash):
+        try:
+            timeout = float(ItemHash(request.match_info["timeout"]))
+        except (KeyError, ValueError) as error:
+            raise web.HTTPBadRequest(body="Invalid timeout duration") from error
+        if not 0 < timeout < timedelta(days=10).total_seconds():
+            return web.HTTPBadRequest(body="Invalid timeout duration")
 
-    pool: VmPool = request.app["vm_pool"]
-    execution = get_execution_or_404(vm_hash, pool=pool)
+        pool: VmPool = request.app["vm_pool"]
+        execution = get_execution_or_404(vm_hash, pool=pool)
 
-    if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=403, body="Unauthorized sender")
+        if not is_sender_authorized(authenticated_sender, execution.message):
+            return web.Response(status=403, body="Unauthorized sender")
 
-    logger.info(f"Expiring in {timeout} seconds: {execution.vm_hash}")
-    await execution.expire(timeout=timeout)
-    execution.persistent = False
+        logger.info(f"Expiring in {timeout} seconds: {execution.vm_hash}")
+        await execution.expire(timeout=timeout)
+        execution.persistent = False
 
-    return web.Response(status=200, body=f"Expiring VM with ref {vm_hash} in {timeout} seconds")
+        return web.Response(status=200, body=f"Expiring VM with ref {vm_hash} in {timeout} seconds")
 
 
 @cors_allow_all
@@ -179,53 +183,54 @@ async def operate_expire(request: web.Request, authenticated_sender: str) -> web
 async def operate_confidential_initialize(request: web.Request, authenticated_sender: str) -> web.Response:
     """Start the confidential virtual machine if possible."""
     vm_hash = get_itemhash_or_400(request.match_info)
+    with set_vm_for_logging(vm_hash=vm_hash):
 
-    pool: VmPool = request.app["vm_pool"]
-    logger.debug(f"Iterating through running executions... {pool.executions}")
-    execution = get_execution_or_404(vm_hash, pool=pool)
+        pool: VmPool = request.app["vm_pool"]
+        logger.debug(f"Iterating through running executions... {pool.executions}")
+        execution = get_execution_or_404(vm_hash, pool=pool)
 
-    if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=403, body="Unauthorized sender")
+        if not is_sender_authorized(authenticated_sender, execution.message):
+            return web.Response(status=403, body="Unauthorized sender")
 
-    if execution.is_running:
-        return web.json_response(
-            {"code": "vm_running", "description": "Operation not allowed, instance already running"},
-            status=HTTPStatus.BAD_REQUEST,
-        )
-    if not execution.is_confidential:
-        return web.json_response(
-            {"code": "not_confidential", "description": "Instance is not a confidential instance"},
-            status=HTTPStatus.BAD_REQUEST,
-        )
+        if execution.is_running:
+            return web.json_response(
+                {"code": "vm_running", "description": "Operation not allowed, instance already running"},
+                status=HTTPStatus.BAD_REQUEST,
+            )
+        if not execution.is_confidential:
+            return web.json_response(
+                {"code": "not_confidential", "description": "Instance is not a confidential instance"},
+                status=HTTPStatus.BAD_REQUEST,
+            )
 
-    post = await request.post()
+        post = await request.post()
 
-    vm_session_path = settings.CONFIDENTIAL_SESSION_DIRECTORY / vm_hash
-    vm_session_path.mkdir(exist_ok=True)
+        vm_session_path = settings.CONFIDENTIAL_SESSION_DIRECTORY / vm_hash
+        vm_session_path.mkdir(exist_ok=True)
 
-    session_file_content = post.get("session")
-    if not session_file_content:
-        return web.json_response(
-            {"code": "field_missing", "description": "Session field is missing"},
-            status=HTTPStatus.BAD_REQUEST,
-        )
+        session_file_content = post.get("session")
+        if not session_file_content:
+            return web.json_response(
+                {"code": "field_missing", "description": "Session field is missing"},
+                status=HTTPStatus.BAD_REQUEST,
+            )
 
-    session_file_path = vm_session_path / "vm_session.b64"
-    session_file_path.write_bytes(session_file_content.file.read())
+        session_file_path = vm_session_path / "vm_session.b64"
+        session_file_path.write_bytes(session_file_content.file.read())
 
-    godh_file_content = post.get("godh")
-    if not godh_file_content:
-        return web.json_response(
-            {"code": "field_missing", "description": "godh field is missing. Please provide a GODH file"},
-            status=HTTPStatus.BAD_REQUEST,
-        )
+        godh_file_content = post.get("godh")
+        if not godh_file_content:
+            return web.json_response(
+                {"code": "field_missing", "description": "godh field is missing. Please provide a GODH file"},
+                status=HTTPStatus.BAD_REQUEST,
+            )
 
-    godh_file_path = vm_session_path / "vm_godh.b64"
-    godh_file_path.write_bytes(godh_file_content.file.read())
+        godh_file_path = vm_session_path / "vm_godh.b64"
+        godh_file_path.write_bytes(godh_file_content.file.read())
 
-    pool.systemd_manager.enable_and_start(execution.controller_service)
+        pool.systemd_manager.enable_and_start(execution.controller_service)
 
-    return web.Response(status=200, body=f"Started VM with ref {vm_hash}")
+        return web.Response(status=200, body=f"Started VM with ref {vm_hash}")
 
 
 @cors_allow_all
@@ -233,23 +238,23 @@ async def operate_confidential_initialize(request: web.Request, authenticated_se
 async def operate_stop(request: web.Request, authenticated_sender: str) -> web.Response:
     """Stop the virtual machine, smoothly if possible."""
     vm_hash = get_itemhash_or_400(request.match_info)
+    with set_vm_for_logging(vm_hash=vm_hash):
+        pool: VmPool = request.app["vm_pool"]
+        logger.debug(f"Iterating through running executions... {pool.executions}")
+        execution = get_execution_or_404(vm_hash, pool=pool)
 
-    pool: VmPool = request.app["vm_pool"]
-    logger.debug(f"Iterating through running executions... {pool.executions}")
-    execution = get_execution_or_404(vm_hash, pool=pool)
-
-    if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=403, body="Unauthorized sender")
+        if not is_sender_authorized(authenticated_sender, execution.message):
+            return web.Response(status=403, body="Unauthorized sender")
 
-    if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=403, body="Unauthorized sender")
+        if not is_sender_authorized(authenticated_sender, execution.message):
+            return web.Response(status=403, body="Unauthorized sender")
 
-    if execution.is_running:
-        logger.info(f"Stopping {execution.vm_hash}")
-        await pool.stop_vm(execution.vm_hash)
-        return web.Response(status=200, body=f"Stopped VM with ref {vm_hash}")
-    else:
-        return web.Response(status=200, body="Already stopped, nothing to do")
+        if execution.is_running:
+            logger.info(f"Stopping {execution.vm_hash}")
+            await pool.stop_vm(execution.vm_hash)
+            return web.Response(status=200, body=f"Stopped VM with ref {vm_hash}")
+        else:
+            return web.Response(status=200, body="Already stopped, nothing to do")
 
 
 @cors_allow_all
@@ -259,24 +264,25 @@ async def operate_reboot(request: web.Request, authenticated_sender: str) -> web
     Reboots the virtual machine, smoothly if possible.
     """
     vm_hash = get_itemhash_or_400(request.match_info)
-    pool: VmPool = request.app["vm_pool"]
-    execution = get_execution_or_404(vm_hash, pool=pool)
-
-    if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=403, body="Unauthorized sender")
-
-    if execution.is_running:
-        logger.info(f"Rebooting {execution.vm_hash}")
-        if execution.persistent:
-            pool.systemd_manager.restart(execution.controller_service)
+    with set_vm_for_logging(vm_hash=vm_hash):
+        pool: VmPool = request.app["vm_pool"]
+        execution = get_execution_or_404(vm_hash, pool=pool)
+
+        if not is_sender_authorized(authenticated_sender, execution.message):
+            return web.Response(status=403, body="Unauthorized sender")
+
+        if execution.is_running:
+            logger.info(f"Rebooting {execution.vm_hash}")
+            if execution.persistent:
+                pool.systemd_manager.restart(execution.controller_service)
+            else:
+                await pool.stop_vm(vm_hash)
+                pool.forget_vm(vm_hash)
+
+                await create_vm_execution_or_raise_http_error(vm_hash=vm_hash, pool=pool)
+            return web.Response(status=200, body=f"Rebooted VM with ref {vm_hash}")
         else:
-            await pool.stop_vm(vm_hash)
-            pool.forget_vm(vm_hash)
-
-            await create_vm_execution_or_raise_http_error(vm_hash=vm_hash, pool=pool)
-        return web.Response(status=200, body=f"Rebooted VM with ref {vm_hash}")
-    else:
-        return web.Response(status=200, body=f"Starting VM (was not running) with ref {vm_hash}")
+            return web.Response(status=200, body=f"Starting VM (was not running) with ref {vm_hash}")
 
 
 @cors_allow_all
@@ -286,23 +292,24 @@ async def operate_confidential_measurement(request: web.Request, authenticated_s
     Fetch the sev measurement for the VM
     """
     vm_hash = get_itemhash_or_400(request.match_info)
-    pool: VmPool = request.app["vm_pool"]
-    execution = get_execution_or_404(vm_hash, pool=pool)
+    with set_vm_for_logging(vm_hash=vm_hash):
+        pool: VmPool = request.app["vm_pool"]
+        execution = get_execution_or_404(vm_hash, pool=pool)
 
-    if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=403, body="Unauthorized sender")
+        if not is_sender_authorized(authenticated_sender, execution.message):
+            return web.Response(status=403, body="Unauthorized sender")
 
-    if not execution.is_running:
-        raise web.HTTPForbidden(body="Operation not running")
-    vm_client = QemuVmClient(execution.vm)
-    vm_sev_info = vm_client.query_sev_info()
-    launch_measure = vm_client.query_launch_measure()
+        if not execution.is_running:
+            raise web.HTTPForbidden(body="Operation not running")
+        vm_client = QemuVmClient(execution.vm)
+        vm_sev_info = vm_client.query_sev_info()
+        launch_measure = vm_client.query_launch_measure()
 
-    return web.json_response(
-        data={"sev_info": vm_sev_info, "launch_measure": launch_measure},
-        status=200,
-        dumps=dumps_for_json,
-    )
+        return web.json_response(
+            data={"sev_info": vm_sev_info, "launch_measure": launch_measure},
+            status=200,
+            dumps=dumps_for_json,
+        )
 
 
 class InjectSecretParams(BaseModel):
@@ -330,25 +337,26 @@ async def operate_confidential_inject_secret(request: web.Request, authenticated
         return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code)
 
     vm_hash = get_itemhash_or_400(request.match_info)
-    pool: VmPool = request.app["vm_pool"]
-    execution = get_execution_or_404(vm_hash, pool=pool)
-    if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=403, body="Unauthorized sender")
+    with set_vm_for_logging(vm_hash=vm_hash):
+        pool: VmPool = request.app["vm_pool"]
+        execution = get_execution_or_404(vm_hash, pool=pool)
+        if not is_sender_authorized(authenticated_sender, execution.message):
+            return web.Response(status=403, body="Unauthorized sender")
 
-    # if not execution.is_running:
-    #     raise web.HTTPForbidden(body="Operation not running")
-    vm_client = QemuVmClient(execution.vm)
-    vm_client.inject_secret(params.packet_header, params.secret)
-    vm_client.continue_execution()
+        # if not execution.is_running:
+        #     raise web.HTTPForbidden(body="Operation not running")
+        vm_client = QemuVmClient(execution.vm)
+        vm_client.inject_secret(params.packet_header, params.secret)
+        vm_client.continue_execution()
 
-    status = vm_client.query_status()
-    print(status["status"] != "running")
+        status = vm_client.query_status()
+        print(status["status"] != "running")
 
-    return web.json_response(
-        data={"status": status},
-        status=200,
-        dumps=dumps_for_json,
-    )
+        return web.json_response(
+            data={"status": status},
+            status=200,
+            dumps=dumps_for_json,
+        )
 
 
 @cors_allow_all
@@ -358,25 +366,26 @@ async def operate_erase(request: web.Request, authenticated_sender: str) -> web.
     Stop the virtual machine first if needed.
     """
     vm_hash = get_itemhash_or_400(request.match_info)
-    pool: VmPool = request.app["vm_pool"]
-    execution = get_execution_or_404(vm_hash, pool=pool)
+    with set_vm_for_logging(vm_hash=vm_hash):
+        pool: VmPool = request.app["vm_pool"]
+        execution = get_execution_or_404(vm_hash, pool=pool)
 
-    if not is_sender_authorized(authenticated_sender, execution.message):
-        return web.Response(status=403, body="Unauthorized sender")
+        if not is_sender_authorized(authenticated_sender, execution.message):
+            return web.Response(status=403, body="Unauthorized sender")
 
-    logger.info(f"Erasing {execution.vm_hash}")
+        logger.info(f"Erasing {execution.vm_hash}")
 
-    # Stop the VM
-    await pool.stop_vm(execution.vm_hash)
-    if execution.vm_hash in pool.executions:
-        logger.warning(f"VM {execution.vm_hash} was not stopped properly, forgetting it anyway")
-        pool.forget_vm(execution.vm_hash)
-
-    # Delete all data
-    if execution.resources is not None:
-        for volume in execution.resources.volumes:
-            if not volume.read_only:
-                logger.info(f"Deleting volume {volume.path_on_host}")
-                volume.path_on_host.unlink()
-
-    return web.Response(status=200, body=f"Erased VM with ref {vm_hash}")
+        # Stop the VM
+        await pool.stop_vm(execution.vm_hash)
+        if execution.vm_hash in pool.executions:
+            logger.warning(f"VM {execution.vm_hash} was not stopped properly, forgetting it anyway")
+            pool.forget_vm(execution.vm_hash)
+
+        # Delete all data
+        if execution.resources is not None:
+            for volume in execution.resources.volumes:
+                if not volume.read_only:
+                    logger.info(f"Deleting volume {volume.path_on_host}")
+                    volume.path_on_host.unlink()
+
+        return web.Response(status=200, body=f"Erased VM with ref {vm_hash}")

From fafed8e45120342c98ada433aa9c740a8ff19ac1 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 29 Oct 2024 10:27:26 +0100
Subject: [PATCH 890/990] Reduce log output for version

---
 src/aleph/vm/version.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/version.py b/src/aleph/vm/version.py
index ba4f34336..73118aa74 100644
--- a/src/aleph/vm/version.py
+++ b/src/aleph/vm/version.py
@@ -1,17 +1,17 @@
 import logging
-from subprocess import CalledProcessError, check_output
+from subprocess import STDOUT, CalledProcessError, check_output
 
 logger = logging.getLogger(__name__)
 
 
 def get_version_from_git() -> str | None:
     try:
-        return check_output(("git", "describe", "--tags")).strip().decode()
+        return check_output(("git", "describe", "--tags"), stderr=STDOUT).strip().decode()
     except FileNotFoundError:
-        logger.warning("git not found")
+        logger.warning("version: git not found")
         return None
-    except CalledProcessError:
-        logger.warning("git description not available")
+    except CalledProcessError as err:
+        logger.info("version: git description not available: %s", err.output.decode().strip())
         return None
 
 

From 0e2770ac74a28a1113ee5d8b74c0ee3f3594b963 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 29 Oct 2024 10:30:36 +0100
Subject: [PATCH 891/990] Fix broken ruff

---
 pyproject.toml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 50c314c2f..d33f05a04 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -191,15 +191,16 @@ lint.ignore = [
   # Allow the use of assert statements
   "S101",
 ]
-# Tests can use magic values, assertions, and relative imports
-lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ]
 #[tool.ruff.flake8-tidy-imports]
 #ban-relative-imports = "all"
 #unfixable = [
 #  # Don't touch unused imports
 #  "F401",
 #]
-lint.isort = [ "aleph.vm" ]
+#lint.isort = [ "aleph.vm" ]
+
+# Tests can use magic values, assertions, and relative imports
+lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ]
 
 [tool.pytest.ini_options]
 pythonpath = [

From d1b15c682f151795ad1ffa79a594e218043b2b4b Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 29 Oct 2024 10:45:36 +0100
Subject: [PATCH 892/990] Fix: Log level could not be configured from settings

Solution: Add a `LOG_LEVEL` setting defaulting to "WARNING".

Replace PR #581

https://github.com/aleph-im/aleph-vm/pull/581
---
 src/aleph/vm/conf.py                     | 1 +
 src/aleph/vm/orchestrator/cli.py         | 9 +++++++--
 src/aleph/vm/orchestrator/custom_logs.py | 6 +-----
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 230aa5bfd..739cfda9f 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -136,6 +136,7 @@ class Settings(BaseSettings):
     # System logs make boot ~2x slower
     PRINT_SYSTEM_LOGS = False
     IGNORE_TRACEBACK_FROM_DIAGNOSTICS = True
+    LOG_LEVEL = "WARNING"
     DEBUG_ASYNCIO = False
 
     # Networking does not work inside Docker/Podman
diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index db5959b5d..754b4e59c 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -66,7 +66,7 @@ def parse_args(args):
         help="set loglevel to INFO",
         action="store_const",
         const=logging.INFO,
-        default=logging.WARNING,
+        default=settings.LOG_LEVEL,
     )
     parser.add_argument(
         "-vv",
@@ -298,7 +298,12 @@ def main():
     )
     # log_format = "[%(asctime)s] p%(process)s {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s"
 
-    setup_handlers(args, log_format)
+    handlers = setup_handlers(args, log_format)
+    logging.basicConfig(
+        level=args.loglevel,
+        format=log_format,
+        handlers=handlers,
+    )
 
     logging.getLogger("aiosqlite").setLevel(logging.WARNING)
     logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
diff --git a/src/aleph/vm/orchestrator/custom_logs.py b/src/aleph/vm/orchestrator/custom_logs.py
index 5809b4136..9150fdd73 100644
--- a/src/aleph/vm/orchestrator/custom_logs.py
+++ b/src/aleph/vm/orchestrator/custom_logs.py
@@ -51,8 +51,4 @@ def setup_handlers(args, log_format):
     non_execution_handler.setFormatter(
         logging.Formatter("%(asctime)s | %(levelname)s %(name)s:%(lineno)s | %(message)s ")
     )
-    logging.basicConfig(
-        level=args.loglevel,
-        format=log_format,
-        handlers=[non_execution_handler, execution_handler],
-    )
+    return [non_execution_handler, execution_handler]

From 11f0924e8525f9888661a089a50470c08bd2f6fa Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 18 Nov 2024 15:04:22 +0100
Subject: [PATCH 893/990] set LOG_LEVEL as the same between for sqlite that for
 the general app

Co-authored-by: nesitor 
---
 src/aleph/vm/orchestrator/cli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index 754b4e59c..bbae396d4 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -305,8 +305,8 @@ def main():
         handlers=handlers,
     )
 
-    logging.getLogger("aiosqlite").setLevel(logging.WARNING)
-    logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
+    logging.getLogger("aiosqlite").setLevel(settings.LOG_LEVEL)
+    logging.getLogger("sqlalchemy.engine").setLevel(settings.LOG_LEVEL)
 
     settings.update(
         USE_JAILER=args.use_jailer,

From 4d319f03b2b7bdc7a75da48a052ada8744679a1c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 18 Nov 2024 16:04:11 +0100
Subject: [PATCH 894/990] Add tests for opreate_stop

---
 tests/supervisor/views/test_operator.py | 76 +++++++++++++++++++++++++
 1 file changed, 76 insertions(+)

diff --git a/tests/supervisor/views/test_operator.py b/tests/supervisor/views/test_operator.py
index b8e370de7..86c6c5cd5 100644
--- a/tests/supervisor/views/test_operator.py
+++ b/tests/supervisor/views/test_operator.py
@@ -102,6 +102,82 @@ async def test_operator_confidential_initialize_already_running(aiohttp_client,
     }
 
 
+@pytest.mark.asyncio
+@pytest.mark.skip()
+async def test_operator_expire(aiohttp_client, mocker):
+    """Test that the expires endpoint work. SPOILER it doesn't"""
+
+    settings.ENABLE_QEMU_SUPPORT = True
+    settings.ENABLE_CONFIDENTIAL_COMPUTING = True
+    settings.setup()
+
+    vm_hash = ItemHash(settings.FAKE_INSTANCE_ID)
+    instance_message = await get_message(ref=vm_hash)
+
+    fake_vm_pool = mocker.Mock(
+        executions={
+            vm_hash: mocker.Mock(
+                vm_hash=vm_hash,
+                message=instance_message.content,
+                is_confidential=False,
+                is_running=False,
+            ),
+        },
+    )
+
+    # Disable auth
+    mocker.patch(
+        "aleph.vm.orchestrator.views.authentication.authenticate_jwk",
+        return_value=instance_message.sender,
+    )
+    app = setup_webapp()
+    app["vm_pool"] = fake_vm_pool
+    client: TestClient = await aiohttp_client(app)
+    response = await client.post(
+        f"/control/machine/{vm_hash}/expire",
+        data={"timeout": 1},
+        # json={"timeout": 1},
+    )
+    assert response.status == 200, await response.text()
+    assert fake_vm_pool["executions"][vm_hash].expire.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_operator_stop(aiohttp_client, mocker):
+    """Test that the stop endpoint call the method on pool"""
+
+    settings.ENABLE_QEMU_SUPPORT = True
+    settings.ENABLE_CONFIDENTIAL_COMPUTING = True
+    settings.setup()
+
+    vm_hash = ItemHash(settings.FAKE_INSTANCE_ID)
+    instance_message = await get_message(ref=vm_hash)
+
+    fake_vm_pool = mocker.AsyncMock(
+        executions={
+            vm_hash: mocker.AsyncMock(
+                vm_hash=vm_hash,
+                message=instance_message.content,
+                is_running=True,
+            ),
+        },
+    )
+
+    # Disable auth
+    mocker.patch(
+        "aleph.vm.orchestrator.views.authentication.authenticate_jwk",
+        return_value=instance_message.sender,
+    )
+    app = setup_webapp()
+    app["vm_pool"] = fake_vm_pool
+    client: TestClient = await aiohttp_client(app)
+    response = await client.post(
+        f"/control/machine/{vm_hash}/stop",
+    )
+    assert response.status == 200, await response.text()
+    assert fake_vm_pool.stop_vm.call_count == 1
+
+
 @pytest.mark.asyncio
 async def test_operator_confidential_initialize_not_confidential(aiohttp_client, mocker):
     """Test that the confidential initialize endpoint rejects if the VM is not confidential"""

From efa7559080828bf912812da1e1ace8eeda7551d6 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 22 Oct 2024 14:46:49 +0200
Subject: [PATCH 895/990] Add debug string on VmExecution

to debug more easily
---
 src/aleph/vm/models.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 35de4076d..9aee9320a 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -145,6 +145,9 @@ def has_resources(self) -> bool:
         else:
             return True
 
+    def __repr__(self):
+        return f""
+
     def __init__(
         self,
         vm_hash: ItemHash,

From 44ca8e08e9b79b61dec50a61d3cb1d56cde12817 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 22 Oct 2024 14:49:24 +0200
Subject: [PATCH 896/990] fix docstring

---
 tests/supervisor/test_authentication.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/supervisor/test_authentication.py b/tests/supervisor/test_authentication.py
index 6fb3d0811..73769c97c 100644
--- a/tests/supervisor/test_authentication.py
+++ b/tests/supervisor/test_authentication.py
@@ -243,7 +243,7 @@ async def view(request, authenticated_sender):
 async def generate_sol_signer_and_signed_headers_for_operation(
     patch_datetime_now, operation_payload: dict
 ) -> tuple[solathon.Keypair, dict]:
-    """Generate a temporary eth_account for testing and sign the operation with it"""
+    """Generate a temporary sol account for testing and sign the operation with it"""
 
     kp = solathon.Keypair()
     key = jwk.JWK.generate(

From ef2052f347f75f3b87c1fb479724da778f6528eb Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 23 Oct 2024 16:16:40 +0200
Subject: [PATCH 897/990] fix docstring

---
 src/aleph/vm/pool.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 025bfe45c..58a3e6fae 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -86,7 +86,7 @@ def teardown(self) -> None:
     async def create_a_vm(
         self, vm_hash: ItemHash, message: ExecutableContent, original: ExecutableContent, persistent: bool
     ) -> VmExecution:
-        """Create a new Aleph Firecracker VM from an Aleph function message."""
+        """Create a new VM from an Aleph function or instance message."""
         async with self.creation_lock:
             # Check if an execution is already present for this VM, then return it.
             # Do not `await` in this section.

From 890e4e8aaae2f3ffc4bb83ddd515aaee33bb4cd3 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 30 Oct 2024 12:36:14 +0100
Subject: [PATCH 898/990] Developer setup section

---
 README.md  | 73 ++++++++++++++++++++++++++++++++++++++++++++++++------
 TESTING.md |  2 +-
 2 files changed, 67 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 783f02a09..89b1966e4 100644
--- a/README.md
+++ b/README.md
@@ -13,32 +13,91 @@ Writing programs in Python using ASGI compatible frameworks (
 [Django](https://docs.djangoproject.com/en/3.0/topics/async/),
 ...) allows developers to use advanced functionalities not yet available for other languages.
 
-## 1. Install Aleph-VM from packages
+# Production install for Aleph-VM
+## Installation from packages
 
-Install Aleph-VM to run an Aleph.im Compute Resource Node easily from official pre-built packages.
 
-See the official user doc https://docs.aleph.im/nodes/compute/
+Head over to the  official user doc https://docs.aleph.im/nodes/compute/ on how to run an Aleph.im Compute Resource
+Node
 
 ## 2. Install Aleph-VM from source
 
-For development and testing, install Aleph-VM from source.
+This method is not recommended, except for development and testing.
+Read the installation document for the various components and the developer documentaation. 
 
 1. Install the [VM-Connector](./vm_connector/README.md)
 2. Install the [VM-Supervisor](src/aleph/vm/orchestrator/README.md).
 3. Install and configure a reverse-proxy such as [Caddy](./CONFIGURE_CADDY.md)
 
-## 3. Create and run an Aleph Program 
+## Create and run an Aleph Program 
 
 Have a look at [tutorials/README.md](tutorials/README.md) for a tutorial on how to program VMs
 as a user.
 
 The rest of this document focuses on how to run an Aleph-VM node that hosts and executes the programs. 
 
-## 4. Architecture
+# Developer setup
+As aleph-vm is highly integrated with the Linux system, modify it with it and run as root; it is HIGHLY advised to deploy it in a separate machine or server in the cloud.
+
+Note that aleph-vm do not run on Mac or Windows, not even the test suite. 
+
+A typical development set up would be to have a copy of the repo on your local machine and a deployment on a remote computer   to run and test it.
+You can sync the remote dev using rsync or using the Remote interpreter option in pycharm.
+
+## Deploying for dev on the remote
+We use the Debian package as a base as it contain the binary such as firecracker and sevctl, system configuration and, will install the dependencies.
+
+Unless specifically working on the vm-connector, it's easier to use the image from Docker. (
+see [VM-Connector/READNE](./vm_connector/README.md) for detail)
+
+```shell
+docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
+```
+
+
+Then install the debian package. Replace 1.2.0 with the latest released version of course.
+
+On Debian 12 (Bookworm):
+```shell
+wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/1.2.0/aleph-vm.debian-12.deb
+apt install /opt/aleph-vm.debian-12.deb
+```
+
+On Ubuntu 22.04 (Jammy Jellyfish):
+```
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/1.2.0/aleph-vm.ubuntu-22.04.deb
+sudo apt install /opt/aleph-vm.ubuntu-22.04.deb
+```
+
+On Ubuntu 24.04 (Noble Numbat):
+```
+sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/1.2.0/aleph-vm.ubuntu-24.04.deb
+sudo apt install /opt/aleph-vm.ubuntu-24.04.deb
+```
+
+Deactivate the systemd service so the system version is not run and doesn't conflict with the version you will launch by hand.  
+
+```shell
+sudo systemctl disable aleph-vm-supervisor.service
+```
+
+Clone the repository and create a virtual env to contain the dependency it.
+
+Inside the virtual env run
+```shell
+pip install -e .
+```
+This will install aleph-vm inside the  venv  in development mode, allowing you to run directly the aleph-vm command.
+
+
+## Testing
+see  [Testinc doc](./TESTING.md)
+
+# Architecture
 
 ![Aleph im VM - Details](https://user-images.githubusercontent.com/404665/127126908-3225a633-2c36-4129-8766-9810f2fcd7d6.png)
 
-### VM Supervisor
+### VM Supervisor (also called Orchestrator)
 
 Actually runs the programs in a secure environment on virtualization enabled systems. 
 
diff --git a/TESTING.md b/TESTING.md
index 7adbae0c6..e28711a76 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -1,6 +1,6 @@
 # Testing aleph-vm
 
-This procedure describes how to run tests on a local system.
+This procedure describes how to run tests on a dev system. See the dev setup section of the README first
 
 Tests also run on GitHub Actions via [the following workflow](./.github/workflows/test-on-droplets-matrix.yml).
 

From 94da10094ab80062457669b45ebf94e36aff1a20 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 30 Oct 2024 12:42:09 +0100
Subject: [PATCH 899/990] improve dev doc

---
 README.md | 96 +++++++++++++++++++++++++++++++++----------------------
 1 file changed, 58 insertions(+), 38 deletions(-)

diff --git a/README.md b/README.md
index 89b1966e4..9e95ea49b 100644
--- a/README.md
+++ b/README.md
@@ -36,59 +36,79 @@ as a user.
 
 The rest of this document focuses on how to run an Aleph-VM node that hosts and executes the programs. 
 
-# Developer setup
-As aleph-vm is highly integrated with the Linux system, modify it with it and run as root; it is HIGHLY advised to deploy it in a separate machine or server in the cloud.
+# Developer Setup
 
-Note that aleph-vm do not run on Mac or Windows, not even the test suite. 
+Due to aleph-vm’s deep integration with the Linux system, it must be run with root privileges and configured
+specifically for Linux. **It is strongly recommended** to deploy aleph-vm on a dedicated machine or a cloud-based server
+to ensure security and stability.
 
-A typical development set up would be to have a copy of the repo on your local machine and a deployment on a remote computer   to run and test it.
-You can sync the remote dev using rsync or using the Remote interpreter option in pycharm.
+> **Note**: aleph-vm does not run on macOS or Windows, including for testing purposes.
 
-## Deploying for dev on the remote
-We use the Debian package as a base as it contain the binary such as firecracker and sevctl, system configuration and, will install the dependencies.
+### Recommended Development Environment
 
-Unless specifically working on the vm-connector, it's easier to use the image from Docker. (
-see [VM-Connector/READNE](./vm_connector/README.md) for detail)
+A typical setup for developing aleph-vm involves:
 
-```shell
-docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
-```
+1. Cloning the repository on your local machine for code editing.
+2. Setting up a remote Linux server for deployment and testing.
 
+You can synchronize changes to the remote server using tools like `rsync` or PyCharm’s Remote Interpreter feature.
 
-Then install the debian package. Replace 1.2.0 with the latest released version of course.
+## Remote Development Deployment
 
-On Debian 12 (Bookworm):
-```shell
-wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/1.2.0/aleph-vm.debian-12.deb
-apt install /opt/aleph-vm.debian-12.deb
-```
+To deploy aleph-vm for development on a remote server, we start with the Debian package as it includes essential binaries like `firecracker` and `sevctl`, system
+   configuration, and dependencies.
 
-On Ubuntu 22.04 (Jammy Jellyfish):
-```
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/1.2.0/aleph-vm.ubuntu-22.04.deb
-sudo apt install /opt/aleph-vm.ubuntu-22.04.deb
-```
+1. **Run the vm-connector.**
 
-On Ubuntu 24.04 (Noble Numbat):
-```
-sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/1.2.0/aleph-vm.ubuntu-24.04.deb
-sudo apt install /opt/aleph-vm.ubuntu-24.04.deb
-```
+The vm-connector need to run for aleph-vm to works, even when running py.test.
 
-Deactivate the systemd service so the system version is not run and doesn't conflict with the version you will launch by hand.  
+Unless your focus is on developing the VM-Connector, using the Docker image is easier.
+   See [VM-Connector README](./vm_connector/README.md) for more details.
 
-```shell
-sudo systemctl disable aleph-vm-supervisor.service
-```
+   ```shell
+   docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
+   ```
 
-Clone the repository and create a virtual env to contain the dependency it.
+2. **Install the Debian Package**
+   Replace `1.2.0` with the latest release version.
 
-Inside the virtual env run
-```shell
-pip install -e .
-```
-This will install aleph-vm inside the  venv  in development mode, allowing you to run directly the aleph-vm command.
+   **On Debian 12 (Bookworm)**:
+   ```shell
+   wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/1.2.0/aleph-vm.debian-12.deb
+   sudo apt install /opt/aleph-vm.debian-12.deb
+   ```
 
+   **On Ubuntu 22.04 (Jammy Jellyfish)**:
+   ```shell
+   sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/1.2.0/aleph-vm.ubuntu-22.04.deb
+   sudo apt install /opt/aleph-vm.ubuntu-22.04.deb
+   ```
+
+   **On Ubuntu 24.04 (Noble Numbat)**:
+   ```shell
+   sudo wget -P /opt https://github.com/aleph-im/aleph-vm/releases/download/1.2.0/aleph-vm.ubuntu-24.04.deb
+   sudo apt install /opt/aleph-vm.ubuntu-24.04.deb
+   ```
+
+3. **Disable Systemd Service**  
+   To prevent conflicts, deactivate the system version of aleph-vm by disabling its `systemd` service.
+
+   ```shell
+   sudo systemctl disable aleph-vm-supervisor.service
+   ```
+
+4. **Clone the Repository and Set Up a Virtual Environment**
+    - Clone the aleph-vm repository to your development environment.
+    - Create a virtual environment to manage dependencies.
+
+   Inside the virtual environment, run:
+
+   ```shell
+   pip install -e .
+   ```
+
+   This installs aleph-vm in "editable" mode within the virtual environment, allowing you to use the `aleph-vm` command
+   directly during development.
 
 ## Testing
 see  [Testinc doc](./TESTING.md)

From b76e6e49f5b20a1c265f892b4cde5cb1c60cc411 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 30 Oct 2024 12:44:20 +0100
Subject: [PATCH 900/990] Fix setup validation for test_create_instance()

---
 tests/supervisor/test_instance.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/supervisor/test_instance.py b/tests/supervisor/test_instance.py
index 6a3ffa509..1319c99c4 100644
--- a/tests/supervisor/test_instance.py
+++ b/tests/supervisor/test_instance.py
@@ -63,8 +63,8 @@ async def test_create_instance():
     # Ensure that the settings are correct and required files present.
     settings.setup()
     settings.check()
-    if not settings.FAKE_DATA_RUNTIME.exists():
-        pytest.xfail("Test Runtime not setup. run `cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh`")
+    if not settings.FAKE_INSTANCE_BASE.exists():
+        pytest.xfail("Test Runtime not setup. run `cd runtimes/instance-rootfs && sudo ./create-debian-12-disk.sh`")
 
     # The database is required for the metrics and is currently not optional.
     engine = metrics.setup_engine()

From f1bad183d64fdfc3a793702ba11bb7df58fbbd31 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 20 Nov 2024 15:38:44 +0100
Subject: [PATCH 901/990] Apply suggestions from code review from Laurent

Co-authored-by: Bram 
---
 README.md  | 8 ++++----
 TESTING.md | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 9e95ea49b..61f9271cc 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,7 @@ Node
 ## 2. Install Aleph-VM from source
 
 This method is not recommended, except for development and testing.
-Read the installation document for the various components and the developer documentaation. 
+Read the installation document for the various components and the developer documentation. 
 
 1. Install the [VM-Connector](./vm_connector/README.md)
 2. Install the [VM-Supervisor](src/aleph/vm/orchestrator/README.md).
@@ -62,8 +62,8 @@ To deploy aleph-vm for development on a remote server, we start with the Debian
 
 The vm-connector need to run for aleph-vm to works, even when running py.test.
 
-Unless your focus is on developing the VM-Connector, using the Docker image is easier.
-   See [VM-Connector README](./vm_connector/README.md) for more details.
+Unless your focus is developing the VM-Connector, using the Docker image is easier.
+   See the [VM-Connector README](./vm_connector/README.md) for more details.
 
    ```shell
    docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha
@@ -111,7 +111,7 @@ Unless your focus is on developing the VM-Connector, using the Docker image is e
    directly during development.
 
 ## Testing
-see  [Testinc doc](./TESTING.md)
+See [Testinc doc](./TESTING.md)
 
 # Architecture
 
diff --git a/TESTING.md b/TESTING.md
index e28711a76..1c6e05bcb 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -1,6 +1,6 @@
 # Testing aleph-vm
 
-This procedure describes how to run tests on a dev system. See the dev setup section of the README first
+This procedure describes how to run tests on a dev system. See the dev setup section of the README first.
 
 Tests also run on GitHub Actions via [the following workflow](./.github/workflows/test-on-droplets-matrix.yml).
 

From 6836d6d5336d99cc327c6aae5e9d577d10968f86 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 26 Nov 2024 13:08:43 +0100
Subject: [PATCH 902/990] Add json logs endpoint which works for finished
 execution (#718)

* Add json logs endpoint which allow for past record

* review comment

* merge logs endpoint and new endpoint

* set logging system
---
 src/aleph/vm/conf.py                        |   6 +-
 src/aleph/vm/orchestrator/metrics.py        |   9 +
 src/aleph/vm/orchestrator/supervisor.py     |   4 +-
 src/aleph/vm/orchestrator/tasks.py          |   3 +-
 src/aleph/vm/orchestrator/views/operator.py |  50 ++++-
 tests/supervisor/views/test_operator.py     | 207 +++++++++++---------
 6 files changed, 166 insertions(+), 113 deletions(-)

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 739cfda9f..18add7170 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -133,8 +133,10 @@ class Settings(BaseSettings):
     CONNECTIVITY_DNS_HOSTNAME = "example.org"
 
     USE_JAILER = True
-    # System logs make boot ~2x slower
-    PRINT_SYSTEM_LOGS = False
+    # Changelog: PRINT_SYSTEM_LOGS use to print the MicroVM logs with the supervisor output.
+    # They are now in separate journald entries, disabling the settings disable the logs output of Firecracker VM (only)
+    # via the serial console. This break the logs endpoint for program, as such disabling it in prod is not recommended.
+    PRINT_SYSTEM_LOGS = True
     IGNORE_TRACEBACK_FROM_DIAGNOSTICS = True
     LOG_LEVEL = "WARNING"
     DEBUG_ASYNCIO = False
diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index 672225212..f7f166481 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -115,3 +115,12 @@ async def get_execution_records() -> Iterable[ExecutionRecord]:
         executions = result.scalars().all()
         await session.commit()
         return executions
+
+
+async def get_last_record_for_vm(vm_hash) -> ExecutionRecord | None:
+    """Get the execution records from the database."""
+    async with AsyncSessionMaker() as session:  # Use AsyncSession in a context manager
+        result = await session.execute(
+            select(ExecutionRecord).where(ExecutionRecord.vm_hash == vm_hash).limit(1)
+        )  # Use execute for querying
+        return result.scalar()
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index 118706370..a5ca999a8 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -51,7 +51,7 @@
     operate_confidential_measurement,
     operate_erase,
     operate_expire,
-    operate_logs,
+    operate_logs_json,
     operate_reboot,
     operate_stop,
     stream_logs,
@@ -104,7 +104,7 @@ def setup_webapp():
         # /control APIs are used to control the VMs and access their logs
         web.post("/control/allocation/notify", notify_allocation),
         web.get("/control/machine/{ref}/stream_logs", stream_logs),
-        web.get("/control/machine/{ref}/logs", operate_logs),
+        web.get("/control/machine/{ref}/logs", operate_logs_json),
         web.post("/control/machine/{ref}/expire", operate_expire),
         web.post("/control/machine/{ref}/stop", operate_stop),
         web.post("/control/machine/{ref}/erase", operate_erase),
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index c7062d931..921a2265f 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -11,7 +11,6 @@
 from aiohttp import web
 from aleph_message.models import (
     AlephMessage,
-    ItemHash,
     PaymentType,
     ProgramMessage,
     parse_message,
@@ -23,7 +22,7 @@
 from aleph.vm.pool import VmPool
 from aleph.vm.utils import create_task_log_exceptions
 
-from .messages import get_message_status, load_updated_message
+from .messages import get_message_status
 from .payment import (
     compute_required_balance,
     compute_required_flow,
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 72218f3ea..7985687fa 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -15,6 +15,7 @@
 from aleph.vm.conf import settings
 from aleph.vm.controllers.qemu.client import QemuVmClient
 from aleph.vm.models import VmExecution
+from aleph.vm.orchestrator import metrics
 from aleph.vm.orchestrator.custom_logs import set_vm_for_logging
 from aleph.vm.orchestrator.run import create_vm_execution_or_raise_http_error
 from aleph.vm.orchestrator.views.authentication import (
@@ -22,7 +23,12 @@
     require_jwk_authentication,
 )
 from aleph.vm.pool import VmPool
-from aleph.vm.utils import cors_allow_all, dumps_for_json
+from aleph.vm.utils import (
+    cors_allow_all,
+    dumps_for_json,
+    get_message_executable_content,
+)
+from aleph.vm.utils.logs import get_past_vm_logs
 
 logger = logging.getLogger(__name__)
 
@@ -99,22 +105,48 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
 
 @cors_allow_all
 @require_jwk_authentication
-async def operate_logs(request: web.Request, authenticated_sender: str) -> web.StreamResponse:
-    """Logs of a VM (not streaming)"""
+async def operate_logs_json(request: web.Request, authenticated_sender: str) -> web.StreamResponse:
+    """Logs of a VM (not streaming) as json"""
     vm_hash = get_itemhash_or_400(request.match_info)
     with set_vm_for_logging(vm_hash=vm_hash):
+        # This endpoint allow logs for past executions, so we look into the database if any execution by that hash
+        # occurred, which we can then use to look for rights. We still check in the pool first, it is faster
         pool: VmPool = request.app["vm_pool"]
-        execution = get_execution_or_404(vm_hash, pool=pool)
-        if not is_sender_authorized(authenticated_sender, execution.message):
+        execution = pool.executions.get(vm_hash)
+        if execution:
+            message = execution.message
+        else:
+            record = await metrics.get_last_record_for_vm(vm_hash=vm_hash)
+            if not record:
+                raise aiohttp.web_exceptions.HTTPNotFound(body="No execution found for this VM")
+            message = get_message_executable_content(json.loads(record.message))
+        if not is_sender_authorized(authenticated_sender, message):
             return web.Response(status=403, body="Unauthorized sender")
 
+        _journal_stdout_name = f"vm-{vm_hash}-stdout"
+        _journal_stderr_name = f"vm-{vm_hash}-stderr"
+
         response = web.StreamResponse()
-        response.headers["Content-Type"] = "text/plain"
+        response.headers["Transfer-encoding"] = "chunked"
+        response.headers["Content-Type"] = "application/json"
         await response.prepare(request)
+        await response.write(b"[")
+
+        first = True
+        for entry in get_past_vm_logs(_journal_stdout_name, _journal_stderr_name):
+            if not first:
+                await response.write(b",\n")
+            first = False
+            log_type = "stdout" if entry["SYSLOG_IDENTIFIER"] == _journal_stdout_name else "stderr"
+            msg = {
+                "SYSLOG_IDENTIFIER": entry["SYSLOG_IDENTIFIER"],
+                "MESSAGE": entry["MESSAGE"],
+                "file": log_type,
+                "__REALTIME_TIMESTAMP": entry["__REALTIME_TIMESTAMP"],
+            }
+            await response.write(dumps_for_json(msg).encode())
+        await response.write(b"]")
 
-        for entry in execution.vm.past_logs():
-            msg = f'{entry["__REALTIME_TIMESTAMP"].isoformat()}> {entry["MESSAGE"]}'
-            await response.write(msg.encode())
         await response.write_eof()
         return response
 
diff --git a/tests/supervisor/views/test_operator.py b/tests/supervisor/views/test_operator.py
index 86c6c5cd5..d5a643fe9 100644
--- a/tests/supervisor/views/test_operator.py
+++ b/tests/supervisor/views/test_operator.py
@@ -9,9 +9,10 @@
 import aiohttp
 import pytest
 from aiohttp.test_utils import TestClient
-from aleph_message.models import ItemHash
+from aleph_message.models import ItemHash, ProgramMessage
 
 from aleph.vm.conf import settings
+from aleph.vm.orchestrator.metrics import ExecutionRecord
 from aleph.vm.orchestrator.supervisor import setup_webapp
 from aleph.vm.pool import VmPool
 from aleph.vm.storage import get_message
@@ -303,103 +304,6 @@ class FakeVmPool:
     assert pool.systemd_manager.restart.call_count == 1
 
 
-@pytest.mark.asyncio
-async def test_logs(aiohttp_client, mocker):
-    mock_address = "mock_address"
-    mock_hash = "fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_"
-    mocker.patch(
-        "aleph.vm.orchestrator.views.authentication.authenticate_jwk",
-        return_value=mock_address,
-    )
-
-    # noinspection PyMissingConstructor
-    class FakeVmPool(VmPool):
-        def __init__(self):
-            pass
-
-        executions = {
-            mock_hash: mocker.Mock(
-                vm_hash=mock_hash,
-                message=mocker.Mock(address=mock_address),
-                is_confidential=False,
-                is_running=True,
-                vm=mocker.Mock(
-                    past_logs=mocker.Mock(
-                        return_value=[
-                            EntryDict(
-                                SYSLOG_IDENTIFIER="stdout",
-                                MESSAGE="logline1",
-                                __REALTIME_TIMESTAMP=datetime.datetime(2020, 10, 12, 1, 2),
-                            ),
-                            EntryDict(
-                                SYSLOG_IDENTIFIER="stdout",
-                                MESSAGE="logline2",
-                                __REALTIME_TIMESTAMP=datetime.datetime(2020, 10, 12, 1, 3),
-                            ),
-                        ]
-                    )
-                ),
-            ),
-        }
-        systemd_manager = mocker.Mock(restart=mocker.Mock())
-
-    app = setup_webapp()
-    pool = FakeVmPool()
-    app["vm_pool"] = pool
-    app["pubsub"] = FakeVmPool()
-    client = await aiohttp_client(app)
-    response = await client.get(
-        f"/control/machine/{mock_hash}/logs",
-    )
-    assert response.status == 200
-    assert await response.text() == "2020-10-12T01:02:00> logline12020-10-12T01:03:00> logline2"
-
-
-@pytest.mark.asyncio
-async def test_websocket_logs(aiohttp_client, mocker):
-    mock_address = "mock_address"
-    mock_hash = "fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_"
-    mocker.patch(
-        "aleph.vm.orchestrator.views.operator.authenticate_websocket_message",
-        return_value=mock_address,
-    )
-    fake_queue: Queue[tuple[str, str]] = asyncio.Queue()
-    await fake_queue.put(("stdout", "this is a first log entry"))
-
-    fakeVmPool = mocker.Mock(
-        executions={
-            mock_hash: mocker.Mock(
-                vm_hash=mock_hash,
-                message=mocker.Mock(address=mock_address),
-                is_confidential=False,
-                is_running=True,
-                vm=mocker.Mock(
-                    get_log_queue=mocker.Mock(return_value=fake_queue),
-                ),
-            ),
-        },
-    )
-    app = setup_webapp()
-    app["vm_pool"] = fakeVmPool
-    app["pubsub"] = None
-    client = await aiohttp_client(app)
-    websocket = await client.ws_connect(
-        f"/control/machine/{mock_hash}/stream_logs",
-    )
-    await websocket.send_json({"auth": "auth is disabled"})
-    response = await websocket.receive_json()
-    assert response == {"status": "connected"}
-
-    response = await websocket.receive_json()
-    assert response == {"message": "this is a first log entry", "type": "stdout"}
-
-    await fake_queue.put(("stdout", "this is a second log entry"))
-    response = await websocket.receive_json()
-    assert response == {"message": "this is a second log entry", "type": "stdout"}
-    await websocket.close()
-    assert websocket.closed
-
-
 @pytest.mark.asyncio
 async def test_websocket_logs_missing_auth(aiohttp_client, mocker):
     mock_address = "mock_address"
@@ -529,3 +433,110 @@ async def test_websocket_logs_good_auth(aiohttp_client, mocker, patch_datetime_n
 
     await websocket.close()
     assert websocket.closed
+
+
+@pytest.mark.asyncio
+async def test_get_past_logs(aiohttp_client, mocker, patch_datetime_now):
+    mock_address = "0x40684b43B88356F62DCc56017547B6A7AC68780B"
+    mock_hash = "fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_"
+    mocker.patch(
+        "aleph.vm.orchestrator.views.authentication.authenticate_jwk",
+        return_value=mock_address,
+    )
+    mocker.patch(
+        "aleph.vm.orchestrator.metrics.get_last_record_for_vm",
+        return_value=ExecutionRecord(
+            message="""{
+  "address": "0x40684b43B88356F62DCc56017547B6A7AC68780B",
+  "time": 1720816744.639107,
+  "allow_amend": false,
+  "metadata": null,
+  "authorized_keys": null,
+  "variables": null,
+  "environment": {
+    "reproducible": false,
+    "internet": true,
+    "aleph_api": true,
+    "shared_cache": false
+  },
+  "resources": {
+    "vcpus": 1,
+    "memory": 1024,
+    "seconds": 300,
+    "published_ports": null
+  },
+  "payment": null,
+  "requirements": null,
+  "volumes": [
+    {
+      "comment": null,
+      "mount": "/opt/packages",
+      "ref": "7338478721e2e966da6395dbfa37dab7b017b48da55b1be22d4eccf3487b836c",
+      "use_latest": true
+    }
+  ],
+  "replaces": null,
+  "type": "vm-function",
+  "code": {
+    "encoding": "squashfs",
+    "entrypoint": "main:app",
+    "ref": "c4253bf514d2e0a271456c9023c4b3f13f324e53c176e9ec29b98b5972b02bc7",
+    "interface": null,
+    "args": null,
+    "use_latest": true
+  },
+  "runtime": {
+    "ref": "63f07193e6ee9d207b7d1fcf8286f9aee34e6f12f101d2ec77c1229f92964696",
+    "use_latest": true,
+    "comment": ""
+  },
+  "data": null,
+  "export": null,
+  "on": {
+    "http": true,
+    "message": null,
+    "persistent": false
+  }
+}"""
+        ),
+    )
+    mocker.patch(
+        "aleph.vm.orchestrator.views.operator.get_past_vm_logs",
+        return_value=[
+            EntryDict(
+                SYSLOG_IDENTIFIER=f"vm-{mock_hash}-stdout",
+                MESSAGE="logline1",
+                __REALTIME_TIMESTAMP=datetime.datetime(2020, 10, 12, 1, 2),
+            ),
+            EntryDict(
+                SYSLOG_IDENTIFIER=f"vm-{mock_hash}-stderr",
+                MESSAGE="logline2",
+                __REALTIME_TIMESTAMP=datetime.datetime(2020, 10, 12, 1, 3),
+            ),
+        ],
+    )
+
+    app = setup_webapp()
+    pool = mocker.MagicMock(executions={})
+    app["vm_pool"] = pool
+    app["pubsub"] = mocker.MagicMock()
+    client = await aiohttp_client(app)
+    response = await client.get(
+        f"/control/machine/{mock_hash}/logs",
+    )
+
+    assert response.status == 200
+    assert await response.json() == [
+        {
+            "MESSAGE": "logline1",
+            "SYSLOG_IDENTIFIER": "vm-fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_-stdout",
+            "__REALTIME_TIMESTAMP": "2020-10-12 01:02:00",
+            "file": "stdout",
+        },
+        {
+            "MESSAGE": "logline2",
+            "SYSLOG_IDENTIFIER": "vm-fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_-stderr",
+            "__REALTIME_TIMESTAMP": "2020-10-12 01:03:00",
+            "file": "stderr",
+        },
+    ]

From 0241c044fa66e0bd0bee8b6c7cd34042aea82ae1 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Tue, 3 Dec 2024 14:21:10 +0100
Subject: [PATCH 903/990] Solve CI compiling issue (#727)

Fix: Solved CI issue creating the compiled package.
---
 packaging/Makefile                                   | 3 ++-
 pyproject.toml                                       | 2 ++
 runtimes/aleph-debian-12-python/create_disk_image.sh | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index cc217ce3f..3c4f8a6b1 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -15,7 +15,8 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.5.0' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2'
+	# Fixing this protobuf dependency version to avoid getting CI errors as version 5.29.0 have this compilation issue
+	pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.5.0' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' 'protobuf==5.28.3'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo  target/bin/sevctl
diff --git a/pyproject.toml b/pyproject.toml
index d33f05a04..faebfb9a4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,6 +45,8 @@ dependencies = [
   "msgpack==1.0.7",
   "nftables @ git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py",
   "packaging==23.2",
+  # Fixing this protobuf dependency version to avoid getting CI errors as version 5.29.0 have this compilation issue
+  "protobuf==5.28.3",
   "psutil==5.9.5",
   "py-cpuinfo==9",
   "pydantic[dotenv]~=1.10.13",
diff --git a/runtimes/aleph-debian-12-python/create_disk_image.sh b/runtimes/aleph-debian-12-python/create_disk_image.sh
index 10c8ae404..98fbb2766 100755
--- a/runtimes/aleph-debian-12-python/create_disk_image.sh
+++ b/runtimes/aleph-debian-12-python/create_disk_image.sh
@@ -36,7 +36,8 @@ locale-gen en_US.UTF-8
 
 echo "Pip installing aleph-sdk-python"
 mkdir -p /opt/aleph/libs
-pip3 install --target /opt/aleph/libs 'aleph-sdk-python==1.0.0' 'aleph-message==0.4.9' 'fastapi~=0.109.2'
+# Fixing this protobuf dependency version to avoid getting CI errors as version 5.29.0 have this compilation issue.
+pip3 install --target /opt/aleph/libs 'aleph-sdk-python==1.0.0' 'aleph-message==0.4.9' 'fastapi~=0.109.2' 'protobuf==5.28.3'
 
 # Compile Python code to bytecode for faster execution
 # -o2 is needed to compile with optimization level 2 which is what we launch init1.py ("python -OO")

From e93a461e4a6aaca9b466a256ccf3b24e1cd86fbe Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 12 Dec 2024 13:21:09 +0100
Subject: [PATCH 904/990] hatch fmt tests/

---
 tests/supervisor/test_authentication.py | 1 -
 tests/supervisor/test_instance.py       | 5 ++---
 tests/supervisor/test_qemu_instance.py  | 5 ++---
 tests/supervisor/test_views.py          | 2 +-
 tests/supervisor/views/test_operator.py | 8 ++------
 5 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/tests/supervisor/test_authentication.py b/tests/supervisor/test_authentication.py
index 73769c97c..d9848ad34 100644
--- a/tests/supervisor/test_authentication.py
+++ b/tests/supervisor/test_authentication.py
@@ -16,7 +16,6 @@
 )
 from aleph.vm.utils.test_helpers import (
     generate_signer_and_signed_headers_for_operation,
-    patch_datetime_now,
     to_0x_hex,
 )
 
diff --git a/tests/supervisor/test_instance.py b/tests/supervisor/test_instance.py
index 1319c99c4..1fc1f12ba 100644
--- a/tests/supervisor/test_instance.py
+++ b/tests/supervisor/test_instance.py
@@ -2,7 +2,6 @@
 import logging
 from asyncio.subprocess import Process
 from pathlib import Path
-from typing import Optional
 
 import pytest
 from aleph_message.models import ItemHash
@@ -21,8 +20,8 @@
 
 @pytest.mark.asyncio
 class MockSystemDManager(SystemDManager):
-    execution: Optional[MicroVM] = None
-    process: Optional[Process] = None
+    execution: MicroVM | None = None
+    process: Process | None = None
 
     async def enable_and_start(self, vm_hash: str):
         config_path = Path(f"{settings.EXECUTION_ROOT}/{vm_hash}-controller.json")
diff --git a/tests/supervisor/test_qemu_instance.py b/tests/supervisor/test_qemu_instance.py
index 6da59a625..56d4fc145 100644
--- a/tests/supervisor/test_qemu_instance.py
+++ b/tests/supervisor/test_qemu_instance.py
@@ -2,7 +2,6 @@
 import logging
 from asyncio.subprocess import Process
 from pathlib import Path
-from typing import Optional
 
 import pytest
 from aleph_message.models import ItemHash
@@ -21,8 +20,8 @@
 
 @pytest.mark.asyncio
 class MockSystemDManager(SystemDManager):
-    execution: Optional[QemuVM] = None
-    process: Optional[Process] = None
+    execution: QemuVM | None = None
+    process: Process | None = None
 
     async def enable_and_start(self, vm_hash: str):
         config_path = Path(f"{settings.EXECUTION_ROOT}/{vm_hash}-controller.json")
diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
index fff8b5492..cd32bdc7e 100644
--- a/tests/supervisor/test_views.py
+++ b/tests/supervisor/test_views.py
@@ -1,5 +1,5 @@
 import tempfile
-from pathlib import Path, PosixPath
+from pathlib import Path
 from unittest import mock
 from unittest.mock import call
 
diff --git a/tests/supervisor/views/test_operator.py b/tests/supervisor/views/test_operator.py
index d5a643fe9..09c88f334 100644
--- a/tests/supervisor/views/test_operator.py
+++ b/tests/supervisor/views/test_operator.py
@@ -9,18 +9,14 @@
 import aiohttp
 import pytest
 from aiohttp.test_utils import TestClient
-from aleph_message.models import ItemHash, ProgramMessage
+from aleph_message.models import ItemHash
 
 from aleph.vm.conf import settings
 from aleph.vm.orchestrator.metrics import ExecutionRecord
 from aleph.vm.orchestrator.supervisor import setup_webapp
-from aleph.vm.pool import VmPool
 from aleph.vm.storage import get_message
 from aleph.vm.utils.logs import EntryDict
-from aleph.vm.utils.test_helpers import (
-    generate_signer_and_signed_headers_for_operation,
-    patch_datetime_now,
-)
+from aleph.vm.utils.test_helpers import generate_signer_and_signed_headers_for_operation
 
 
 @pytest.mark.asyncio

From 1be3486dcb2856b79359f52ea91ca70e7ed4f85b Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 12 Dec 2024 14:09:08 +0100
Subject: [PATCH 905/990] Run ruff on code outside src/

---
 examples/example_django/manage.py        | 1 +
 runtimes/aleph-debian-12-python/init1.py | 1 +
 tests/supervisor/test_utils.py           | 2 --
 3 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/example_django/manage.py b/examples/example_django/manage.py
index 77dced937..bcc50390c 100755
--- a/examples/example_django/manage.py
+++ b/examples/example_django/manage.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 """Django's command-line utility for administrative tasks."""
+
 import os
 import sys
 
diff --git a/runtimes/aleph-debian-12-python/init1.py b/runtimes/aleph-debian-12-python/init1.py
index 11c4a7dd0..9f70bef64 100644
--- a/runtimes/aleph-debian-12-python/init1.py
+++ b/runtimes/aleph-debian-12-python/init1.py
@@ -558,6 +558,7 @@ async def main() -> None:
 
     class ServerReference:
         "Reference used to close the server from within `handle_instruction"
+
         server: asyncio.AbstractServer
 
     server_reference = ServerReference()
diff --git a/tests/supervisor/test_utils.py b/tests/supervisor/test_utils.py
index 0451d9607..51eef026a 100644
--- a/tests/supervisor/test_utils.py
+++ b/tests/supervisor/test_utils.py
@@ -9,7 +9,6 @@
 
 
 def test_check_system_module_enabled():
-
     with mock.patch(
         "pathlib.Path.exists",
         return_value=True,
@@ -19,7 +18,6 @@ def test_check_system_module_enabled():
             "aleph.vm.utils.Path.open",
             mock.mock_open(read_data=expected_value),
         ):
-
             output = check_system_module("kvm_amd/parameters/sev_enp")
             assert output == expected_value
 

From 9272b6f37c06ac93ae9300c9fe0d74fa36d4f5d3 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 12 Dec 2024 14:30:06 +0100
Subject: [PATCH 906/990] Make hatch linting use ruff format

Co-authored-by: Bram 
---
 pyproject.toml | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index faebfb9a4..3196f5fbb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -126,16 +126,18 @@ dependencies = [
 ]
 [tool.hatch.envs.linting.scripts]
 typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}"
+# Check
 style = [
-  #  "ruff {args:.}",
+  "ruff format --diff {args:.}",
   "black --check --diff {args:.}",
   "isort --check-only --profile black {args:.}",
   "yamlfix --check .",
   "pyproject-fmt --check pyproject.toml",
 ]
+# Do modification
 fmt = [
+  "ruff format {args:.}",
   "black {args:.}",
-  #  "ruff --fix {args:.}",
   "isort --profile black {args:.}",
   "yamlfix .",
   "pyproject-fmt pyproject.toml",
@@ -182,6 +184,8 @@ lint.select = [
   "YTT",
 ]
 lint.ignore = [
+  "ISC001",
+  # https://docs.astral.sh/ruff/rules/single-line-implicit-string-concatenation/#single-line-implicit-string-concatenation-isc001
   #  # Allow non-abstract empty methods in abstract base classes
   #  "B027",
   #  # Allow boolean positional values in function calls, like `dict.get(... True)`
@@ -193,17 +197,22 @@ lint.ignore = [
   # Allow the use of assert statements
   "S101",
 ]
+src = ["src"]
+
+
 #[tool.ruff.flake8-tidy-imports]
 #ban-relative-imports = "all"
 #unfixable = [
 #  # Don't touch unused imports
 #  "F401",
 #]
-#lint.isort = [ "aleph.vm" ]
 
 # Tests can use magic values, assertions, and relative imports
 lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ]
 
+[tool.isort]
+profile = "black"
+
 [tool.pytest.ini_options]
 pythonpath = [
   "src",

From e2f1cf98a3ef575bba54a0e4c5463ec6c5e4bf8d Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 12 Dec 2024 14:32:42 +0100
Subject: [PATCH 907/990] Make import sorting between ruff check and isort
 compatible

For some reason isort was detecting the "packaging" module as first party and not std
---
 pyproject.toml                              | 5 ++---
 src/aleph/vm/orchestrator/views/__init__.py | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 3196f5fbb..c68b701fa 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -156,6 +156,7 @@ line-length = 120
 [tool.ruff]
 target-version = "py310"
 line-length = 120
+src = [ "src" ]
 lint.select = [
   "A",
   "ARG",
@@ -197,9 +198,6 @@ lint.ignore = [
   # Allow the use of assert statements
   "S101",
 ]
-src = ["src"]
-
-
 #[tool.ruff.flake8-tidy-imports]
 #ban-relative-imports = "all"
 #unfixable = [
@@ -212,6 +210,7 @@ lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ]
 
 [tool.isort]
 profile = "black"
+extra_standard_library = [ "packaging" ]
 
 [tool.pytest.ini_options]
 pythonpath = [
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 4c9b3866d..94209c59f 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -4,6 +4,7 @@
 from decimal import Decimal
 from hashlib import sha256
 from json import JSONDecodeError
+from packaging.version import InvalidVersion, Version
 from pathlib import Path
 from secrets import compare_digest
 from string import Template
@@ -55,7 +56,6 @@
     get_ref_from_dns,
 )
 from aleph.vm.version import __version__
-from packaging.version import InvalidVersion, Version
 
 logger = logging.getLogger(__name__)
 

From e02dd0b3cefcade5136a0ee8266e4e27a413ce27 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 12 Dec 2024 14:43:50 +0100
Subject: [PATCH 908/990] formating

---
 src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py | 1 -
 src/aleph/vm/orchestrator/custom_logs.py             | 1 -
 src/aleph/vm/orchestrator/views/authentication.py    | 2 +-
 src/aleph/vm/orchestrator/views/operator.py          | 1 -
 4 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
index 5e32e8990..85ca63c1e 100644
--- a/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
@@ -12,7 +12,6 @@
 
 
 class QemuConfidentialVM(QemuVM):
-
     sev_policy: str = hex(AMDSEVPolicy.NO_DBG)
     sev_dh_cert_file: Path  # "vm_godh.b64"
     sev_session_file: Path  # "vm_session.b64"
diff --git a/src/aleph/vm/orchestrator/custom_logs.py b/src/aleph/vm/orchestrator/custom_logs.py
index 9150fdd73..c6a2a96e8 100644
--- a/src/aleph/vm/orchestrator/custom_logs.py
+++ b/src/aleph/vm/orchestrator/custom_logs.py
@@ -25,7 +25,6 @@ class InjectingFilter(logging.Filter):
     """
 
     def filter(self, record):
-
         vm_hash = ctx_current_execution_hash.get(None)
         if not vm_hash:
             vm_execution: VmExecution | None = ctx_current_execution.get(None)
diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py
index 419662072..55ed624ef 100644
--- a/src/aleph/vm/orchestrator/views/authentication.py
+++ b/src/aleph/vm/orchestrator/views/authentication.py
@@ -257,7 +257,7 @@ async def authenticate_websocket_message(message) -> str:
 
 
 def require_jwk_authentication(
-    handler: Callable[[web.Request, str], Coroutine[Any, Any, web.StreamResponse]]
+    handler: Callable[[web.Request, str], Coroutine[Any, Any, web.StreamResponse]],
 ) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
     """A decorator to enforce JWK-based authentication for HTTP requests.
 
diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index 7985687fa..fc7436d70 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -216,7 +216,6 @@ async def operate_confidential_initialize(request: web.Request, authenticated_se
     """Start the confidential virtual machine if possible."""
     vm_hash = get_itemhash_or_400(request.match_info)
     with set_vm_for_logging(vm_hash=vm_hash):
-
         pool: VmPool = request.app["vm_pool"]
         logger.debug(f"Iterating through running executions... {pool.executions}")
         execution = get_execution_or_404(vm_hash, pool=pool)

From 60375248bfdc461767602df545b8e4e1b2d48e30 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 12 Dec 2024 14:50:37 +0100
Subject: [PATCH 909/990] do not run black it is uneeded

---
 pyproject.toml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index c68b701fa..1fb2cf7ab 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -117,7 +117,6 @@ python = [ "3.10", "3.11", "3.12" ]
 [tool.hatch.envs.linting]
 detached = true
 dependencies = [
-  "black==24.3.0",
   "mypy==1.8.0",
   "ruff==0.4.6",
   "isort==5.13.2",
@@ -129,7 +128,6 @@ typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/alep
 # Check
 style = [
   "ruff format --diff {args:.}",
-  "black --check --diff {args:.}",
   "isort --check-only --profile black {args:.}",
   "yamlfix --check .",
   "pyproject-fmt --check pyproject.toml",
@@ -137,7 +135,6 @@ style = [
 # Do modification
 fmt = [
   "ruff format {args:.}",
-  "black {args:.}",
   "isort --profile black {args:.}",
   "yamlfix .",
   "pyproject-fmt pyproject.toml",

From 5a27c82ef0e1ecdf43f936a7acfa8f27a85c60cb Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 12 Dec 2024 16:11:14 +0100
Subject: [PATCH 910/990] Doc: Add section on formatting and linting

---
 README.md | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/README.md b/README.md
index 61f9271cc..7e21abcb7 100644
--- a/README.md
+++ b/README.md
@@ -113,6 +113,35 @@ Unless your focus is developing the VM-Connector, using the Docker image is easi
 ## Testing
 See [Testinc doc](./TESTING.md)
 
+## Code Formatting and Linting
+
+To help maintain a clean and consistent codebase, we provide automated tools for formatting and style checks.
+To ensure your code is properly **formatted** according to project standards, you can use:
+
+```bash
+hatch linting:fmt
+```
+
+**Typing** helps ensure your code adheres to expected type annotations, improving reliability and clarity. To validate
+typing in your code, use:
+```bash
+hatch linting:typing
+```
+
+These checks are also validated in Continuous Integration (CI) alongside unit tests. To ensure a smooth workflow, we 
+recommend running these commands before committing changes.
+
+**Linting** checks for potential errors, coding style violations, and patterns that may lead to bugs or reduce code
+quality (e.g., unused variables, incorrect imports, or inconsistent naming). While linting is not currently enforced in
+Continuous Integration (CI), it is considered a best practice to check linting manually to maintain high-quality code.
+You can manually lint your code by running:
+
+```bash
+hatch fmt
+```
+
+Following these best practices can help streamline code reviews and improve overall project quality.
+
 # Architecture
 
 ![Aleph im VM - Details](https://user-images.githubusercontent.com/404665/127126908-3225a633-2c36-4129-8766-9810f2fcd7d6.png)

From db356afd69da99b21f0861dae49e27da3efe1bee Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 12 Dec 2024 16:31:55 +0100
Subject: [PATCH 911/990] fix import removed by error

---
 tests/supervisor/test_authentication.py | 4 ++++
 tests/supervisor/views/test_operator.py | 8 +++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tests/supervisor/test_authentication.py b/tests/supervisor/test_authentication.py
index d9848ad34..b46dd315f 100644
--- a/tests/supervisor/test_authentication.py
+++ b/tests/supervisor/test_authentication.py
@@ -16,9 +16,13 @@
 )
 from aleph.vm.utils.test_helpers import (
     generate_signer_and_signed_headers_for_operation,
+    patch_datetime_now,
     to_0x_hex,
 )
 
+# Ensure this is not removed by ruff
+assert patch_datetime_now
+
 
 @pytest.mark.asyncio
 async def test_require_jwk_authentication_missing_header(aiohttp_client):
diff --git a/tests/supervisor/views/test_operator.py b/tests/supervisor/views/test_operator.py
index 09c88f334..8a6c70485 100644
--- a/tests/supervisor/views/test_operator.py
+++ b/tests/supervisor/views/test_operator.py
@@ -16,7 +16,13 @@
 from aleph.vm.orchestrator.supervisor import setup_webapp
 from aleph.vm.storage import get_message
 from aleph.vm.utils.logs import EntryDict
-from aleph.vm.utils.test_helpers import generate_signer_and_signed_headers_for_operation
+from aleph.vm.utils.test_helpers import (
+    generate_signer_and_signed_headers_for_operation,
+    patch_datetime_now,
+)
+
+# Ensure this is not removed by ruff
+assert patch_datetime_now
 
 
 @pytest.mark.asyncio

From c37c7c6658c835442e49e12781feb579f69049f1 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 16 Dec 2024 11:32:07 +0100
Subject: [PATCH 912/990] Fix error Deleting network interface

JIRA: ALEPH-115

**Problem**:
When reusing an existing stale network interface, the following error would occasionally occur:

```
Unknown exception while deleting address 2a01:4f8:171:787:1:63fa:f8b5:db10/124 to interface vmtap...
```

See details in Sentry: [https://alephim.sentry.io/issues/5993120643/?project=4506303231819776](https://alephim.sentry.io/issues/5993120643/?project=4506303231819776)

**Analysis**:
This error occurred during the process of cleaning up an old interface (that was not deleted after usage for various possible reasons). The code was attempting to delete the IP addresses associated with this interface (as calculated in the code). However, when these IP addresses did not exist, an error was raised.

**Solution**:
Our initial approach was to use the `IPRoute` module to list the IP addresses associated with the network interface and then delete them instead of calculating their names in the Python code as is done presently.

After experimentation, we determined that deleting the interface directly also removes the associated IP addresses. Therefore, it is unnecessary to delete them manually, which simplifies the code.

**To test**:
Create and stop Program
Create network interface
```bash
sudo ip tuntap add dev vmtap4 mode tap
```
and attach ip to them
```bash
sudo ip addr add 1.1.1.1/30 dev  vmtap4
```

Try different combinaison and validate that everything still work
correctly

To check the networks interfaces use the following command to list them
```bash
sudo ip link
```

and to list the ip address use:
```
sudo ip addr
```
---
 src/aleph/vm/network/interfaces.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index f401b507f..3d093d542 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -170,6 +170,4 @@ async def delete(self) -> None:
         if self.ndp_proxy:
             await self.ndp_proxy.delete_range(self.device_name)
         with IPRoute() as ipr:
-            delete_ip_address(ipr, self.device_name, self.host_ip)
-            delete_ip_address(ipr, self.device_name, self.host_ipv6)
             delete_tap_interface(ipr, self.device_name)

From 3b9ac94d4e56d983f29d07ec9638df639c2a96da Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 16 Dec 2024 11:58:30 +0100
Subject: [PATCH 913/990] Remove unused  delete_ip_address()

mainly to improve code coverage
---
 src/aleph/vm/network/interfaces.py | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index 3d093d542..5f0120725 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -61,20 +61,6 @@ def add_ip_address(ipr: IPRoute, device_name: str, ip: IPv4Interface | IPv6Inter
         logger.error(f"Unknown exception while adding address {ip} to interface {device_name}: {e}")
 
 
-def delete_ip_address(ipr: IPRoute, device_name: str, ip: IPv4Interface | IPv6Interface):
-    """Delete an IP address to the given interface."""
-    interface_index: list[int] = ipr.link_lookup(ifname=device_name)
-    if not interface_index:
-        msg = f"Interface {device_name} does not exist, can't delete address {ip} to it."
-        raise MissingInterfaceError(msg)
-    try:
-        ipr.addr("del", index=interface_index[0], address=str(ip.ip), mask=ip.network.prefixlen)
-    except NetlinkError as e:
-        logger.exception(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
-    except OSError as e:
-        logger.exception(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
-
-
 def set_link_up(ipr: IPRoute, device_name: str):
     """Set the given interface up."""
     interface_index: list[int] = ipr.link_lookup(ifname=device_name)

From 2ef3b893e0632965daa7ff1475b9c83557d77cde Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 16 Dec 2024 13:56:26 +0100
Subject: [PATCH 914/990] List all ip on interface and delete them

---
 src/aleph/vm/network/interfaces.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/src/aleph/vm/network/interfaces.py b/src/aleph/vm/network/interfaces.py
index 5f0120725..8c40f5eaa 100644
--- a/src/aleph/vm/network/interfaces.py
+++ b/src/aleph/vm/network/interfaces.py
@@ -61,6 +61,20 @@ def add_ip_address(ipr: IPRoute, device_name: str, ip: IPv4Interface | IPv6Inter
         logger.error(f"Unknown exception while adding address {ip} to interface {device_name}: {e}")
 
 
+def delete_ip_address(ipr: IPRoute, device_name: str, ip: str, mask: int):
+    """Delete an IP address to the given interface."""
+    interface_index: list[int] = ipr.link_lookup(ifname=device_name)
+    if not interface_index:
+        msg = f"Interface {device_name} does not exist, can't delete address {ip} to it."
+        raise MissingInterfaceError(msg)
+    try:
+        ipr.addr("del", index=interface_index[0], address=ip, mask=mask)
+    except NetlinkError as e:
+        logger.exception(f"Unknown exception while deleting address {ip}/{mask} to interface {device_name}: {e}")
+    except OSError as e:
+        logger.exception(f"Unknown exception while deleting address {ip}/{mask} to interface {device_name}: {e}")
+
+
 def set_link_up(ipr: IPRoute, device_name: str):
     """Set the given interface up."""
     interface_index: list[int] = ipr.link_lookup(ifname=device_name)
@@ -156,4 +170,11 @@ async def delete(self) -> None:
         if self.ndp_proxy:
             await self.ndp_proxy.delete_range(self.device_name)
         with IPRoute() as ipr:
+            interface_index: list[int] = ipr.link_lookup(ifname=self.device_name)
+            for addr in ipr.get_addr(index=interface_index):
+                # The order of attributes in the attrs field comes from the Netlink protocol
+                attrs = dict(addr["attrs"])
+                ip_addr: str = attrs["IFA_ADDRESS"]
+                mask: int = addr["prefixlen"]
+                delete_ip_address(ipr, self.device_name, ip_addr, mask)
             delete_tap_interface(ipr, self.device_name)

From 2f93e70cea031621bb073c4321d6267128d971d3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Dec 2024 13:08:15 +0000
Subject: [PATCH 915/990] Chore(deps): Bump aiohttp from 3.9.5 to 3.10.11

---
updated-dependencies:
- dependency-name: aiohttp
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 1fb2cf7ab..48542a5a8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,7 +31,7 @@ dynamic = [ "version" ]
 # Upon adding or updating dependencies, update `packaging/Makefile` for the Debian package
 dependencies = [
   "aiodns==3.1",
-  "aiohttp==3.9.5",
+  "aiohttp==3.10.11",
   "aiohttp-cors~=0.7.0",
   "aioredis==1.3.1",
   "aiosqlite==0.19",

From 57cb485885a6a5f02112b4600a2932483ec3370b Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Fri, 20 Dec 2024 15:01:43 +0100
Subject: [PATCH 916/990] Problem Deleted Instance not stopping -
 monitor_payment
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Make monitory_payment never stop

= Problem
Reported by user “Roby” on Telegram with his own CRN and VMs.
PAYG Instance he had deleted from the web frontend never stopped
running.

= Analysis
After some investigation and dicussion with the user, the probable cause is that the monitor payment task has crashed and stopped. Thus VM didn’t get stopped anymore this way. (when forgotten or failing payment)

Upon investigation there is no error handling over the whole task thus in case of uncatched exception the whole task stopped.

= Solution
Wrap the monitor_payment task around a try_catch so it never stop
running
---
 src/aleph/vm/orchestrator/tasks.py | 108 +++++++++++++++++------------
 1 file changed, 64 insertions(+), 44 deletions(-)

diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 921a2265f..bd89a8816 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -143,59 +143,79 @@ async def stop_watch_for_messages_task(app: web.Application):
 
 
 async def monitor_payments(app: web.Application):
-    logger.debug("Monitoring balances")
+    """Periodically checks and stops VMs if payment conditions are unmet, such as insufficient
+    wallet balance or payment stream coverage. Handles forgotten VMs, balance checks for the
+    "hold" tier, and stream flow validation for the "superfluid" tier to ensure compliance.
+    """
     pool: VmPool = app["vm_pool"]
     while True:
         await asyncio.sleep(settings.PAYMENT_MONITOR_INTERVAL)
-
-        # Check if the executions continues existing or are forgotten before checking the payment
-        for vm_hash in list(pool.executions.keys()):
-            message_status = await get_message_status(vm_hash)
-            if message_status != MessageStatus.PROCESSED:
-                logger.debug(f"Stopping {vm_hash} execution due to {message_status} message status")
-                await pool.stop_vm(vm_hash)
-                pool.forget_vm(vm_hash)
-
-        # Check if the balance held in the wallet is sufficient holder tier resources (Not do it yet)
-        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.hold).items():
-            for chain, executions in chains.items():
-                executions = [execution for execution in executions if execution.is_confidential]
-                balance = await fetch_balance_of_address(sender)
-
-                # Stop executions until the required balance is reached
+        # noinspection PyBroadException
+        try:
+            logger.debug("Monitoring balances task running")
+            await check_payment(pool)
+        except Exception as e:
+            # Catch all exceptions as to never stop the task.
+            logger.warning(f"check_payment failed {e}", exc_info=True)
+
+
+async def check_payment(pool: VmPool):
+    """Ensures VMs are stopped if payment conditions are unmet, such as insufficient
+    funds in the wallet or inadequate payment stream coverage. Handles forgotten VMs
+    balance checks for the "hold" tier, and stream flow validation for the "superfluid" tier
+    stopping executions as needed to maintain compliance.
+    """
+    # Check if the executions continues existing or are forgotten before checking the payment
+    # this is actually the main workflow for properly stopping PAYG instances, a user agent would stop the payment stream
+    # and forget the instance message. Compared to just stopping or decreasing the payment stream as the CRN don't know
+    # which VM it affects.
+    for vm_hash in list(pool.executions.keys()):
+        message_status = await get_message_status(vm_hash)
+        if message_status != MessageStatus.PROCESSED:
+            logger.debug(f"Stopping {vm_hash} execution due to {message_status} message status")
+            await pool.stop_vm(vm_hash)
+            pool.forget_vm(vm_hash)
+
+    # Check if the balance held in the wallet is sufficient holder tier resources (Not do it yet)
+    for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.hold).items():
+        for chain, executions in chains.items():
+            executions = [execution for execution in executions if execution.is_confidential]
+            balance = await fetch_balance_of_address(sender)
+
+            # Stop executions until the required balance is reached
+            required_balance = await compute_required_balance(executions)
+            logger.debug(f"Required balance for Sender {sender} executions: {required_balance}")
+            # Stop executions until the required balance is reached
+            while executions and balance < (required_balance + settings.PAYMENT_BUFFER):
+                last_execution = executions.pop(-1)
+                logger.debug(f"Stopping {last_execution} due to insufficient balance")
+                await pool.stop_vm(last_execution.vm_hash)
                 required_balance = await compute_required_balance(executions)
-                logger.debug(f"Required balance for Sender {sender} executions: {required_balance}")
-                # Stop executions until the required balance is reached
-                while executions and balance < (required_balance + settings.PAYMENT_BUFFER):
-                    last_execution = executions.pop(-1)
-                    logger.debug(f"Stopping {last_execution} due to insufficient balance")
-                    await pool.stop_vm(last_execution.vm_hash)
-                    required_balance = await compute_required_balance(executions)
-
-        # Check if the balance held in the wallet is sufficient stream tier resources
-        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():
-            for chain, executions in chains.items():
-                stream = await get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
-                logger.debug(
-                    f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
-                )
 
+    # Check if the balance held in the wallet is sufficient stream tier resources
+    for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():
+        for chain, executions in chains.items():
+            stream = await get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
+            logger.debug(
+                f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
+            )
+
+            required_stream = await compute_required_flow(executions)
+            logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
+            # Stop executions until the required stream is reached
+            while (stream + settings.PAYMENT_BUFFER) < required_stream:
+                try:
+                    last_execution = executions.pop(-1)
+                except IndexError:  # Empty list
+                    logger.debug("No execution can be maintained due to insufficient stream")
+                    break
+                logger.debug(f"Stopping {last_execution} due to insufficient stream")
+                await pool.stop_vm(last_execution.vm_hash)
                 required_stream = await compute_required_flow(executions)
-                logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
-                # Stop executions until the required stream is reached
-                while (stream + settings.PAYMENT_BUFFER) < required_stream:
-                    try:
-                        last_execution = executions.pop(-1)
-                    except IndexError:  # Empty list
-                        logger.debug("No execution can be maintained due to insufficient stream")
-                        break
-                    logger.debug(f"Stopping {last_execution} due to insufficient stream")
-                    await pool.stop_vm(last_execution.vm_hash)
-                    required_stream = await compute_required_flow(executions)
 
 
 async def start_payment_monitoring_task(app: web.Application):
-    app["payments_monitor"] = create_task_log_exceptions(monitor_payments(app))
+    app["payments_monitor"] = create_task_log_exceptions(monitor_payments(app), name="payment_monitor")
 
 
 async def stop_balances_monitoring_task(app: web.Application):

From 5f5b708e4d9c0908c5fb2a3ab47f8ba86cbf07b9 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 20 Nov 2024 15:19:50 +0100
Subject: [PATCH 917/990] Add test for normal case and problem case

---
 tests/supervisor/test_execution.py | 195 +++++++++++++++++++++++++++++
 1 file changed, 195 insertions(+)

diff --git a/tests/supervisor/test_execution.py b/tests/supervisor/test_execution.py
index 40c6fd71b..3df9ef4a6 100644
--- a/tests/supervisor/test_execution.py
+++ b/tests/supervisor/test_execution.py
@@ -1,7 +1,10 @@
 import asyncio
+import json
 import logging
+from typing import Any
 
 import pytest
+from aleph_message import parse_message
 from aleph_message.models import ItemHash
 
 from aleph.vm.conf import Settings, settings
@@ -10,6 +13,7 @@
 from aleph.vm.orchestrator import metrics
 from aleph.vm.orchestrator.messages import load_updated_message
 from aleph.vm.storage import get_message
+from aleph.vm.utils import fix_message_validation
 
 
 @pytest.mark.asyncio
@@ -110,6 +114,197 @@ async def test_create_execution_online(vm_hash: ItemHash = None):
     await execution.stop()
 
 
+@pytest.fixture()
+def fake_message():
+    fake = {
+        "sender": "0x101d8D16372dBf5f1614adaE95Ee5CCE61998Fc9",
+        "chain": "ETH",
+        "signature": "0x12592841210ef84888315d12b9c39225b8ba6b958b067790540a7971a95e8d4e6ce81deeb8e1f05f6141d8d62218641be1aa9b335463cdc5a43354205d4c9e351c",
+        "type": "PROGRAM",
+        "item_type": "inline",
+        "item_hash": "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace",
+        "time": "2024-04-23T12:10:41.801703+00:00",
+        "channel": None,
+        "content": {
+            "address": "0x101d8D16372dBf5f1614adaE95Ee5CCE61998Fc9",
+            "time": 1713874241.800818,
+            "allow_amend": False,
+            "metadata": None,
+            "authorized_keys": None,
+            "variables": None,
+            "environment": {"reproducible": False, "internet": True, "aleph_api": True, "shared_cache": False},
+            "resources": {"vcpus": 1, "memory": 256, "seconds": 30, "published_ports": None},
+            "payment": None,
+            "requirements": None,
+            "volumes": [
+                {
+                    "comment": "Persistence",
+                    "mount": "/var/lib/example",
+                    "parent": None,
+                    "persistence": "host",
+                    "name": "increment-storage",
+                    "size_mib": 1,
+                },
+            ],
+            "replaces": None,
+            "type": "vm-function",
+            "code": {
+                "encoding": "zip",
+                "entrypoint": "main:app",
+                "ref": "79f19811f8e843f37ff7535f634b89504da3d8f03e1f0af109d1791cf6add7af",
+                "interface": None,
+                "args": None,
+                "use_latest": True,
+            },
+            "runtime": {
+                "ref": "63f07193e6ee9d207b7d1fcf8286f9aee34e6f12f101d2ec77c1229f92964696",
+                "use_latest": True,
+                "comment": "",
+            },
+            "data": None,
+            "export": None,
+            "on": {"http": True, "message": None, "persistent": False},
+        },
+        "confirmed": True,
+        "confirmations": [
+            {
+                "chain": "ETH",
+                "height": 19718321,
+                "hash": "0x4b8f9f232602ef8ca9bf0ba4fd907f1feef2bfc865a32b2c51fa40b72fa5ba49",
+            }
+        ],
+    }
+
+    return fake
+
+
+def drop_none_recursively(data: dict) -> dict:
+    """
+    Recursively removes keys with None values from a dictionary.
+
+    """
+    if not isinstance(data, dict):
+        return data  # Base case: if not a dictionary, return as-is.
+
+    cleaned_dict: dict[Any, Any] = {}
+
+    for key, value in data.items():
+        if value is None:
+            continue  # Skip keys with None values.
+        elif isinstance(value, dict):
+            # Recur for nested dictionaries.
+            nested_cleaned = drop_none_recursively(value)
+            if nested_cleaned:  # Include only if not empty.
+                cleaned_dict[key] = nested_cleaned
+        elif isinstance(value, list):
+            # Recur for dictionaries within lists.
+            cleaned_list = [drop_none_recursively(item) if isinstance(item, dict) else item for item in value]
+            cleaned_dict[key] = [item for item in cleaned_list if item]
+        else:
+            cleaned_dict[key] = value  # Keep other values.
+
+    return cleaned_dict
+
+
+@pytest.mark.asyncio
+async def test_create_execution_from_fake_message(fake_message):
+    # Ensure that the settings are correct and required files present.
+    settings.setup()
+    settings.check()
+
+    # The database is required for the metrics and is currently not optional.
+    engine = metrics.setup_engine()
+    await metrics.create_tables(engine)
+
+    vm_hash = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")
+
+    fake_message = drop_none_recursively(fake_message)
+    fix_message_validation(fake_message)
+
+    parsed_fake_message = parse_message(message_dict=fake_message)
+
+    message, original_message = parsed_fake_message, parsed_fake_message
+
+    execution = VmExecution(
+        vm_hash=vm_hash,
+        message=message.content,
+        original=original_message.content,
+        snapshot_manager=None,
+        systemd_manager=None,
+        persistent=False,
+    )
+
+    # Downloading the resources required may take some time, limit it to 120 seconds
+    # since it is a bit slow in GitHub Actions
+    await asyncio.wait_for(execution.prepare(), timeout=120)
+
+    vm = execution.create(vm_id=3, tap_interface=None)
+
+    # Test that the VM is created correctly. It is not started yet.
+    assert isinstance(vm, AlephFirecrackerProgram)
+    vm.enable_console = True
+    vm.fvm.enable_log = True
+    assert vm.vm_id == 3
+
+    await execution.start()
+    await execution.stop()
+
+
+@pytest.xfail("to fix")
+@pytest.mark.asyncio
+async def test_create_execution_volume_with_no_name(fake_message):
+    """Regression test for ALEPH-307: VM init fail if volume name is empty string"""
+
+    vm_hash = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")
+
+    # Ensure that the settings are correct and required files present.
+    settings.setup()
+    settings.check()
+
+    # The database is required for the metrics and is currently not optional.
+    engine = metrics.setup_engine()
+    await metrics.create_tables(engine)
+    volume_with_no_name = {
+        "comment": "Persistence with no name",
+        "mount": "/var/lib/example",
+        "parent": None,
+        "persistence": "host",
+        "name": "",
+        "size_mib": 1,
+    }
+    fake_message["content"]["volumes"] = [volume_with_no_name]
+    fake_message = drop_none_recursively(fake_message)
+    fix_message_validation(fake_message)
+
+    parsed_fake_message = parse_message(message_dict=fake_message)
+
+    message, original_message = parsed_fake_message, parsed_fake_message
+
+    execution = VmExecution(
+        vm_hash=vm_hash,
+        message=message.content,
+        original=original_message.content,
+        snapshot_manager=None,
+        systemd_manager=None,
+        persistent=False,
+    )
+
+    # Downloading the resources required may take some time, limit it to 120 seconds
+    # since it is a bit slow in GitHub Actions
+    await asyncio.wait_for(execution.prepare(), timeout=120)
+
+    vm = execution.create(vm_id=3, tap_interface=None)
+
+    # Test that the VM is created correctly. It is not started yet.
+    assert isinstance(vm, AlephFirecrackerProgram)
+    vm.enable_console = True
+    vm.fvm.enable_log = True
+    assert vm.vm_id == 3
+
+    await execution.start()
+    await execution.stop()
+
+
 # This test depends on having a vm-connector running on port 4021
 @pytest.mark.asyncio
 async def test_create_execution_legacy():

From bbfcda61448623d89ab402ae190efc7365d74e1a Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 20 Nov 2024 15:43:33 +0100
Subject: [PATCH 918/990] ALEPH-307 VM init failed for persistant volume with
 empty string name

Solution : Add a randomly generated name
---
 src/aleph/vm/storage.py            | 5 ++++-
 tests/supervisor/test_execution.py | 1 -
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index 7e289dca2..ebc4c7c27 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -10,6 +10,7 @@
 import logging
 import re
 import sys
+import uuid
 from datetime import datetime, timezone
 from pathlib import Path
 from shutil import copy2, make_archive
@@ -371,11 +372,13 @@ async def get_volume_path(volume: MachineVolume, namespace: str) -> Path:
         return await get_existing_file(ref)
     elif isinstance(volume, PersistentVolume | RootfsVolume):
         volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
+        if not volume.name:
+            volume_name = f"unamed_{uuid.uuid4().hex}"
         if volume.persistence != VolumePersistence.host:
             msg = "Only 'host' persistence is supported"
             raise NotImplementedError(msg)
         if not re.match(r"^[\w\-_/]+$", volume_name):
-            msg = f"Invalid value for volume name: {volume_name}"
+            msg = f"Invalid value for volume name: {repr(volume_name)}"
             raise ValueError(msg)
         (Path(settings.PERSISTENT_VOLUMES_DIR) / namespace).mkdir(exist_ok=True)
         if volume.parent:
diff --git a/tests/supervisor/test_execution.py b/tests/supervisor/test_execution.py
index 3df9ef4a6..f9a823f83 100644
--- a/tests/supervisor/test_execution.py
+++ b/tests/supervisor/test_execution.py
@@ -250,7 +250,6 @@ async def test_create_execution_from_fake_message(fake_message):
     await execution.stop()
 
 
-@pytest.xfail("to fix")
 @pytest.mark.asyncio
 async def test_create_execution_volume_with_no_name(fake_message):
     """Regression test for ALEPH-307: VM init fail if volume name is empty string"""

From 175f49b46edab19a08c467ec73064c34f7bc72a8 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 12 Dec 2024 08:58:14 +0100
Subject: [PATCH 919/990] Fix use case where Volume has no mount point either

---
 src/aleph/vm/controllers/firecracker/executable.py |  6 +++++-
 src/aleph/vm/storage.py                            |  2 --
 tests/supervisor/test_execution.py                 | 10 +++++++++-
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index cbbad03c6..7d33d0521 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -114,7 +114,11 @@ async def download_kernel(self):
     async def download_volumes(self):
         volumes = []
         # TODO: Download in parallel
-        for volume in self.message_content.volumes:
+        for i, volume in enumerate(self.message_content.volumes):
+            if not volume.name:
+                volume.name = f"unamed_volume_{i}"
+            if not volume.mount:
+                volume.mount = f"/mnt/{volume.name}"
             volumes.append(
                 HostVolume(
                     mount=volume.mount,
diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index ebc4c7c27..df0505ca7 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -372,8 +372,6 @@ async def get_volume_path(volume: MachineVolume, namespace: str) -> Path:
         return await get_existing_file(ref)
     elif isinstance(volume, PersistentVolume | RootfsVolume):
         volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
-        if not volume.name:
-            volume_name = f"unamed_{uuid.uuid4().hex}"
         if volume.persistence != VolumePersistence.host:
             msg = "Only 'host' persistence is supported"
             raise NotImplementedError(msg)
diff --git a/tests/supervisor/test_execution.py b/tests/supervisor/test_execution.py
index f9a823f83..b064a084a 100644
--- a/tests/supervisor/test_execution.py
+++ b/tests/supervisor/test_execution.py
@@ -271,7 +271,15 @@ async def test_create_execution_volume_with_no_name(fake_message):
         "name": "",
         "size_mib": 1,
     }
-    fake_message["content"]["volumes"] = [volume_with_no_name]
+    volume_with_no_mount = {
+        "comment": "Persistence with no mount name",
+        "mount": "",
+        "parent": None,
+        "persistence": "host",
+        "name": "",
+        "size_mib": 1,
+    }
+    fake_message["content"]["volumes"] = [volume_with_no_name, volume_with_no_mount]
     fake_message = drop_none_recursively(fake_message)
     fix_message_validation(fake_message)
 

From bcdf0c083e25a4000925ebd2deb8e11299c6400e Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 17 Dec 2024 14:09:02 +0100
Subject: [PATCH 920/990] fix tests

---
 src/aleph/vm/controllers/firecracker/executable.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index 7d33d0521..c6d6721fa 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -13,6 +13,7 @@
 from aiohttp import ClientResponseError
 from aleph_message.models import ExecutableContent, ItemHash
 from aleph_message.models.execution.environment import MachineResources
+from aleph_message.models.execution.volume import PersistentVolume
 
 from aleph.vm.conf import settings
 from aleph.vm.controllers.configuration import (
@@ -115,10 +116,12 @@ async def download_volumes(self):
         volumes = []
         # TODO: Download in parallel
         for i, volume in enumerate(self.message_content.volumes):
-            if not volume.name:
-                volume.name = f"unamed_volume_{i}"
-            if not volume.mount:
-                volume.mount = f"/mnt/{volume.name}"
+            # only persistant volume has name and mount
+            if isinstance(volume, PersistentVolume):
+                if not volume.name:
+                    volume.name = f"unamed_volume_{i}"
+                if not volume.mount:
+                    volume.mount = f"/mnt/{volume.name}"
             volumes.append(
                 HostVolume(
                     mount=volume.mount,

From df98ec65a0a7da2e221ace81dd77e91ffd677991 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 19 Dec 2024 09:56:21 +0100
Subject: [PATCH 921/990] Fix Qemu hang silently on failed boot

Ticket: JIRA-344

Problem:
When QEMU was failing to boot the hard drive file image provided by the user, for example we have cases of user  using an ext4 image for firefracker
instead of a qemu disk image (this was facilitated by an oversight in the typescript sdk), the qemu process and hence the controller would hang indefinetly
without showing an error message.

Analysis

1. the Boot process was not part of the logs or the process output. (even inside the server) which is part of what was making it hard to debug.
2. QEMU try to boot via the network even if it is useless
3. After failing all boot method the qemu process and thus the controller  is still running indefinitely

Solution:
Change the option for qemu
-nographics make it output the boot process on the standard output (and thus the logs)
-boot order=c only boot the first hard drive (not sure if this actually
work)
-boot reboot-timeout=1 make it reboot if if fail to boot, but since we have -no-reboot the process just stop (default is -1 no reboot)
---
 src/aleph/vm/hypervisors/qemu/qemuvm.py              |  7 +++++++
 src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py | 12 +++++++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index 5949fbdc4..5bcb1313c 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -102,6 +102,13 @@ async def start(
             # Tell to put the output to std fd, so we can include them in the log
             "-serial",
             "stdio",
+            # nographics. Seems redundant with -serial stdio but without it the boot process is not displayed on stdout
+            "-nographic",
+            # Boot
+            # order=c only first hard drive
+            # reboot-timeout in combination with -no-reboot, makes it so qemu stop if there is no bootable device
+            "-boot",
+            "order=c,reboot-timeout=1",
             # Uncomment for debug
             # "-serial", "telnet:localhost:4321,server,nowait",
             # "-snapshot",  # Do not save anything to disk
diff --git a/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
index 85ca63c1e..89e9c3e80 100644
--- a/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
@@ -87,12 +87,18 @@ async def start(
             "-qmp",
             f"unix:{self.qmp_socket_path},server,nowait",
             # Tell to put the output to std fd, so we can include them in the log
-            "-nographic",
             "-serial",
             "stdio",
-            "--no-reboot",  # Rebooting from inside the VM shuts down the machine
-            "-S",
+            # nographics. Seems redundant with -serial stdio but without it the boot process is not displayed on stdout
+            "-nographic",
+            # Boot
+            # order=c only first hard drive
+            # reboot-timeout in combination with -no-reboot, makes it so qemu stop if there is no bootable device
+            "-boot",
+            "order=c,reboot-timeout=1",
             # Confidential options
+            # Do not start CPU at startup, we will start it via QMP after injecting the secret
+            "-S",
             "-object",
             f"sev-guest,id=sev0,policy={self.sev_policy},cbitpos={sev_info.c_bit_position},"
             f"reduced-phys-bits={sev_info.phys_addr_reduction},"

From c2ad82a9b7cbe2d6de700b827d11e4cc03b4eab1 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 19 Dec 2024 10:01:16 +0100
Subject: [PATCH 922/990] Problem Default log level in controler was not set

---
 src/aleph/vm/controllers/__main__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py
index 519270b48..f3cef3171 100644
--- a/src/aleph/vm/controllers/__main__.py
+++ b/src/aleph/vm/controllers/__main__.py
@@ -53,6 +53,7 @@ def parse_args(args):
         help="set loglevel to DEBUG",
         action="store_const",
         const=logging.DEBUG,
+        default=logging.INFO,
     )
     return parser.parse_args(args)
 

From c10f675b8f9024d136c2d6dbef7e2029d5cb80e2 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Wed, 8 Jan 2025 10:19:56 +0100
Subject: [PATCH 923/990] Fix VM logs websocket endpoint getting stuck

Jira ticket: ALEPH-316

At some point the websocket endpoint stopped sending log entries to client

Original report from Arnaud:
"aleph instance logs" output gets stuck
VM started and is accessible, but logs seem frozen when querying them via CLI.

Analysis:
The log entries are in journald and retrievable but they don't get put at on the internal queue. My guess is that there is a lock bug between the journal polling and the queue max length, probably that if the queue is full and get emptied, the polling doesn't reset unless new message arrive in journald.

Solution:
Rework the journald to Queue implementation.

Note:
I have tried a lot of combinaison and variation of the logic and rewrote that code block multiple times, this version works as far as I can tell.

The interaction between asyncio select handling, journald fd reset and async task is complicated and confusing.

Other versions broke in some way or others, sometimes subtle. For examples:
- Past logs working but new log entries not getting consumed
- Queue (re)filling starting only when a new entrie was created
- infinite tasks always popping up
- multiple task consuming the same fd
- Polling never resetting and consumer being called in a loop

Some of theses issues get hidden by the TCP buffer on the websocket and only popup after some time or depending on the network or how the client interact.

So beware if you try to rewrite that logic.
Also DO NOT TRUST ChatGPT or Deepseek on this, they will produce nice looking code that do not works properly.

To test:
Start an instance or a program on your dev CRN. Fetch the logs using `aleph instance logs` with the --domain option.

For testing you can insert additional log entries using the systemd-cat command
 ```bash
echo 1 $(date)|systemd-cat -t vm-63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace-stdout
```
or

```bash
for ((i=1; i<=400; i++));
do
 echo $i
 echo $i |systemd-cat -t vm-63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace-stdout
 sleep 1
done
```

Log server side can be  checked using
```
journalctl -t vm-63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace-stderr -t vm-63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace-stdout -f
```

For reproducing or debugging issues adding an asyncio.sleep(0.2) inside the websocket handling code
usually helps, most inner loop of `stream_logs` method inside src/aleph/vm/orchestrator/views/operator.py
---
 src/aleph/vm/orchestrator/views/operator.py |  1 +
 src/aleph/vm/utils/logs.py                  | 29 ++++++++++++++-------
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py
index fc7436d70..b808e94fe 100644
--- a/src/aleph/vm/orchestrator/views/operator.py
+++ b/src/aleph/vm/orchestrator/views/operator.py
@@ -93,6 +93,7 @@ async def stream_logs(request: web.Request) -> web.StreamResponse:
                     logger.debug(message)
 
                     await ws.send_json({"type": log_type, "message": message})
+                    queue.task_done()
 
             finally:
                 await ws.close()
diff --git a/src/aleph/vm/utils/logs.py b/src/aleph/vm/utils/logs.py
index 1bf0dc449..005016040 100644
--- a/src/aleph/vm/utils/logs.py
+++ b/src/aleph/vm/utils/logs.py
@@ -1,8 +1,8 @@
 import asyncio
 import logging
 from collections.abc import Callable, Generator
-from datetime import datetime
-from typing import TypedDict
+from datetime import datetime, timedelta
+from typing import List, TypedDict
 
 from systemd import journal
 
@@ -35,27 +35,38 @@ def make_logs_queue(stdout_identifier, stderr_identifier, skip_past=False) -> tu
     r = journal.Reader()
     r.add_match(SYSLOG_IDENTIFIER=stdout_identifier)
     r.add_match(SYSLOG_IDENTIFIER=stderr_identifier)
-    queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
+    queue: asyncio.Queue = asyncio.Queue(maxsize=5)
+    tasks: List[asyncio.Future] = []
 
-    def _ready_for_read() -> None:
-        change_type = r.process()  # reset fd status
-        if change_type != journal.APPEND:
-            return
+    async def process_messages() -> None:
+        loop.remove_reader(r.fileno())
         entry: EntryDict
         for entry in r:
             log_type = "stdout" if entry["SYSLOG_IDENTIFIER"] == stdout_identifier else "stderr"
             msg = entry["MESSAGE"]
-            asyncio.create_task(queue.put((log_type, msg)))
+            await queue.put((log_type, msg))
+            r.process()  # reset fd status
+        r.process()  # reset fd status
+        loop.add_reader(r.fileno(), _ready_for_read)
+
+    def _ready_for_read() -> None:
+        task = loop.create_task(process_messages(), name=f"process_messages-queue-{id(queue)}")
+        tasks.append(task)
+        task.add_done_callback(tasks.remove)
 
     if skip_past:
-        r.seek_tail()
+        # seek_tail doesn't work see https://github.com/systemd/systemd/issues/17662
+        r.seek_realtime(datetime.now() - timedelta(seconds=10))
 
     loop = asyncio.get_event_loop()
     loop.add_reader(r.fileno(), _ready_for_read)
+    r.process()
 
     def do_cancel():
         logger.info(f"cancelling reader {r}")
         loop.remove_reader(r.fileno())
+        for task in tasks:
+            task.cancel()
         r.close()
 
     return queue, do_cancel

From b3f1728a6c0cc1a2fae705dfe7b00cb23354e760 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 9 Jan 2025 09:48:33 +0100
Subject: [PATCH 924/990] Rework make_logs_queue to make it more readable

---
 src/aleph/vm/utils/logs.py | 39 +++++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/src/aleph/vm/utils/logs.py b/src/aleph/vm/utils/logs.py
index 005016040..868aad7a3 100644
--- a/src/aleph/vm/utils/logs.py
+++ b/src/aleph/vm/utils/logs.py
@@ -2,7 +2,7 @@
 import logging
 from collections.abc import Callable, Generator
 from datetime import datetime, timedelta
-from typing import List, TypedDict
+from typing import TypedDict
 
 from systemd import journal
 
@@ -32,42 +32,47 @@ def make_logs_queue(stdout_identifier, stderr_identifier, skip_past=False) -> tu
     For more information refer to the sd-journal(3) manpage
     and systemd.journal module documentation.
     """
-    r = journal.Reader()
-    r.add_match(SYSLOG_IDENTIFIER=stdout_identifier)
-    r.add_match(SYSLOG_IDENTIFIER=stderr_identifier)
+    journal_reader = journal.Reader()
+    journal_reader.add_match(SYSLOG_IDENTIFIER=stdout_identifier)
+    journal_reader.add_match(SYSLOG_IDENTIFIER=stderr_identifier)
     queue: asyncio.Queue = asyncio.Queue(maxsize=5)
-    tasks: List[asyncio.Future] = []
+    tasks: list[asyncio.Task] = []
+
+    loop = asyncio.get_event_loop()
 
     async def process_messages() -> None:
-        loop.remove_reader(r.fileno())
+        """Enqueue all the available log entries, wait if queue is full, then wait for new message via add_reader"""
+        # Remove reader so we don't get called again while processing
+        loop.remove_reader(journal_reader.fileno())
         entry: EntryDict
-        for entry in r:
+        for entry in journal_reader:
             log_type = "stdout" if entry["SYSLOG_IDENTIFIER"] == stdout_identifier else "stderr"
             msg = entry["MESSAGE"]
+            # will wait if queue is full
             await queue.put((log_type, msg))
-            r.process()  # reset fd status
-        r.process()  # reset fd status
-        loop.add_reader(r.fileno(), _ready_for_read)
+            journal_reader.process()  # reset fd status
+        journal_reader.process()  # reset fd status
+        # Call _ready_for_read read when entries are readable again, this is non-blocking
+        loop.add_reader(journal_reader.fileno(), _ready_for_read)
 
     def _ready_for_read() -> None:
+        # wrapper around process_messages as add_reader don't take an async func
         task = loop.create_task(process_messages(), name=f"process_messages-queue-{id(queue)}")
         tasks.append(task)
         task.add_done_callback(tasks.remove)
 
     if skip_past:
         # seek_tail doesn't work see https://github.com/systemd/systemd/issues/17662
-        r.seek_realtime(datetime.now() - timedelta(seconds=10))
+        journal_reader.seek_realtime(datetime.now() - timedelta(seconds=10))
 
-    loop = asyncio.get_event_loop()
-    loop.add_reader(r.fileno(), _ready_for_read)
-    r.process()
+    _ready_for_read()
 
     def do_cancel():
-        logger.info(f"cancelling reader {r}")
-        loop.remove_reader(r.fileno())
+        logger.info(f"cancelling queue and reader {journal_reader}")
+        loop.remove_reader(journal_reader.fileno())
         for task in tasks:
             task.cancel()
-        r.close()
+        journal_reader.close()
 
     return queue, do_cancel
 

From 732aa1afcf61aaf9a4b76950d66d2b80338d9d47 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 14 Jan 2025 16:01:44 +0100
Subject: [PATCH 925/990] Test test_make_logs_queue

---
 tests/supervisor/test_log.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 tests/supervisor/test_log.py

diff --git a/tests/supervisor/test_log.py b/tests/supervisor/test_log.py
new file mode 100644
index 000000000..23f27aaaa
--- /dev/null
+++ b/tests/supervisor/test_log.py
@@ -0,0 +1,15 @@
+from asyncio import QueueEmpty
+
+from aleph.vm.utils.logs import make_logs_queue
+
+
+def test_make_logs_queue():
+    stdout_identifier = "test_stdout"
+    stderr_identifier = "test_stderr"
+    queue, do_cancel = make_logs_queue(stdout_identifier, stderr_identifier)
+    import pytest
+
+    with pytest.raises(QueueEmpty):
+        while queue.get_nowait():
+            queue.task_done()
+    do_cancel()

From 45ac6c647f1805e0460d634a6fb4354b8fa21c53 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 13 Jan 2025 18:28:24 +0100
Subject: [PATCH 926/990] Fix update_allocation failing on missing message

Jira Ticket: ALEH-337

If a message was inexisting, for example in the ref for an instance, update_allocation halted at the VM it was processing
leaving the other vm as is.

Solution:
Catch the HttpNotFound error that is raised when a message cannot be retrieved
---
 src/aleph/vm/orchestrator/views/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 94209c59f..32a012396 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -403,6 +403,7 @@ async def update_allocations(request: web.Request):
         VmSetupError,
         MicroVMFailedInitError,
         HostNotFoundError,
+        HTTPNotFound,
     )
 
     scheduling_errors: dict[ItemHash, Exception] = {}

From bb587e02cccaae74505f38a2db5f260c8e439551 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 13 Jan 2025 18:32:58 +0100
Subject: [PATCH 927/990] Problem: update_allocation stopped on unkown error

While creating a VM if any unkown error was raised, update_allocation halted where it was, failing to process the rest of the list

Solution:
Catch all the possible errors

Continuation of ALEPH-337
---
 src/aleph/vm/orchestrator/views/__init__.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 32a012396..b6a45c0d7 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -415,8 +415,12 @@ async def update_allocations(request: web.Request):
             vm_hash = ItemHash(vm_hash)
             await start_persistent_vm(vm_hash, pubsub, pool)
         except vm_creation_exceptions as error:
-            logger.exception(error)
+            logger.exception("Error while starting VM '%s': %s", vm_hash, error)
             scheduling_errors[vm_hash] = error
+        except Exception as error:
+            # Handle unknown exception separately, to avoid leaking data
+            logger.exception("Unhandled Error while starting VM '%s': %s", vm_hash, error)
+            scheduling_errors[vm_hash] = Exception("Unhandled Error")
 
     # Schedule the start of instances:
     for instance_hash in allocation.instances:
@@ -425,8 +429,12 @@ async def update_allocations(request: web.Request):
         try:
             await start_persistent_vm(instance_item_hash, pubsub, pool)
         except vm_creation_exceptions as error:
-            logger.exception(error)
+            logger.exception("Error while starting VM '%s': %s", instance_hash, error)
             scheduling_errors[instance_item_hash] = error
+        except Exception as error:
+            # Handle unknown exception separately, to avoid leaking data
+            logger.exception("Unhandled Error while starting VM '%s': %s", instance_hash, error)
+            scheduling_errors[vm_hash] = Exception("Unhandled Error")
 
     # Log unsupported features
     if allocation.on_demand_vms:

From b860265e26d5c38f701810db3be0d2a50f1bf557 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 13 Jan 2025 18:33:52 +0100
Subject: [PATCH 928/990] Enhance verbosity on failed downloads

---
 src/aleph/vm/storage.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index df0505ca7..15d3b9384 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -110,9 +110,12 @@ async def download_file(url: str, local_path: Path) -> None:
                 await asyncio.wait_for(file_downloaded_by_another_task(local_path), timeout=30)
             except TimeoutError as error:
                 if attempt < (download_attempts - 1):
-                    logger.warning(f"Download failed, retrying attempt {attempt + 1}/{download_attempts}...")
+                    logger.warning(
+                        f"Download failed (waiting for another taks), retrying attempt {attempt + 1}/{download_attempts}..."
+                    )
                     continue
                 else:
+                    logger.warning(f"Download of {url} failed  (waiting for another task), aborting...")
                     raise error from file_exists_error
         except (
             aiohttp.ClientConnectionError,
@@ -123,6 +126,7 @@ async def download_file(url: str, local_path: Path) -> None:
                 logger.warning(f"Download failed, retrying attempt {attempt + 1}/{download_attempts}...")
                 # continue  #  continue inside try/finally block is unimplemented in `mypyc`
             else:
+                logger.warning(f"Download of {url} failed  (aborting...")
                 raise error
         finally:
             # Ensure no partial file is left behind

From 4e23ce923b775c27c50b08c9b0f5695cfc94fa9c Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 14 Jan 2025 16:13:07 +0100
Subject: [PATCH 929/990] Display proper error for message not found
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

or not being downloadable

Problem:
When calling an program on a CRN, whose message the CRN couldn’t reach, it returned the error
```
500: Unhandled error during initialisation
```

instead of a proper explaination error

Solution:
Catch that error. Catch also the other possible error when downloading

This also now works when failing to download a runtime or volume
---
 src/aleph/vm/orchestrator/run.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py
index 9c2a8b298..f82a8ae17 100644
--- a/src/aleph/vm/orchestrator/run.py
+++ b/src/aleph/vm/orchestrator/run.py
@@ -3,7 +3,7 @@
 from typing import Any
 
 import msgpack
-from aiohttp import web
+from aiohttp import ClientResponseError, web
 from aiohttp.web_exceptions import (
     HTTPBadGateway,
     HTTPBadRequest,
@@ -88,6 +88,12 @@ async def create_vm_execution_or_raise_http_error(vm_hash: ItemHash, pool: VmPoo
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
         raise HTTPInternalServerError(reason="Host did not respond to ping") from error
+    except ClientResponseError as error:
+        logger.exception(error)
+        if error.status == 404:
+            raise HTTPInternalServerError(reason=f"Item hash {vm_hash} not found") from error
+        else:
+            raise HTTPInternalServerError(reason=f"Error downloading {vm_hash}") from error
     except Exception as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)

From 76c68972e8dbf0ded7318758ecbef0f355b4eaa0 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Tue, 14 Jan 2025 17:37:23 +0100
Subject: [PATCH 930/990] Fix guest-api ConnectionClosedError: Reader at end of
 file

Sentry Issue: ALEPH-VM-STAGING-41
Jira Issue: ALEPH-353

This error was making the diagnostic down constently, raising 3K error
in 48h on Sentry

In aleph.vm.guest_api._main_.put_in_cache
```
ConnectionClosedError: Reader at end of file
  File "aiohttp/web_app.py", line 569, in _handle
    return await handler(request)
  File "aleph/vm/guest_api/__main__.py", line 128, in put_in_cache
    return web.json_response(await redis.set(f"{prefix}:{key}", value, expire=CACHE_EXPIRES_AFTER))
```

*Investigation*
The error started at Jan 12, 2025 7:26:47 AM CET
The redis server was restarted around the same time by the
server unattended-upgrades (apt)

*Analysis*
The guest api for the diagnostic VM lost the connexion to the redis server (via unix
connexion) when it was restarted. Since the guest api always reuse
the same connexion the error was always triggered.

In addition as the diagnostic vm is called regularly by monitoring
services, it doesn't timeout and stop, so the init process that
establish the redis connection was never redone

*Solution*
Check if the redis connection is still ok by pinging the service, if it
raise an error, create a new connection

*How to test*
Start CRN, call the diagnostic vm redis endpoint
http://localhost:4020/vm/63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace/cache/get/a

Then restart the redis service on the CRN

```bash
systemctl restart redis
```

and call the diagnostic vm redis ndpoint again
---
 src/aleph/vm/guest_api/__main__.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/guest_api/__main__.py b/src/aleph/vm/guest_api/__main__.py
index 8000d52bc..e5b89ebe8 100644
--- a/src/aleph/vm/guest_api/__main__.py
+++ b/src/aleph/vm/guest_api/__main__.py
@@ -24,8 +24,15 @@
 
 async def get_redis(address: str = REDIS_ADDRESS) -> aioredis.Redis:
     global _redis
-    if _redis is None:
+    # Ensure the redis connection is still up before returning it
+    if _redis:
+        try:
+            await _redis.ping()
+        except aioredis.ConnectionClosedError:
+            _redis = None
+    if not _redis:
         _redis = await aioredis.create_redis(address=address)
+
     return _redis
 
 

From 98b8fc850bf6fed4fa970caac414dde7be609b52 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 23 Jan 2025 13:59:33 +0100
Subject: [PATCH 931/990] Fix instance ip4 network stopping (nat) (#742)

Temporary disable tearing down the network when stopping supervisor
Fix issue of persistent instances running inside systemd controller losing their ipv4 Nat access
upon supervisor restart or upgrade.

Github issue https://github.com/aleph-im/support/issues/9
---
 src/aleph/vm/pool.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 58a3e6fae..9a9e69f3a 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -81,7 +81,11 @@ def setup(self) -> None:
     def teardown(self) -> None:
         """Stop the VM pool and the network properly."""
         if self.network:
-            self.network.teardown()
+            # self.network.teardown()
+            # FIXME Temporary disable tearing down the network
+            # Fix issue of persistent instances running inside systemd controller losing their ipv4 nat access
+            #  upon supervisor restart or upgrade.
+            pass
 
     async def create_a_vm(
         self, vm_hash: ItemHash, message: ExecutableContent, original: ExecutableContent, persistent: bool

From 6292bcdfc0ab1d19954567d2a54c6dfe01a5fc85 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 23 Jan 2025 15:19:35 +0100
Subject: [PATCH 932/990] Upgrade aleph-message to 0.6 (#744)

---
 packaging/Makefile | 2 +-
 pyproject.toml     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packaging/Makefile b/packaging/Makefile
index 3c4f8a6b1..a601f1c9a 100644
--- a/packaging/Makefile
+++ b/packaging/Makefile
@@ -16,7 +16,7 @@ debian-package-code:
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
 	# Fixing this protobuf dependency version to avoid getting CI errors as version 5.29.0 have this compilation issue
-	pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.5.0' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' 'protobuf==5.28.3'
+	pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.6' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' 'protobuf==5.28.3'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo  target/bin/sevctl
diff --git a/pyproject.toml b/pyproject.toml
index 48542a5a8..c892a53fe 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -36,7 +36,7 @@ dependencies = [
   "aioredis==1.3.1",
   "aiosqlite==0.19",
   "alembic==1.13.1",
-  "aleph-message==0.5",
+  "aleph-message==0.6",
   "aleph-superfluid~=0.2.1",
   "dbus-python==1.3.2",
   "eth-account~=0.10",

From 38c74fea5ac98ff286e5dcf6224301e895bc2be1 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 23 Jan 2025 15:31:44 +0100
Subject: [PATCH 933/990] Fix AssertionError 
 aleph.vm.orchestrator.views.status_check_fastapi (#740)

Jira Issue: ALEPH-355
Sentry Issue: ALEPH-VM-STAGING-2G

Problem:

The diagnostic vm can return False or True for the ip check, return its result
instead of expecting true

 otherwise the whole status_check_fastapi was failing
---
 src/aleph/vm/orchestrator/status.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py
index 12692f6a9..07b21b33f 100644
--- a/src/aleph/vm/orchestrator/status.py
+++ b/src/aleph/vm/orchestrator/status.py
@@ -98,8 +98,7 @@ async def check_ipv4(session: ClientSession, vm_id: ItemHash) -> bool:
     """Check that the VM has IPv4 connectivity."""
     try:
         result: dict = await get_json_from_vm(session, vm_id, "/ip/4")
-        assert result["result"] is True
-        return True
+        return result["result"]
     except ClientResponseError:
         return False
 
@@ -108,9 +107,7 @@ async def check_ipv6(session: ClientSession, vm_id: ItemHash) -> bool:
     """Check that the VM has IPv6 connectivity."""
     try:
         result: dict = await get_json_from_vm(session, vm_id, "/ip/6")
-        assert result["result"] is True
-        assert "headers" in result
-        return True
+        return result["result"]
     except ClientResponseError:
         return False
 

From 77d5898e5b6cadf1b594ba0153d571cd98dd9b79 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Thu, 23 Jan 2025 15:32:06 +0100
Subject: [PATCH 934/990] Sanitize volume names (#743)

* Problem: If a user want to attach a volume that have a name with spaces or other weird symbols, it raises an error and don't allocate the VM.

Solution: Sanitize the volume name before creating it.

* Fix: add another action on the pending TODO.
---
 src/aleph/vm/controllers/firecracker/executable.py | 2 +-
 src/aleph/vm/storage.py                            | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/aleph/vm/controllers/firecracker/executable.py b/src/aleph/vm/controllers/firecracker/executable.py
index c6d6721fa..7249be9c7 100644
--- a/src/aleph/vm/controllers/firecracker/executable.py
+++ b/src/aleph/vm/controllers/firecracker/executable.py
@@ -114,7 +114,7 @@ async def download_kernel(self):
 
     async def download_volumes(self):
         volumes = []
-        # TODO: Download in parallel
+        # TODO: Download in parallel and prevent duplicated volume names
         for i, volume in enumerate(self.message_content.volumes):
             # only persistant volume has name and mount
             if isinstance(volume, PersistentVolume):
diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py
index 15d3b9384..58d6f78c2 100644
--- a/src/aleph/vm/storage.py
+++ b/src/aleph/vm/storage.py
@@ -376,12 +376,14 @@ async def get_volume_path(volume: MachineVolume, namespace: str) -> Path:
         return await get_existing_file(ref)
     elif isinstance(volume, PersistentVolume | RootfsVolume):
         volume_name = volume.name if isinstance(volume, PersistentVolume) else "rootfs"
+
         if volume.persistence != VolumePersistence.host:
             msg = "Only 'host' persistence is supported"
             raise NotImplementedError(msg)
         if not re.match(r"^[\w\-_/]+$", volume_name):
-            msg = f"Invalid value for volume name: {repr(volume_name)}"
-            raise ValueError(msg)
+            # Sanitize volume names
+            logger.debug(f"Invalid values for volume name: {repr(volume_name)} detected, sanitizing")
+            volume_name = re.sub(r"[^\w\-_]", "_", volume_name)
         (Path(settings.PERSISTENT_VOLUMES_DIR) / namespace).mkdir(exist_ok=True)
         if volume.parent:
             return await create_devmapper(volume, namespace)

From e6f2ff705092ee211568a5de2eab4833f8fbdb74 Mon Sep 17 00:00:00 2001
From: Hugo Herter 
Date: Thu, 23 Jan 2025 15:32:31 +0100
Subject: [PATCH 935/990] Fix: Runtime emitted wrong reboot signal (#741)

The runtime init `init1.py` emitted the signal `0x4321fedc`
to the Linux kernel when shutting down.

The correct procedure [according to the Firecracker docs]
(https://github.com/firecracker-microvm/firecracker/blob/dfb45dc4213bcb1c9704435457e233d3a210dce2/docs/getting-started.md?plain=1#L298) is to issue a `reboot`.

The syscall number for a reboot on Linux is `0x1234567`.

Tested using Firecracker independently of `aleph-vm`.
---
 runtimes/aleph-debian-12-python/init1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtimes/aleph-debian-12-python/init1.py b/runtimes/aleph-debian-12-python/init1.py
index 9f70bef64..5af375855 100644
--- a/runtimes/aleph-debian-12-python/init1.py
+++ b/runtimes/aleph-debian-12-python/init1.py
@@ -619,6 +619,6 @@ async def handle_instruction(reader, writer):
     # Send reboot syscall, see man page
     # https://man7.org/linux/man-pages/man2/reboot.2.html
     libc = ctypes.CDLL(None)
-    libc.syscall(169, 0xFEE1DEAD, 672274793, 0x4321FEDC, None)
+    libc.syscall(169, 0xFEE1DEAD, 672274793, 0x1234567, None)
     # The exit should not happen due to system halt.
     sys.exit(0)

From 7f89c89a1d2c1770cbc9fe1ba71c617dbf12d47f Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Wed, 29 Jan 2025 18:25:51 +0100
Subject: [PATCH 936/990] Unified branding on node index page style (#745)

* Fix: Added new branding logo to unify node index page aesthetic.

* Fix: Replace Aleph.im by Aleph Cloud on the first sentence.
---
 .../views/static/aleph-cloud-v1.svg           | 16 ++++++++++++++++
 .../views/static/aleph-cloud-v2.svg           | 19 +++++++++++++++++++
 .../orchestrator/views/templates/index.html   |  4 ++--
 3 files changed, 37 insertions(+), 2 deletions(-)
 create mode 100644 src/aleph/vm/orchestrator/views/static/aleph-cloud-v1.svg
 create mode 100644 src/aleph/vm/orchestrator/views/static/aleph-cloud-v2.svg

diff --git a/src/aleph/vm/orchestrator/views/static/aleph-cloud-v1.svg b/src/aleph/vm/orchestrator/views/static/aleph-cloud-v1.svg
new file mode 100644
index 000000000..c32715bdc
--- /dev/null
+++ b/src/aleph/vm/orchestrator/views/static/aleph-cloud-v1.svg
@@ -0,0 +1,16 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/aleph/vm/orchestrator/views/static/aleph-cloud-v2.svg b/src/aleph/vm/orchestrator/views/static/aleph-cloud-v2.svg
new file mode 100644
index 000000000..f6b39e621
--- /dev/null
+++ b/src/aleph/vm/orchestrator/views/static/aleph-cloud-v2.svg
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/aleph/vm/orchestrator/views/templates/index.html b/src/aleph/vm/orchestrator/views/templates/index.html
index 96d229864..09715acb5 100644
--- a/src/aleph/vm/orchestrator/views/templates/index.html
+++ b/src/aleph/vm/orchestrator/views/templates/index.html
@@ -9,11 +9,11 @@
 
     
              
 
-        
              Aleph.im Compute Node
              
+        
              
 
         
              
             
              
-                This is an Aleph.im compute resource node.
+                This is an Aleph Cloud Compute Resource Node.
             
              
             
              
                 It executes user programs stored on the aleph.im network in Virtual Machines.

From b020227b3c1573f3663846cea847328a5432f6d3 Mon Sep 17 00:00:00 2001
From: philogicae 
Date: Mon, 3 Feb 2025 19:13:34 +0200
Subject: [PATCH 937/990] Add GPU list to /about/usage/system endpoint (#726)

* Add GPU list to /about/usage/system

* Fix: Use created specified class.

* Fix: Added GPU resources to the main pool and exposed it on the endpoint.

* Fix: Solve code quality issues.

* Fix: Improve code organization and structure.

* Fix: Fixed existing tests and add new test case.

* Implement GPU Support (#728)

* Problem: If a user wants to assign a GPU to a QEmu VM he cannot do it.

Solution: Implement GPU assignation feature that will be pass-though to QEmu VMs with native performance.

* Fix: Solved code quality issues

* Fix: Solved compilation issue and fixed gpu logic.

* Fix: Solved issue getting already running executions with GPU

* Fix: Expose GPU support option in `status/config` endpoint

* Fix: Applied some code review suggestions

* Add migration

* Fix: Allow to use the notify endpoint for GPU instances also.

* Fix: Remove migration duplicity.

* Fix: Changes DB initialization order to ensure that DB always exists before running the migrations.

* Fix: Updated migration to only insert the column if isn't inside.

---------

Co-authored-by: Olivier Le Thanh Duong 

* Fix: Solve test code quality issues.

---------

Co-authored-by: Andres D. Molins 
Co-authored-by: Olivier Le Thanh Duong 
---
 src/aleph/vm/conf.py                          |   7 ++
 src/aleph/vm/controllers/configuration.py     |   6 +
 src/aleph/vm/controllers/qemu/instance.py     |   7 +-
 .../controllers/qemu_confidential/instance.py |   2 +
 src/aleph/vm/hypervisors/qemu/qemuvm.py       |  34 +++++-
 .../hypervisors/qemu_confidential/qemuvm.py   |  12 +-
 src/aleph/vm/models.py                        |  34 +++++-
 src/aleph/vm/orchestrator/chain.py            |   6 +-
 src/aleph/vm/orchestrator/cli.py              |   7 +-
 src/aleph/vm/orchestrator/metrics.py          |   2 +
 ...c69b_add_gpu_column_to_executions_table.py |  38 ++++++
 src/aleph/vm/orchestrator/payment.py          |   6 +-
 src/aleph/vm/orchestrator/resources.py        |  27 ++++-
 src/aleph/vm/orchestrator/supervisor.py       |   4 -
 src/aleph/vm/orchestrator/tasks.py            |  19 +--
 src/aleph/vm/orchestrator/views/__init__.py   |  13 +-
 src/aleph/vm/pool.py                          |  27 +++++
 src/aleph/vm/resources.py                     | 111 ++++++++++++++++++
 tests/supervisor/test_resources.py            |  38 ++++++
 tests/supervisor/test_views.py                |  17 +++
 20 files changed, 374 insertions(+), 43 deletions(-)
 create mode 100644 src/aleph/vm/orchestrator/migrations/versions/0002_5c6ae643c69b_add_gpu_column_to_executions_table.py
 create mode 100644 src/aleph/vm/resources.py
 create mode 100644 tests/supervisor/test_resources.py

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 18add7170..264be819e 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -275,6 +275,11 @@ class Settings(BaseSettings):
 
     CONFIDENTIAL_SESSION_DIRECTORY: Path = Field(None, description="Default to EXECUTION_ROOT/sessions")
 
+    ENABLE_GPU_SUPPORT: bool = Field(
+        default=False,
+        description="Enable GPU pass-through support to VMs, only allowed for QEmu hypervisor",
+    )
+
     # Tests on programs
 
     FAKE_DATA_PROGRAM: Path | None = None
@@ -391,6 +396,8 @@ def check(self):
             # assert check_amd_sev_snp_supported(), "SEV-SNP feature isn't enabled, enable it in BIOS"
             assert self.ENABLE_QEMU_SUPPORT, "Qemu Support is needed for confidential computing and it's disabled, "
             "enable it setting the env variable `ENABLE_QEMU_SUPPORT=True` in configuration"
+        if self.ENABLE_GPU_SUPPORT:
+            assert self.ENABLE_QEMU_SUPPORT, "Qemu Support is needed for GPU support and it's disabled, "
 
     def setup(self):
         """Setup the environment defined by the settings. Call this method after loading the settings."""
diff --git a/src/aleph/vm/controllers/configuration.py b/src/aleph/vm/controllers/configuration.py
index da10d8395..fb4b4ff1f 100644
--- a/src/aleph/vm/controllers/configuration.py
+++ b/src/aleph/vm/controllers/configuration.py
@@ -23,6 +23,10 @@ class QemuVMHostVolume(BaseModel):
     read_only: bool
 
 
+class QemuGPU(BaseModel):
+    pci_host: str
+
+
 class QemuVMConfiguration(BaseModel):
     qemu_bin_path: str
     cloud_init_drive_path: str | None
@@ -33,6 +37,7 @@ class QemuVMConfiguration(BaseModel):
     mem_size_mb: int
     interface_name: str | None
     host_volumes: list[QemuVMHostVolume]
+    gpus: list[QemuGPU]
 
 
 class QemuConfidentialVMConfiguration(BaseModel):
@@ -45,6 +50,7 @@ class QemuConfidentialVMConfiguration(BaseModel):
     mem_size_mb: int
     interface_name: str | None
     host_volumes: list[QemuVMHostVolume]
+    gpus: list[QemuGPU]
     ovmf_path: Path
     sev_session_file: Path
     sev_dh_cert_file: Path
diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py
index dd840e22b..259f84744 100644
--- a/src/aleph/vm/controllers/qemu/instance.py
+++ b/src/aleph/vm/controllers/qemu/instance.py
@@ -5,7 +5,7 @@
 from asyncio import Task
 from asyncio.subprocess import Process
 from pathlib import Path
-from typing import Generic, TypeVar
+from typing import Generic, List, TypeVar
 
 import psutil
 from aleph_message.models import ItemHash
@@ -17,6 +17,7 @@
 from aleph.vm.controllers.configuration import (
     Configuration,
     HypervisorType,
+    QemuGPU,
     QemuVMConfiguration,
     QemuVMHostVolume,
     save_controller_configuration,
@@ -29,6 +30,7 @@
 from aleph.vm.controllers.qemu.cloudinit import CloudInitMixin
 from aleph.vm.network.firewall import teardown_nftables_for_vm
 from aleph.vm.network.interfaces import TapInterface
+from aleph.vm.resources import HostGPU
 from aleph.vm.storage import get_rootfs_base_path
 from aleph.vm.utils import HostNotFoundError, ping, run_in_subprocess
 
@@ -36,6 +38,8 @@
 
 
 class AlephQemuResources(AlephFirecrackerResources):
+    gpus: List[HostGPU] = []
+
     async def download_runtime(self) -> None:
         volume = self.message_content.rootfs
         parent_image_path = await get_rootfs_base_path(volume.parent.ref)
@@ -200,6 +204,7 @@ async def configure(self):
                 )
                 for volume in self.resources.volumes
             ],
+            gpus=[QemuGPU(pci_host=gpu.pci_host) for gpu in self.resources.gpus],
         )
 
         configuration = Configuration(
diff --git a/src/aleph/vm/controllers/qemu_confidential/instance.py b/src/aleph/vm/controllers/qemu_confidential/instance.py
index f432cff69..37986b10c 100644
--- a/src/aleph/vm/controllers/qemu_confidential/instance.py
+++ b/src/aleph/vm/controllers/qemu_confidential/instance.py
@@ -13,6 +13,7 @@
     Configuration,
     HypervisorType,
     QemuConfidentialVMConfiguration,
+    QemuGPU,
     QemuVMHostVolume,
     save_controller_configuration,
 )
@@ -126,6 +127,7 @@ async def configure(self):
                 )
                 for volume in self.resources.volumes
             ],
+            gpus=[QemuGPU(pci_host=gpu.pci_host) for gpu in self.resources.gpus],
         )
 
         configuration = Configuration(
diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py
index 5bcb1313c..df7559613 100644
--- a/src/aleph/vm/hypervisors/qemu/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py
@@ -7,7 +7,7 @@
 import qmp
 from systemd import journal
 
-from aleph.vm.controllers.configuration import QemuVMConfiguration
+from aleph.vm.controllers.configuration import QemuGPU, QemuVMConfiguration
 from aleph.vm.controllers.qemu.instance import logger
 
 
@@ -28,6 +28,7 @@ class QemuVM:
     interface_name: str
     qemu_process: Process | None = None
     host_volumes: list[HostVolume]
+    gpus: list[QemuGPU]
     journal_stdout: TextIO | None
     journal_stderr: TextIO | None
 
@@ -55,6 +56,7 @@ def __init__(self, vm_hash, config: QemuVMConfiguration):
             )
             for volume in config.host_volumes
         ]
+        self.gpus = config.gpus
 
     @property
     def _journal_stdout_name(self) -> str:
@@ -113,17 +115,15 @@ async def start(
             # "-serial", "telnet:localhost:4321,server,nowait",
             # "-snapshot",  # Do not save anything to disk
         ]
-        for volume in self.host_volumes:
-            args += [
-                "-drive",
-                f"file={volume.path_on_host},format=raw,readonly={'on' if volume.read_only else 'off'},media=disk,if=virtio",
-            ]
         if self.interface_name:
             # script=no, downscript=no tell qemu not to try to set up the network itself
             args += ["-net", "nic,model=virtio", "-net", f"tap,ifname={self.interface_name},script=no,downscript=no"]
 
         if self.cloud_init_drive_path:
             args += ["-cdrom", f"{self.cloud_init_drive_path}"]
+
+        args += self._get_host_volumes_args()
+        args += self._get_gpu_args()
         print(*args)
 
         self.qemu_process = proc = await asyncio.create_subprocess_exec(
@@ -138,6 +138,28 @@ async def start(
         )
         return proc
 
+    def _get_host_volumes_args(self):
+        args = []
+        for volume in self.host_volumes:
+            args += [
+                "-drive",
+                f"file={volume.path_on_host},format=raw,readonly={'on' if volume.read_only else 'off'},media=disk,if=virtio",
+            ]
+        return args
+
+    def _get_gpu_args(self):
+        args = [
+            # Use host-phys-bits-limit argument for GPU support. TODO: Investigate how to get the correct bits size
+            "-cpu",
+            "host,host-phys-bits-limit=0x28",
+        ]
+        for gpu in self.gpus:
+            args += [
+                "-device",
+                f"vfio-pci,host={gpu.pci_host},multifunction=on,x-vga=on",
+            ]
+        return args
+
     def _get_qmpclient(self) -> qmp.QEMUMonitorProtocol | None:
         if not (self.qmp_socket_path and self.qmp_socket_path.exists()):
             return None
diff --git a/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
index 89e9c3e80..353c3f78d 100644
--- a/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
+++ b/src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py
@@ -110,24 +110,24 @@ async def start(
             # raise an error and prevent boot. Passing the argument --cpu host instruct the VM to use the same CPU
             # model than the host thus the VM's kernel knows which method is used to get random numbers (Intel and
             # AMD have different methods) and properly boot.
+            # Use host-phys-bits-limit argument for GPU support. TODO: Investigate how to get the correct bits size
             "-cpu",
-            "host",
+            "host,host-phys-bits-limit=0x28",
             # Uncomment following for debug
             # "-serial", "telnet:localhost:4321,server,nowait",
             # "-snapshot",  # Do not save anything to disk
         ]
-        for volume in self.host_volumes:
-            args += [
-                "-drive",
-                f"file={volume.path_on_host},format=raw,readonly={'on' if volume.read_only else 'off'},media=disk,if=virtio",
-            ]
         if self.interface_name:
             # script=no, downscript=no tell qemu not to try to set up the network itself
             args += ["-net", "nic,model=virtio", "-net", f"tap,ifname={self.interface_name},script=no,downscript=no"]
 
         if self.cloud_init_drive_path:
             args += ["-cdrom", f"{self.cloud_init_drive_path}"]
+
+        args += self._get_host_volumes_args()
+        args += self._get_gpu_args()
         print(*args)
+
         self.qemu_process = proc = await asyncio.create_subprocess_exec(
             *args,
             stdin=asyncio.subprocess.DEVNULL,
diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py
index 9aee9320a..7dd59091b 100644
--- a/src/aleph/vm/models.py
+++ b/src/aleph/vm/models.py
@@ -1,10 +1,12 @@
 import asyncio
+import json
 import logging
 import uuid
 from asyncio import Task
 from collections.abc import Callable, Coroutine
 from dataclasses import dataclass
 from datetime import datetime, timezone
+from typing import List
 
 from aleph_message.models import (
     ExecutableContent,
@@ -12,14 +14,14 @@
     ItemHash,
     ProgramContent,
 )
-from aleph_message.models.execution.environment import HypervisorType
+from aleph_message.models.execution.environment import GpuProperties, HypervisorType
+from pydantic.json import pydantic_encoder
 
 from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.executable import AlephFirecrackerExecutable
 from aleph.vm.controllers.firecracker.instance import AlephInstanceResources
 from aleph.vm.controllers.firecracker.program import (
     AlephFirecrackerProgram,
-    AlephFirecrackerResources,
     AlephProgramResources,
 )
 from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
@@ -38,6 +40,7 @@
 )
 from aleph.vm.orchestrator.pubsub import PubSub
 from aleph.vm.orchestrator.vm import AlephFirecrackerInstance
+from aleph.vm.resources import GpuDevice, HostGPU
 from aleph.vm.systemd import SystemDManager
 from aleph.vm.utils import create_task_log_exceptions, dumps_for_json
 
@@ -69,8 +72,11 @@ class VmExecution:
     vm_hash: ItemHash
     original: ExecutableContent
     message: ExecutableContent
-    resources: AlephFirecrackerResources | None = None
-    vm: AlephFirecrackerExecutable | AlephQemuInstance | None = None
+    resources: (
+        AlephProgramResources | AlephInstanceResources | AlephQemuResources | AlephQemuConfidentialInstance | None
+    ) = None
+    vm: AlephFirecrackerExecutable | AlephQemuInstance | AlephQemuConfidentialInstance | None = None
+    gpus: List[HostGPU] = []
 
     times: VmExecutionTimes
 
@@ -202,6 +208,7 @@ async def prepare(self) -> None:
                         resources = AlephQemuConfidentialResources(self.message, namespace=self.vm_hash)
                     else:
                         resources = AlephQemuResources(self.message, namespace=self.vm_hash)
+                    resources.gpus = self.gpus
                 else:
                     msg = f"Unknown hypervisor type {self.hypervisor}"
                     raise ValueError(msg)
@@ -216,6 +223,24 @@ async def prepare(self) -> None:
             self.times.prepared_at = datetime.now(tz=timezone.utc)
             self.resources = resources
 
+    def prepare_gpus(self, available_gpus: List[GpuDevice]) -> None:
+        gpus = []
+        if self.message.requirements and self.message.requirements.gpu:
+            for gpu in self.message.requirements.gpu:
+                gpu = GpuProperties.parse_obj(gpu)
+                for available_gpu in available_gpus:
+                    if available_gpu.device_id == gpu.device_id:
+                        gpus.append(HostGPU(pci_host=available_gpu.pci_host))
+                        break
+        self.gpus = gpus
+
+    def uses_gpu(self, pci_host: str) -> bool:
+        for gpu in self.gpus:
+            if gpu.pci_host == pci_host:
+                return True
+
+        return False
+
     def create(
         self, vm_id: int, tap_interface: TapInterface | None = None, prepare: bool = True
     ) -> AlephVmControllerInterface:
@@ -437,6 +462,7 @@ async def save(self):
                     message=self.message.json(),
                     original_message=self.original.json(),
                     persistent=self.persistent,
+                    gpus=json.dumps(self.gpus, default=pydantic_encoder),
                 )
             )
 
diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py
index 7321aa458..0b4174397 100644
--- a/src/aleph/vm/orchestrator/chain.py
+++ b/src/aleph/vm/orchestrator/chain.py
@@ -60,9 +60,13 @@ def check_tokens(cls, values):
 }
 
 
+class InvalidChainError(ValueError):
+    pass
+
+
 def get_chain(chain: str) -> ChainInfo:
     try:
         return STREAM_CHAINS[chain]
     except KeyError:
         msg = f"Unknown chain id for chain {chain}"
-        raise ValueError(msg)
+        raise InvalidChainError(msg)
diff --git a/src/aleph/vm/orchestrator/cli.py b/src/aleph/vm/orchestrator/cli.py
index bbae396d4..740733e61 100644
--- a/src/aleph/vm/orchestrator/cli.py
+++ b/src/aleph/vm/orchestrator/cli.py
@@ -167,9 +167,6 @@ async def benchmark(runs: int):
     """Measure program performance by immediately running the supervisor
     with fake requests.
     """
-    engine = metrics.setup_engine()
-    await metrics.create_tables(engine)
-
     ref = ItemHash("cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe")
     settings.FAKE_DATA_PROGRAM = settings.BENCHMARK_FAKE_DATA_PROGRAM
 
@@ -357,6 +354,10 @@ def main():
     settings.check()
 
     logger.debug("Initialising the DB...")
+    # Check and create execution database
+    engine = metrics.setup_engine()
+    asyncio.run(metrics.create_tables(engine))
+    # After creating it run the DB migrations
     asyncio.run(run_async_db_migrations())
     logger.debug("DB up to date.")
 
diff --git a/src/aleph/vm/orchestrator/metrics.py b/src/aleph/vm/orchestrator/metrics.py
index f7f166481..6c9b8eea0 100644
--- a/src/aleph/vm/orchestrator/metrics.py
+++ b/src/aleph/vm/orchestrator/metrics.py
@@ -76,6 +76,8 @@ class ExecutionRecord(Base):
     original_message = Column(JSON, nullable=True)
     persistent = Column(Boolean, nullable=True)
 
+    gpus = Column(JSON, nullable=True)
+
     def __repr__(self):
         return f""
 
diff --git a/src/aleph/vm/orchestrator/migrations/versions/0002_5c6ae643c69b_add_gpu_column_to_executions_table.py b/src/aleph/vm/orchestrator/migrations/versions/0002_5c6ae643c69b_add_gpu_column_to_executions_table.py
new file mode 100644
index 000000000..4b739323b
--- /dev/null
+++ b/src/aleph/vm/orchestrator/migrations/versions/0002_5c6ae643c69b_add_gpu_column_to_executions_table.py
@@ -0,0 +1,38 @@
+"""add gpu table
+
+Revision ID: 5c6ae643c69b
+Revises: bbb12a12372e
+Create Date: 2024-12-09 19:40:19.279735
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+from sqlalchemy import create_engine
+from sqlalchemy.engine import reflection
+
+from aleph.vm.conf import make_db_url
+
+revision = "5c6ae643c69b"
+down_revision = "bbb12a12372e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    engine = create_engine(make_db_url())
+    inspector = reflection.Inspector.from_engine(engine)
+
+    # The table already exists on most CRNs.
+    tables = inspector.get_table_names()
+    if "executions" in tables:
+        columns = inspector.get_columns("executions")
+        column_names = [c["name"] for c in columns]
+        if "gpus" not in column_names:
+            op.add_column("executions", sa.Column("gpus", sa.JSON(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("executions", "gpus")
diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py
index 7194f873a..f5a79bbca 100644
--- a/src/aleph/vm/orchestrator/payment.py
+++ b/src/aleph/vm/orchestrator/payment.py
@@ -13,7 +13,7 @@
 from aleph.vm.models import VmExecution
 from aleph.vm.utils import to_normalized_address
 
-from .chain import ChainInfo, get_chain
+from .chain import ChainInfo, InvalidChainError, get_chain
 
 logger = logging.getLogger(__name__)
 
@@ -91,10 +91,6 @@ class InvalidAddressError(ValueError):
     pass
 
 
-class InvalidChainError(ValueError):
-    pass
-
-
 async def get_stream(sender: str, receiver: str, chain: str) -> Decimal:
     """
     Get the stream of the user from the Superfluid API.
diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py
index d4b9c8985..b6d34a9f0 100644
--- a/src/aleph/vm/orchestrator/resources.py
+++ b/src/aleph/vm/orchestrator/resources.py
@@ -1,6 +1,7 @@
 import math
 from datetime import datetime, timezone
 from functools import lru_cache
+from typing import List, Optional
 
 import cpuinfo
 import psutil
@@ -10,6 +11,8 @@
 from pydantic import BaseModel, Field
 
 from aleph.vm.conf import settings
+from aleph.vm.pool import VmPool
+from aleph.vm.resources import GpuDevice
 from aleph.vm.sevclient import SevClient
 from aleph.vm.utils import (
     check_amd_sev_es_supported,
@@ -73,15 +76,32 @@ class MachineProperties(BaseModel):
     cpu: CpuProperties
 
 
+class GpuProperties(BaseModel):
+    devices: Optional[List[GpuDevice]]
+    available_devices: Optional[List[GpuDevice]]
+
+
 class MachineUsage(BaseModel):
     cpu: CpuUsage
     mem: MemoryUsage
     disk: DiskUsage
     period: UsagePeriod
     properties: MachineProperties
+    gpu: GpuProperties
     active: bool = True
 
 
+def get_machine_gpus(request: web.Request) -> GpuProperties:
+    pool: VmPool = request.app["vm_pool"]
+    gpus = pool.gpus
+    available_gpus = pool.get_available_gpus()
+
+    return GpuProperties(
+        devices=gpus,
+        available_devices=available_gpus,
+    )
+
+
 @lru_cache
 def get_machine_properties() -> MachineProperties:
     """Fetch machine properties such as architecture, CPU vendor, ...
@@ -90,6 +110,7 @@ def get_machine_properties() -> MachineProperties:
     In the future, some properties may have to be fetched from within a VM.
     """
     cpu_info = cpuinfo.get_cpu_info()  # Slow
+
     return MachineProperties(
         cpu=CpuProperties(
             architecture=cpu_info.get("raw_arch_string", cpu_info.get("arch_string_raw")),
@@ -109,9 +130,10 @@ def get_machine_properties() -> MachineProperties:
 
 
 @cors_allow_all
-async def about_system_usage(_: web.Request):
+async def about_system_usage(request: web.Request):
     """Public endpoint to expose information about the system usage."""
     period_start = datetime.now(timezone.utc).replace(second=0, microsecond=0)
+    machine_properties = get_machine_properties()
 
     usage: MachineUsage = MachineUsage(
         cpu=CpuUsage(
@@ -131,7 +153,8 @@ async def about_system_usage(_: web.Request):
             start_timestamp=period_start,
             duration_seconds=60,
         ),
-        properties=get_machine_properties(),
+        properties=machine_properties,
+        gpu=get_machine_gpus(request),
     )
 
     return web.json_response(text=usage.json(exclude_none=True))
diff --git a/src/aleph/vm/orchestrator/supervisor.py b/src/aleph/vm/orchestrator/supervisor.py
index a5ca999a8..ae6436291 100644
--- a/src/aleph/vm/orchestrator/supervisor.py
+++ b/src/aleph/vm/orchestrator/supervisor.py
@@ -20,7 +20,6 @@
 from aleph.vm.sevclient import SevClient
 from aleph.vm.version import __version__
 
-from .metrics import create_tables, setup_engine
 from .resources import about_certificates, about_system_usage
 from .tasks import (
     start_payment_monitoring_task,
@@ -151,9 +150,6 @@ def run():
     """Run the VM Supervisor."""
     settings.check()
 
-    engine = setup_engine()
-    asyncio.run(create_tables(engine))
-
     loop = asyncio.new_event_loop()
     pool = VmPool(loop)
     pool.setup()
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index bd89a8816..84d9ca498 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -4,6 +4,7 @@
 import math
 import time
 from collections.abc import AsyncIterable
+from decimal import Decimal
 from typing import TypeVar
 
 import aiohttp
@@ -192,13 +193,17 @@ async def check_payment(pool: VmPool):
                 await pool.stop_vm(last_execution.vm_hash)
                 required_balance = await compute_required_balance(executions)
 
-    # Check if the balance held in the wallet is sufficient stream tier resources
-    for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():
-        for chain, executions in chains.items():
-            stream = await get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
-            logger.debug(
-                f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
-            )
+        # Check if the balance held in the wallet is sufficient stream tier resources
+        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():
+            for chain, executions in chains.items():
+                try:
+                    stream = await get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
+                    logger.debug(
+                        f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
+                    )
+                except ValueError as error:
+                    logger.error(f"Error found getting stream for chain {chain} and sender {sender}: {error}")
+                    continue
 
             required_stream = await compute_required_flow(executions)
             logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index b6a45c0d7..899a038f8 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -347,6 +347,7 @@ async def status_public_config(request: web.Request):
                 "ENABLE_QEMU_SUPPORT": settings.ENABLE_QEMU_SUPPORT,
                 "INSTANCE_DEFAULT_HYPERVISOR": settings.INSTANCE_DEFAULT_HYPERVISOR,
                 "ENABLE_CONFIDENTIAL_COMPUTING": settings.ENABLE_CONFIDENTIAL_COMPUTING,
+                "ENABLE_GPU_SUPPORT": settings.ENABLE_GPU_SUPPORT,
             },
         },
         dumps=dumps_for_json,
@@ -486,10 +487,14 @@ async def notify_allocation(request: web.Request):
     payment_type = message.content.payment and message.content.payment.type or PaymentType.hold
 
     is_confidential = message.content.environment.trusted_execution is not None
-
-    if payment_type == PaymentType.hold and is_confidential:
-        # At the moment we will allow hold for PAYG
-        logger.debug("Confidential instance not using PAYG")
+    have_gpu = message.content.requirements and message.content.requirements.gpu is not None
+
+    if payment_type == PaymentType.hold and (is_confidential or have_gpu):
+        # Log confidential and instances with GPU support
+        if is_confidential:
+            logger.debug(f"Confidential instance {item_hash} not using PAYG")
+        if have_gpu:
+            logger.debug(f"GPU Instance {item_hash} not using PAYG")
         user_balance = await payment.fetch_balance_of_address(message.sender)
         hold_price = await payment.fetch_execution_hold_price(item_hash)
         logger.debug(f"Address {message.sender} Balance: {user_balance}, Price: {hold_price}")
diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py
index 9a9e69f3a..d377da567 100644
--- a/src/aleph/vm/pool.py
+++ b/src/aleph/vm/pool.py
@@ -5,6 +5,7 @@
 import logging
 from collections.abc import Iterable
 from datetime import datetime, timezone
+from typing import List
 
 from aleph_message.models import (
     Chain,
@@ -13,11 +14,13 @@
     Payment,
     PaymentType,
 )
+from pydantic import parse_raw_as
 
 from aleph.vm.conf import settings
 from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager
 from aleph.vm.network.hostnetwork import Network, make_ipv6_allocator
 from aleph.vm.orchestrator.metrics import get_execution_records
+from aleph.vm.resources import GpuDevice, HostGPU, get_gpu_devices
 from aleph.vm.systemd import SystemDManager
 from aleph.vm.utils import get_message_executable_content
 from aleph.vm.vm_type import VmType
@@ -41,6 +44,7 @@ class VmPool:
     snapshot_manager: SnapshotManager | None = None
     systemd_manager: SystemDManager
     creation_lock: asyncio.Lock
+    gpus: List[GpuDevice] = []
 
     def __init__(self, loop: asyncio.AbstractEventLoop):
         self.executions = {}
@@ -78,6 +82,10 @@ def setup(self) -> None:
             logger.debug("Initializing SnapshotManager ...")
             self.snapshot_manager.run_in_thread()
 
+        if settings.ENABLE_GPU_SUPPORT:
+            logger.debug("Detecting GPU devices ...")
+            self.gpus = get_gpu_devices()
+
     def teardown(self) -> None:
         """Stop the VM pool and the network properly."""
         if self.network:
@@ -109,7 +117,11 @@ async def create_a_vm(
                 self.executions[vm_hash] = execution
 
             try:
+                # First assign Host GPUs from the available
+                execution.prepare_gpus(self.get_available_gpus())
+                # Prepare VM general Resources and also the GPUs
                 await execution.prepare()
+
                 vm_id = self.get_unique_vm_id()
 
                 if self.network:
@@ -233,6 +245,9 @@ async def load_persistent_executions(self):
 
             if execution.is_running:
                 # TODO: Improve the way that we re-create running execution
+                # Load existing GPUs assigned to VMs
+                execution.gpus = parse_raw_as(List[HostGPU], saved_execution.gpus)
+                # Load and instantiate the rest of resources and already assigned GPUs
                 await execution.prepare()
                 if self.network:
                     vm_type = VmType.from_message_content(execution.message)
@@ -285,6 +300,18 @@ def get_instance_executions(self) -> Iterable[VmExecution]:
         )
         return executions or []
 
+    def get_available_gpus(self) -> List[GpuDevice]:
+        available_gpus = []
+        for gpu in self.gpus:
+            used = False
+            for _, execution in self.executions.items():
+                if execution.uses_gpu(gpu.pci_host):
+                    used = True
+                    break
+            if not used:
+                available_gpus.append(gpu)
+        return available_gpus
+
     def get_executions_by_sender(self, payment_type: PaymentType) -> dict[str, dict[str, list[VmExecution]]]:
         """Return all executions of the given type, grouped by sender and by chain."""
         executions_by_sender: dict[str, dict[str, list[VmExecution]]] = {}
diff --git a/src/aleph/vm/resources.py b/src/aleph/vm/resources.py
new file mode 100644
index 000000000..767b64906
--- /dev/null
+++ b/src/aleph/vm/resources.py
@@ -0,0 +1,111 @@
+import subprocess
+from enum import Enum
+from typing import List, Optional
+
+from aleph_message.models import HashableModel
+from pydantic import BaseModel, Extra, Field
+
+
+class HostGPU(BaseModel):
+    """Host GPU properties detail."""
+
+    pci_host: str = Field(description="GPU PCI host address")
+
+    class Config:
+        extra = Extra.forbid
+
+
+class GpuDeviceClass(str, Enum):
+    """GPU device class. Look at https://admin.pci-ids.ucw.cz/read/PD/03"""
+
+    VGA_COMPATIBLE_CONTROLLER = "0300"
+    _3D_CONTROLLER = "0302"
+
+
+class GpuDevice(HashableModel):
+    """GPU properties."""
+
+    vendor: str = Field(description="GPU vendor name")
+    device_name: str = Field(description="GPU vendor card name")
+    device_class: GpuDeviceClass = Field(
+        description="GPU device class. Look at https://admin.pci-ids.ucw.cz/read/PD/03"
+    )
+    pci_host: str = Field(description="Host PCI bus for this device")
+    device_id: str = Field(description="GPU vendor & device ids")
+
+    class Config:
+        extra = Extra.forbid
+
+
+def is_gpu_device_class(device_class: str) -> bool:
+    try:
+        GpuDeviceClass(device_class)
+        return True
+    except ValueError:
+        return False
+
+
+def get_vendor_name(vendor_id: str) -> str:
+    match vendor_id:
+        case "10de":
+            return "NVIDIA"
+        case "1002":
+            return "AMD"
+        case "8086":
+            return "Intel"
+        case _:
+            raise ValueError("Device vendor not compatible")
+
+
+def is_kernel_enabled_gpu(pci_host: str) -> bool:
+    # Get detailed info about Kernel drivers used by this device.
+    # Needs to use specifically only the kernel driver vfio-pci to be compatible for QEmu virtualization
+    result = subprocess.run(["lspci", "-s", pci_host, "-nnk"], capture_output=True, text=True, check=True)
+    details = result.stdout.split("\n")
+    if "\tKernel driver in use: vfio-pci" in details:
+        return True
+
+    return False
+
+
+def parse_gpu_device_info(line: str) -> Optional[GpuDevice]:
+    """Parse GPU device info from a line of lspci output."""
+
+    pci_host, device = line.split(' "', maxsplit=1)
+
+    if not is_kernel_enabled_gpu(pci_host):
+        return None
+
+    device_class, device_vendor, device_info = device.split('" "', maxsplit=2)
+    device_class = device_class.split("[", maxsplit=1)[1][:-1]
+
+    if not is_gpu_device_class(device_class):
+        return None
+
+    device_class = GpuDeviceClass(device_class)
+
+    vendor, vendor_id = device_vendor.rsplit(" [", maxsplit=1)
+    vendor_id = vendor_id[:-1]
+    vendor_name = get_vendor_name(vendor_id)
+    device_name = device_info.split('"', maxsplit=1)[0]
+    device_name, model_id = device_name.rsplit(" [", maxsplit=1)
+    model_id = model_id[:-1]
+    device_id = f"{vendor_id}:{model_id}"
+
+    return GpuDevice(
+        pci_host=pci_host,
+        vendor=vendor_name,
+        device_name=device_name,
+        device_class=device_class,
+        device_id=device_id,
+    )
+
+
+def get_gpu_devices() -> Optional[List[GpuDevice]]:
+    """Get GPU info using lspci command."""
+
+    result = subprocess.run(["lspci", "-mmnnn"], capture_output=True, text=True, check=True)
+    gpu_devices = list(
+        {device for line in result.stdout.split("\n") if line and (device := parse_gpu_device_info(line)) is not None}
+    )
+    return gpu_devices if gpu_devices else None
diff --git a/tests/supervisor/test_resources.py b/tests/supervisor/test_resources.py
new file mode 100644
index 000000000..fea79fe71
--- /dev/null
+++ b/tests/supervisor/test_resources.py
@@ -0,0 +1,38 @@
+from unittest import mock
+
+from aleph.vm.resources import get_gpu_devices
+
+
+def mock_is_kernel_enabled_gpu(pci_host: str) -> bool:
+    value = True if pci_host == "01:00.0" else False
+    return value
+
+
+def test_get_gpu_devices():
+    class DevicesReturn:
+        stdout: str = (
+            '00:1f.0 "ISA bridge [0601]" "Intel Corporation [8086]" "Device [7a06]" -r11 -p00 "ASUSTeK Computer Inc. [1043]" "Device [8882]"'
+            '\n00:1f.4 "SMBus [0c05]" "Intel Corporation [8086]" "Raptor Lake-S PCH SMBus Controller [7a23]" -r11 -p00 "ASUSTeK Computer Inc. [1043]" "Device [8882]"'
+            '\n00:1f.5 "Serial bus controller [0c80]" "Intel Corporation [8086]" "Raptor Lake SPI (flash) Controller [7a24]" -r11 -p00 "ASUSTeK Computer Inc. [1043]" "Device [8882]"'
+            '\n01:00.0 "VGA compatible controller [0300]" "NVIDIA Corporation [10de]" "AD104GL [RTX 4000 SFF Ada Generation] [27b0]" -ra1 -p00 "NVIDIA Corporation [10de]" "AD104GL [RTX 4000 SFF Ada Generation] [16fa]"'
+            '\n01:00.1 "Audio device [0403]" "NVIDIA Corporation [10de]" "Device [22bc]" -ra1 -p00 "NVIDIA Corporation [10de]" "Device [16fa]"'
+            '\n02:00.0 "Non-Volatile memory controller [0108]" "Samsung Electronics Co Ltd [144d]" "NVMe SSD Controller PM9A1/PM9A3/980PRO [a80a]" -p02 "Samsung Electronics Co Ltd [144d]" "NVMe SSD Controller PM9A1/PM9A3/980PRO [aa0a]"'
+        )
+
+    with mock.patch(
+        "subprocess.run",
+        return_value=DevicesReturn(),
+    ):
+        with mock.patch(
+            "aleph.vm.resources.is_kernel_enabled_gpu",
+            wraps=mock_is_kernel_enabled_gpu,
+        ):
+            expected_gpu_devices = get_gpu_devices()
+
+            print(expected_gpu_devices)
+
+            assert expected_gpu_devices[0].vendor == "NVIDIA"
+            assert expected_gpu_devices[0].device_name == "AD104GL [RTX 4000 SFF Ada Generation]"
+            assert expected_gpu_devices[0].device_class == "0300"
+            assert expected_gpu_devices[0].pci_host == "01:00.0"
+            assert expected_gpu_devices[0].device_id == "10de:27b0"
diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py
index cd32bdc7e..d94ce60f1 100644
--- a/tests/supervisor/test_views.py
+++ b/tests/supervisor/test_views.py
@@ -36,7 +36,15 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client):
 @pytest.mark.asyncio
 async def test_system_usage(aiohttp_client):
     """Test that the usage system endpoints responds. No auth needed"""
+
+    class FakeVmPool:
+        gpus = []
+
+        def get_available_gpus(self):
+            return []
+
     app = setup_webapp()
+    app["vm_pool"] = FakeVmPool()
     client = await aiohttp_client(app)
     response: web.Response = await client.get("/about/usage/system")
     assert response.status == 200
@@ -49,6 +57,13 @@ async def test_system_usage(aiohttp_client):
 @pytest.mark.asyncio
 async def test_system_usage_mock(aiohttp_client, mocker):
     """Test that the usage system endpoints response value. No auth needed"""
+
+    class FakeVmPool:
+        gpus = []
+
+        def get_available_gpus(self):
+            return []
+
     mocker.patch(
         "cpuinfo.cpuinfo.get_cpu_info",
         {
@@ -64,7 +79,9 @@ async def test_system_usage_mock(aiohttp_client, mocker):
         "psutil.cpu_count",
         lambda: 200,
     )
+
     app = setup_webapp()
+    app["vm_pool"] = FakeVmPool()
     client = await aiohttp_client(app)
     response: web.Response = await client.get("/about/usage/system")
     assert response.status == 200

From 97f07f2a8d5f8ca9db07ac13be413c12d19c9f41 Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Thu, 13 Feb 2025 10:42:17 +0100
Subject: [PATCH 938/990] Fix regression: Monitoring of payment stream (#749)

Problem: in monitoring_payment_task
the check of stream payment was not done

Regression introduced in https://github.com/aleph-im/aleph-vm/pull/726
---
 src/aleph/vm/orchestrator/tasks.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 84d9ca498..75fff2364 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -4,7 +4,6 @@
 import math
 import time
 from collections.abc import AsyncIterable
-from decimal import Decimal
 from typing import TypeVar
 
 import aiohttp
@@ -193,17 +192,17 @@ async def check_payment(pool: VmPool):
                 await pool.stop_vm(last_execution.vm_hash)
                 required_balance = await compute_required_balance(executions)
 
-        # Check if the balance held in the wallet is sufficient stream tier resources
-        for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():
-            for chain, executions in chains.items():
-                try:
-                    stream = await get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
-                    logger.debug(
-                        f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
-                    )
-                except ValueError as error:
-                    logger.error(f"Error found getting stream for chain {chain} and sender {sender}: {error}")
-                    continue
+    # Check if the balance held in the wallet is sufficient stream tier resources
+    for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():
+        for chain, executions in chains.items():
+            try:
+                stream = await get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
+                logger.debug(
+                    f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
+                )
+            except ValueError as error:
+                logger.error(f"Error found getting stream for chain {chain} and sender {sender}: {error}")
+                continue
 
             required_stream = await compute_required_flow(executions)
             logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")

From cd8443ec5612ba860b2ad5030cb01b84bbe2b9bd Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Thu, 13 Feb 2025 16:30:42 +0100
Subject: [PATCH 939/990] Solve Aleph VMs automatic restarts (#750)

Problem: On Ubuntu 24.04 OS version, a new service called `needrestart` restart some services automatically and this also resets some confidential VMs that cannot init again as we don't store the password key.

Solution: Add a rule on `needrestart` service to skip the restart on aleph services.
---
 packaging/aleph-vm/etc/needrestart/conf.d/aleph-vm.conf | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 packaging/aleph-vm/etc/needrestart/conf.d/aleph-vm.conf

diff --git a/packaging/aleph-vm/etc/needrestart/conf.d/aleph-vm.conf b/packaging/aleph-vm/etc/needrestart/conf.d/aleph-vm.conf
new file mode 100644
index 000000000..74963eebf
--- /dev/null
+++ b/packaging/aleph-vm/etc/needrestart/conf.d/aleph-vm.conf
@@ -0,0 +1,3 @@
+# Do not restart Aleph Network Services
+$nrconf{override_rc}{qr(^aleph-vm-supervisor)} = 0;
+$nrconf{override_rc}{qr(^aleph-vm-controller@.*\.service$)} = 0;

From cc47d589fc524ffd29c7ab3fb42392e627ed85a0 Mon Sep 17 00:00:00 2001
From: Gustavo Delfino 
Date: Mon, 17 Feb 2025 08:56:13 -0500
Subject: [PATCH 940/990] Minor typo in README.md fixed (#752)

Update README.md

Minor typo fixed
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7e21abcb7..e79d39388 100644
--- a/README.md
+++ b/README.md
@@ -111,7 +111,7 @@ Unless your focus is developing the VM-Connector, using the Docker image is easi
    directly during development.
 
 ## Testing
-See [Testinc doc](./TESTING.md)
+See [Testing doc](./TESTING.md)
 
 ## Code Formatting and Linting
 

From 7c9392cb51eebdf75dc3342584ea73f3aced948a Mon Sep 17 00:00:00 2001
From: Olivier Le Thanh Duong 
Date: Mon, 17 Feb 2025 15:59:58 +0100
Subject: [PATCH 941/990] Garbate collector: Free disk space from inactive VM
 (#746)

* Garbate collector: Free disk space from inactive VM

Add a script to manually  list and remove volume linked to inactive VM.
It fetches data from the scheduler and pyaleph main's node as to fetch information on the status of the VM.
Then display them to the user to determine if they can be removed safely.

JIRA ticket ALEPH-37

* Add diagnostic vm to ignore list
---
 src/aleph/vm/garbage_collector.py | 147 ++++++++++++++++++++++++++++++
 1 file changed, 147 insertions(+)
 create mode 100644 src/aleph/vm/garbage_collector.py

diff --git a/src/aleph/vm/garbage_collector.py b/src/aleph/vm/garbage_collector.py
new file mode 100644
index 000000000..4bc6c34b8
--- /dev/null
+++ b/src/aleph/vm/garbage_collector.py
@@ -0,0 +1,147 @@
+"""Free disk space by removing unused volume from the hard drive to free up
+
+
+This script allow to manually  list and remove volume linked to inactive VM
+It fetches data from the scheduler and pyaleph main's node as to fetch information on the status of the VM.
+Then display them to the user to determine if they can be removed safely.
+
+Requires to be run as root.
+"""
+
+import os
+import subprocess
+from pathlib import Path
+
+import requests
+
+# following hashes are used in tests or debug VM, we can ignore them.
+TEST_HASHES = [
+    "fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_fake_vm_",
+    "cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe",
+    "decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca",
+    "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace",
+    "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8",
+]
+
+api_server = [
+    "https://api1.aleph.im",
+    "https://api2.aleph.im",
+    "https://api3.aleph.im",
+    # 'https://official.aleph.cloud',
+]
+
+endpoint = "/api/v0/messages/"
+
+
+def check_api(item_hash):
+    """Check on which api the ITEM_HASH msg is available."""
+    for api in api_server:
+        response = requests.get(api + endpoint + item_hash)
+        print(api + " ", end="")
+        print(response.status_code, end="")
+        j = response.json()
+        print(" " + j["status"], end="")
+        print()
+
+
+p = Path("/var/lib/aleph/vm/volumes/persistent")
+# print current size
+os.system(" ".join(["df", "-h", str(p)]))
+
+# Before anything check that we can reach the api server and the scheduler server
+res = requests.get("https://api2.aleph.im/api/v0/info/public.json")
+assert res.status_code == 200
+res = requests.get("https://scheduler.api.aleph.cloud/api/v0/plan")
+assert res.status_code == 200
+
+volume_dirs = list(p.glob("*"))
+for i, f in enumerate(reversed(volume_dirs)):
+    if not f.is_dir():
+        continue
+    item_hash = f.name
+    print(f"= {i}/{len(volume_dirs) -1} {item_hash}")
+    if item_hash in TEST_HASHES:
+        print("Test VM, skipping")
+        continue
+
+    res = requests.get(f"https://api2.aleph.im/api/v0/messages/{item_hash}")
+
+    if res.status_code == 404:
+        print("Not found on API server")
+        continue
+    message = res.json()
+    message_status = message.get("status")
+    # if message_status == "forgotten" or message_status == "rejected":
+    #     print(f"{item_hash} status: {j.message_status('status')}")
+    #     continue
+    # print(f"{item_hash} status: {j.message_status('status')}")
+    sender = message["message"]["sender"]
+    print(f"Sender {sender}. State: {message_status}")
+    if not message["message"]["type"] == "INSTANCE":
+        print("Type: ", message["message"]["type"], "not an instance")
+        continue
+    scheduler_res = requests.get(f"https://scheduler.api.aleph.cloud/api/v0/allocation/{item_hash}")
+    schedule = None
+
+    if scheduler_res.status_code == 404:
+        print("Not found on scheduler plan")
+    else:
+        schedule = scheduler_res.json()
+        print(f"scheduled on {schedule['node']['node_id']}")
+
+    balance = requests.get(f"https://api2.aleph.im/api/v0/addresses/{sender}/balance").json()
+    print(f"User balance: {balance['balance']:.2f}, locked amount {balance['locked_amount']:.2f}")
+    # print(balance)
+
+    # check if process is still running
+
+    proc_ret = subprocess.run(
+        f"systemctl status aleph-vm-controller@{item_hash}.service --no-pager",
+        shell=True,
+        capture_output=True,
+    )
+    exit_code = proc_ret.returncode
+    if exit_code == 0:
+        proc_status = "running"
+    elif exit_code == 3:
+        proc_status = "stopped"
+    else:
+        proc_status = "error"
+        print("Unknown process state", exit_code)
+    # to remove
+
+    if proc_status != "running":
+        # not running and forgotten
+
+        if message_status == "forgotten" or message_status == "rejected":
+            print("Recommendation: remove, process not running and message rejected or forgotten")
+        else:
+            print("Process stopped")
+            # print(f"balances: {balance['balance']}, locked amount {balance['locked_amount']}'")
+
+        while True:
+            inp = input("Do you want to delete y/n ? More info (h) [n] ").lower()
+            if inp in ["y", "yes"]:
+                os.system(f"dmsetup remove {item_hash}_base")
+                os.system(f"dmsetup remove {item_hash}_rootfs")
+                os.system(f"rm -r {f.absolute()}")
+                # close all loop device
+                os.system(
+                    "sudo losetup -l | grep 'persistent' | grep deleted | awk  '{print $1}' | sudo xargs losetup -d {}"
+                )
+                break
+            elif inp == "h":
+                print(proc_ret.stdout.decode())
+                check_api(item_hash)
+                print(f"https://api2.aleph.im/api/v0/messages/{item_hash}")
+                print(f"https://api2.aleph.im/api/v0/addresses/{sender}/balance")
+            else:
+                break
+
+    else:
+        print("process is running, do not delete")
+
+
+# print current size.
+print("Size after")
+os.system(" ".join(["df", "-h", str(p)]))

From 01ff0c15bdd29081f41b9ee3daebe1a9ee714a30 Mon Sep 17 00:00:00 2001
From: nesitor 
Date: Mon, 17 Feb 2025 16:41:57 +0100
Subject: [PATCH 942/990] Implemented GPU compatibility system (#747)

* Feature: Added options to get GPU compatibilities from a settings aggregate.

* Fix: Refactored to also return the model name from the aggregate and use the same device_id format.

* Fix: Include GPU list and move the VM egress IPv6 check on the connectivity check to start notifying the users about the next requirement.

* Fix: Solved code quality issues.

* Fix: Put definitive settings aggregate address

* Fix: Solved issue with type casting and moved the aggregate check.

* Check community payment flow (#751)

* Implement community payment check WIP

* isort

* Check community flow at allocation

* Community flow  : fix after testing

* mod Use singleton for the Setting Aggregate

* fix test

* Implement community wallet start time

---------

Co-authored-by: Olivier Le Thanh Duong 
---
 src/aleph/vm/conf.py                          |   6 +-
 src/aleph/vm/orchestrator/tasks.py            |  60 ++++-
 src/aleph/vm/orchestrator/utils.py            | 102 ++++++++
 src/aleph/vm/orchestrator/views/__init__.py   |  47 +++-
 .../vm/orchestrator/views/static/helpers.js   |  36 +++
 .../vm/orchestrator/views/static/main.css     |   4 +
 .../orchestrator/views/templates/index.html   |  73 ++++--
 src/aleph/vm/pool.py                          |   3 +
 src/aleph/vm/resources.py                     |  33 +++
 tests/supervisor/test_checkpayment.py         | 226 ++++++++++++++++++
 10 files changed, 548 insertions(+), 42 deletions(-)
 create mode 100644 src/aleph/vm/orchestrator/utils.py
 create mode 100644 tests/supervisor/test_checkpayment.py

diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py
index 264be819e..f33e02f6c 100644
--- a/src/aleph/vm/conf.py
+++ b/src/aleph/vm/conf.py
@@ -9,7 +9,7 @@
 from os.path import abspath, exists, isdir, isfile, join
 from pathlib import Path
 from subprocess import CalledProcessError, check_output
-from typing import Any, Literal, NewType
+from typing import Any, List, Literal, NewType
 
 from aleph_message.models import Chain
 from aleph_message.models.execution.environment import HypervisorType
@@ -280,8 +280,10 @@ class Settings(BaseSettings):
         description="Enable GPU pass-through support to VMs, only allowed for QEmu hypervisor",
     )
 
-    # Tests on programs
+    # Settings to get from the network aggregates
+    SETTINGS_AGGREGATE_ADDRESS: str = "0xFba561a84A537fCaa567bb7A2257e7142701ae2A"
 
+    # Tests on programs
     FAKE_DATA_PROGRAM: Path | None = None
     BENCHMARK_FAKE_DATA_PROGRAM = Path(abspath(join(__file__, "../../../../examples/example_fastapi")))
 
diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py
index 75fff2364..9819ffcf2 100644
--- a/src/aleph/vm/orchestrator/tasks.py
+++ b/src/aleph/vm/orchestrator/tasks.py
@@ -4,6 +4,7 @@
 import math
 import time
 from collections.abc import AsyncIterable
+from decimal import Decimal
 from typing import TypeVar
 
 import aiohttp
@@ -19,6 +20,10 @@
 from yarl import URL
 
 from aleph.vm.conf import settings
+from aleph.vm.orchestrator.utils import (
+    get_community_wallet_address,
+    is_after_community_wallet_start,
+)
 from aleph.vm.pool import VmPool
 from aleph.vm.utils import create_task_log_exceptions
 
@@ -35,6 +40,7 @@
 logger = logging.getLogger(__name__)
 
 Value = TypeVar("Value")
+COMMUNITY_STREAM_RATIO = Decimal(0.2)
 
 
 async def retry_generator(generator: AsyncIterable[Value], max_seconds: int = 8) -> AsyncIterable[Value]:
@@ -154,6 +160,7 @@ async def monitor_payments(app: web.Application):
         try:
             logger.debug("Monitoring balances task running")
             await check_payment(pool)
+            logger.debug("Monitoring balances task ended")
         except Exception as e:
             # Catch all exceptions as to never stop the task.
             logger.warning(f"check_payment failed {e}", exc_info=True)
@@ -191,31 +198,62 @@ async def check_payment(pool: VmPool):
                 logger.debug(f"Stopping {last_execution} due to insufficient balance")
                 await pool.stop_vm(last_execution.vm_hash)
                 required_balance = await compute_required_balance(executions)
+    community_wallet = await get_community_wallet_address()
+    if not community_wallet:
+        logger.error("Monitor payment ERROR: No community wallet set. Cannot check community payment")
 
     # Check if the balance held in the wallet is sufficient stream tier resources
     for sender, chains in pool.get_executions_by_sender(payment_type=PaymentType.superfluid).items():
         for chain, executions in chains.items():
             try:
                 stream = await get_stream(sender=sender, receiver=settings.PAYMENT_RECEIVER_ADDRESS, chain=chain)
+
                 logger.debug(
-                    f"Get stream flow from Sender {sender} to Receiver {settings.PAYMENT_RECEIVER_ADDRESS} of {stream}"
+                    f"Stream flow from {sender} to {settings.PAYMENT_RECEIVER_ADDRESS} = {stream} {chain.value}"
                 )
+            except ValueError as error:
+                logger.error(f"Error found getting stream for chain {chain} and sender {sender}: {error}")
+                continue
+            try:
+                community_stream = await get_stream(sender=sender, receiver=community_wallet, chain=chain)
+                logger.debug(f"Stream flow from {sender} to {community_wallet} (community) : {stream} {chain}")
+
             except ValueError as error:
                 logger.error(f"Error found getting stream for chain {chain} and sender {sender}: {error}")
                 continue
 
-            required_stream = await compute_required_flow(executions)
-            logger.debug(f"Required stream for Sender {sender} executions: {required_stream}")
-            # Stop executions until the required stream is reached
-            while (stream + settings.PAYMENT_BUFFER) < required_stream:
-                try:
-                    last_execution = executions.pop(-1)
-                except IndexError:  # Empty list
-                    logger.debug("No execution can be maintained due to insufficient stream")
+            while executions:
+                executions_with_community = [
+                    execution
+                    for execution in executions
+                    if await is_after_community_wallet_start(execution.times.started_at)
+                ]
+
+                required_stream = await compute_required_flow(executions_with_community)
+                executions_without_community = [
+                    execution
+                    for execution in executions
+                    if not await is_after_community_wallet_start(execution.times.started_at)
+                ]
+                logger.info("flow community %s", executions_with_community)
+                logger.info("flow without community %s", executions_without_community)
+                required_stream_without_community = await compute_required_flow(executions_without_community)
+
+                required_crn_stream = required_stream * (1 - COMMUNITY_STREAM_RATIO) + required_stream_without_community
+                required_community_stream = required_stream * COMMUNITY_STREAM_RATIO
+                logger.debug(
+                    f"Stream for senders {sender} {len(executions)} executions.  CRN : {stream} /  {required_crn_stream}."
+                    f"Community: {community_stream} / {required_community_stream}"
+                )
+                # Can pay all executions
+                if (stream + settings.PAYMENT_BUFFER) > required_crn_stream and (
+                    community_stream + settings.PAYMENT_BUFFER
+                ) > required_community_stream:
                     break
-                logger.debug(f"Stopping {last_execution} due to insufficient stream")
+                # Stop executions until the required stream is reached
+                last_execution = executions.pop(-1)
+                logger.info(f"Stopping {last_execution} of {sender} due to insufficient stream")
                 await pool.stop_vm(last_execution.vm_hash)
-                required_stream = await compute_required_flow(executions)
 
 
 async def start_payment_monitoring_task(app: web.Application):
diff --git a/src/aleph/vm/orchestrator/utils.py b/src/aleph/vm/orchestrator/utils.py
new file mode 100644
index 000000000..17dcbca03
--- /dev/null
+++ b/src/aleph/vm/orchestrator/utils.py
@@ -0,0 +1,102 @@
+from datetime import datetime, timedelta, timezone
+from logging import getLogger
+from typing import Any, TypedDict
+
+import aiohttp
+
+from aleph.vm.conf import settings
+
+logger = getLogger(__name__)
+
+
+class AggregateSettingsDict(TypedDict):
+    compatible_gpus: list[Any]
+    community_wallet_address: str
+    community_wallet_timestamp: int
+
+
+LAST_AGGREGATE_SETTINGS: AggregateSettingsDict | None = None
+LAST_AGGREGATE_SETTINGS_FETCHED_AT: datetime | None = None
+
+
+async def fetch_aggregate_settings() -> AggregateSettingsDict | None:
+    """
+    Get the settings Aggregate dict from the PyAleph API Aggregate.
+
+    API Endpoint:
+        GET /api/v0/aggregates/{address}.json?keys=settings
+
+    For more details, see the PyAleph API documentation:
+    https://github.com/aleph-im/pyaleph/blob/master/src/aleph/web/controllers/routes.py#L62
+    """
+    async with aiohttp.ClientSession() as session:
+        url = f"{settings.API_SERVER}/api/v0/aggregates/{settings.SETTINGS_AGGREGATE_ADDRESS}.json?keys=settings"
+        logger.info(f"Fetching settings aggregate from {url}")
+        resp = await session.get(url)
+
+        # Raise an error if the request failed
+        resp.raise_for_status()
+
+        resp_data = await resp.json()
+        return resp_data["data"]["settings"]
+
+
+async def update_aggregate_settings():
+    global LAST_AGGREGATE_SETTINGS  # noqa: PLW0603
+    global LAST_AGGREGATE_SETTINGS_FETCHED_AT  # noqa: PLW0603
+
+    LAST_AGGREGATE_SETTINGS = await fetch_aggregate_settings()
+    if (
+        not LAST_AGGREGATE_SETTINGS
+        or LAST_AGGREGATE_SETTINGS_FETCHED_AT
+        and datetime.now(tz=timezone.utc) - LAST_AGGREGATE_SETTINGS_FETCHED_AT > timedelta(minutes=1)
+    ):
+        try:
+            aggregate = await fetch_aggregate_settings()
+            LAST_AGGREGATE_SETTINGS = aggregate
+            LAST_AGGREGATE_SETTINGS_FETCHED_AT = datetime.now(tz=timezone.utc)
+
+        except Exception:
+            logger.exception("Failed to fetch aggregate settings")
+
+
+async def get_aggregate_settings() -> AggregateSettingsDict | None:
+    """The settings aggregate is a special aggregate  used to share some common settings for VM setup
+
+    Ensure the cached version is up to date and return it"""
+    await update_aggregate_settings()
+
+    if not LAST_AGGREGATE_SETTINGS:
+        logger.error("No setting aggregate")
+    return LAST_AGGREGATE_SETTINGS
+
+
+async def get_community_wallet_address() -> str | None:
+    setting_aggr = await get_aggregate_settings()
+    return setting_aggr and setting_aggr.get("community_wallet_address")
+
+
+async def get_community_wallet_start() -> datetime:
+    """Community wallet start time.
+
+    After this timestamp. New PAYG must include a payment to the community wallet"""
+    setting_aggr = await get_aggregate_settings()
+    if setting_aggr is None or "community_wallet_timestamp" not in setting_aggr:
+        return datetime.now(tz=timezone.utc)
+    timestamp = setting_aggr["community_wallet_timestamp"]
+    start_datetime = datetime.fromtimestamp(timestamp, tz=timezone.utc)
+    return start_datetime
+
+
+async def is_after_community_wallet_start(dt: datetime | None = None) -> bool:
+    """Community wallet start time"""
+    if not dt:
+        dt = datetime.now(tz=timezone.utc)
+    start_dt = await get_community_wallet_start()
+    return dt > start_dt
+
+
+def get_compatible_gpus() -> list[Any]:
+    if not LAST_AGGREGATE_SETTINGS:
+        return []
+    return LAST_AGGREGATE_SETTINGS["compatible_gpus"]
diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py
index 899a038f8..6e9460d1c 100644
--- a/src/aleph/vm/orchestrator/views/__init__.py
+++ b/src/aleph/vm/orchestrator/views/__init__.py
@@ -1,5 +1,4 @@
 import binascii
-import contextlib
 import logging
 from decimal import Decimal
 from hashlib import sha256
@@ -8,7 +7,6 @@
 from pathlib import Path
 from secrets import compare_digest
 from string import Template
-from typing import Optional
 
 import aiodns
 import aiohttp
@@ -26,7 +24,7 @@
 from aleph.vm.controllers.firecracker.program import FileTooLargeError
 from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInitError
 from aleph.vm.orchestrator import payment, status
-from aleph.vm.orchestrator.chain import STREAM_CHAINS, ChainInfo
+from aleph.vm.orchestrator.chain import STREAM_CHAINS
 from aleph.vm.orchestrator.custom_logs import set_vm_for_logging
 from aleph.vm.orchestrator.messages import try_get_message
 from aleph.vm.orchestrator.metrics import get_execution_records
@@ -39,6 +37,12 @@
 from aleph.vm.orchestrator.pubsub import PubSub
 from aleph.vm.orchestrator.resources import Allocation, VMNotification
 from aleph.vm.orchestrator.run import run_code_on_request, start_persistent_vm
+from aleph.vm.orchestrator.tasks import COMMUNITY_STREAM_RATIO
+from aleph.vm.orchestrator.utils import (
+    get_community_wallet_address,
+    is_after_community_wallet_start,
+    update_aggregate_settings,
+)
 from aleph.vm.orchestrator.views.host_status import (
     check_dns_ipv4,
     check_dns_ipv6,
@@ -468,6 +472,7 @@ async def update_allocations(request: web.Request):
 @cors_allow_all
 async def notify_allocation(request: web.Request):
     """Notify instance allocation, only used for Pay as you Go feature"""
+    await update_aggregate_settings()
     try:
         data = await request.json()
         vm_notification = VMNotification.parse_obj(data)
@@ -526,16 +531,44 @@ async def notify_allocation(request: web.Request):
             raise web.HTTPPaymentRequired(reason="Empty payment stream for this instance")
 
         required_flow: Decimal = await fetch_execution_flow_price(item_hash)
-
-        if active_flow < required_flow:
+        community_wallet = await get_community_wallet_address()
+        required_crn_stream: Decimal
+        required_community_stream: Decimal
+        if await is_after_community_wallet_start() and community_wallet:
+            required_crn_stream = required_flow * (1 - COMMUNITY_STREAM_RATIO)
+            required_community_stream = required_flow * COMMUNITY_STREAM_RATIO
+        else:  # No community wallet payment
+            required_crn_stream = required_flow
+            required_community_stream = Decimal(0)
+
+        if active_flow < (required_crn_stream - settings.PAYMENT_BUFFER):
             active_flow_per_month = active_flow * 60 * 60 * 24 * (Decimal("30.41666666666923904761904784"))
-            required_flow_per_month = required_flow * 60 * 60 * 24 * Decimal("30.41666666666923904761904784")
+            required_flow_per_month = required_crn_stream * 60 * 60 * 24 * Decimal("30.41666666666923904761904784")
             return web.HTTPPaymentRequired(
                 reason="Insufficient payment stream",
                 text="Insufficient payment stream for this instance\n\n"
-                f"Required: {required_flow_per_month} / month (flow = {required_flow})\n"
+                f"Required: {required_flow_per_month} / month (flow = {required_crn_stream})\n"
                 f"Present: {active_flow_per_month} / month (flow = {active_flow})",
             )
+
+        if community_wallet and required_community_stream:
+            community_flow: Decimal = await get_stream(
+                sender=message.sender,
+                receiver=community_wallet,
+                chain=message.content.payment.chain,
+            )
+            if community_flow < (required_community_stream - settings.PAYMENT_BUFFER):
+                active_flow_per_month = community_flow * 60 * 60 * 24 * (Decimal("30.41666666666923904761904784"))
+                required_flow_per_month = (
+                    required_community_stream * 60 * 60 * 24 * Decimal("30.41666666666923904761904784")
+                )
+                return web.HTTPPaymentRequired(
+                    reason="Insufficient payment stream to community",
+                    text="Insufficient payment stream for community \n\n"
+                    f"Required: {required_flow_per_month} / month (flow = {required_community_stream})\n"
+                    f"Present: {active_flow_per_month} / month (flow = {community_flow})\n"
+                    f"Address: {community_wallet}",
+                )
     else:
         return web.HTTPBadRequest(reason="Invalid payment method")
 
diff --git a/src/aleph/vm/orchestrator/views/static/helpers.js b/src/aleph/vm/orchestrator/views/static/helpers.js
index 8644a11aa..f7e9925dd 100644
--- a/src/aleph/vm/orchestrator/views/static/helpers.js
+++ b/src/aleph/vm/orchestrator/views/static/helpers.js
@@ -53,6 +53,42 @@ async function fetchHostCheckStatus () {
     return res;
 }
 
+async function fetchHostSystemUsage () {
+    const q = await fetch('/about/usage/system');
+    let res = {
+        status: q.status,
+        details: []
+    }
+    if(q.ok){
+        const answer = await q.json();
+        const gpu_devices = answer.gpu.devices;
+        if (gpu_devices.length <= 0) {
+            res.status = "No GPUs detected";
+        }else{
+            res.status = "
                ";
+            for (const gpu_device of gpu_devices){
+                let compatible_str = " is compatible ✅";
+                if (!gpu_device.compatible) {
+                    compatible_str = " isn't compatible ❌";
+                }
+                res.status += "
              • " + gpu_device.vendor + " | " + gpu_device.device_name + "" + compatible_str + "
                • ";
+            }
+            res.status += "
              ";
+        }
+    }
+    else {
+        switch(Number(q.status)){
+            case 500:
+                res.status = "Getting Node usage failed ❌";
+                break;
+            default:
+                res.status = q.status;
+        }
+    }
+
+    return res;
+}
+
 function objectToString (obj) {
     return Object.entries(obj).reduce((acc, [k, v]) => acc + `
            • ${k}: ${v}
              • \n`, '');
 }
diff --git a/src/aleph/vm/orchestrator/views/static/main.css b/src/aleph/vm/orchestrator/views/static/main.css
index bf2cbbf85..2b14d4b60 100644
--- a/src/aleph/vm/orchestrator/views/static/main.css
+++ b/src/aleph/vm/orchestrator/views/static/main.css
@@ -5,6 +5,10 @@ body {
     max-width: 800px;
 }
 
+details {
+    margin-top: 30px;
+}
+
 main {
     width: 90vw;
     margin: 2vh auto;
diff --git a/src/aleph/vm/orchestrator/views/templates/index.html b/src/aleph/vm/orchestrator/views/templates/index.html
index 09715acb5..8222fb8bd 100644
--- a/src/aleph/vm/orchestrator/views/templates/index.html
+++ b/src/aleph/vm/orchestrator/views/templates/index.html
@@ -3,8 +3,8 @@
 
     
     Aleph.im Compute Node
-    
-    
+    
+    
 
 
     
              
@@ -112,6 +112,25 @@ 
              IPv4
              
                 
                
                 
                IPv6
                
                 
                  
+                
                  VM Egress IPv6
                  
+                
                  
+                    VM Egress IPv6 is a test to check if virtual machines are able to connect to the IPv6 internet.
+                    Enabling VM IPv6 Egress requires a specific configuration that is not applied automatically. It is not yet
+                    required to run programs inside, but it's required to run instance, so will be mandatory soon.
+                
                  
+                
                  
+                    
                  
+                        VM Egress IPv6
+                        
+                        is ...
+                        
+                            
+                            
+                            
+                        
+                    
+                    
                  
+                
                  
             
                  
             
         
 
+        
                  
+            
                  GPUs
                  
+            
                  
+                Loading GPU list
+                
+                    ...
+                    
+                        
+                        
+                        
+                    
+                
+            
                  
+        
                  
+
         
                  
             ℹ️ More information
 
@@ -138,25 +172,6 @@ 
                  Latest metrics
                  
                 
                    
             
 
-            
                    VM Egress IPv6
                    
-            
                    
-                VM Egress IPv6 is a test to check if virtual machines are able to connect to the IPv6 internet.
-                Enabling VM IPv6 Egress requires a specific configuration that is not applied automatically. It is not yet
-                required to run virtual machines.
-            
                    
-            
                    
-                
                    
-                    VM Egress IPv6
-                    
-                        is ...
-                        
-                            
-                            
-                            
-                        
-                    
-                
                    
-            
                    
             
                    APIs
                    
             
                    
                 Host status check API: /status/check/host
@@ -214,7 +229,7 @@ 
                    Version
                    
     
 
 
-    
+