FIX: Integration Tests and add cleanup job for hetzner resources

Author: NeoPlays

.github/workflows/test-integration.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v5
       - uses: actions/setup-node@v4
         with:
-          node-version: "19"
+          node-version: 24
       - run: npm install
         working-directory: ./launcher
       - id: get-import-tests
@@ -47,7 +47,7 @@ jobs:
       - uses: actions/checkout@v5
       - uses: actions/setup-node@v4
         with:
-          node-version: "19"
+          node-version: 24
       - run: npm install
         working-directory: ./launcher
       - run: npm run test ${{ matrix.test.name }}
@@ -69,7 +69,7 @@ jobs:
       - uses: actions/checkout@v5
       - uses: actions/setup-node@v4
         with:
-          node-version: "19"
+          node-version: 24
       - run: npm install
         working-directory: ./launcher
       - run: npm run test ${{ matrix.test.name }}
@@ -90,11 +90,32 @@ jobs:
       - uses: actions/checkout@v5
       - uses: actions/setup-node@v4
         with:
-          node-version: "19"
+          node-version: 24
       - run: npm install
         working-directory: ./launcher
       - run: npm run test ${{ matrix.test.name }}
         working-directory: ./launcher
         env:
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
           IS_DEV: "true"
+
+  Cleanup:
+    runs-on: ubuntu-22.04
+    name: Cleanup
+    needs:
+      - Validator-Import-test
+      - Execution-Client-test
+      - Other-Integration-test
+    if: ${{ always() }} # run even if dependencies failed or were cancelled
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+      - run: npm install
+        working-directory: ./launcher
+      - run: npm run test:cleanup
+        working-directory: ./launcher
+        env:
+          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
+          IS_DEV: "true"

launcher/package.json

@@ -24,6 +24,7 @@
     "test:other": "jest --silent --testPathPattern 'tests/(?!.*Service).*\\.int\\.js$'",
     "test:beacon": "jest --silent --testPathPattern 'tests/(?=.*BeaconService).*\\.int\\.js$'",
     "test:service": "jest --silent --testPathPattern 'tests/(?=.*Service)(?!.*Beacon).*\\.int\\.js$'",
+    "test:cleanup": "node src/backend/tests/integration/Cleanup.js",
     "watch:css": "npx tailwindcss -i ./src/main.css -o ./public/output.css --watch",
     "lint:fix": "eslint src --fix",
     "stereum": "concurrently \"npm:electron:serve\" \"npm:watch:css\"",
@@ -111,4 +112,4 @@
     "type": "git",
     "url": "git@github.com:stereum-dev/ethereum-node.git"
   }
-}
+}

launcher/src/backend/tests/integration/Cleanup.js

@@ -0,0 +1,79 @@
+import axios from "axios";
+
+async function cleanup() {
+  const apiToken = process.env.HCLOUD_TOKEN;
+  if (!apiToken) {
+    console.error("HCLOUD_TOKEN environment variable is not set.");
+    return;
+  }
+
+  const config = {
+    headers: {
+      Authorization: `Bearer ${apiToken}`,
+    },
+  };
+
+  // Cleanup servers with names containing "integration-test"
+  try {
+    const response = await axios.get("https://api.hetzner.cloud/v1/servers", config);
+    const servers = response.data.servers;
+    let currentPage = response.data.meta.pagination.page + 1;
+    const lastPage = response.data.meta.pagination.last_page;
+
+    while (currentPage <= lastPage) {
+      const pagedResponse = await axios.get(`https://api.hetzner.cloud/v1/servers?page=${currentPage}`, config);
+      servers.push(...pagedResponse.data.servers);
+      currentPage++;
+    }
+
+    const integrationTestServers = servers.filter((server) => server.name.includes("integration-test"));
+
+    for (const server of integrationTestServers) {
+      try {
+        await axios.delete(`https://api.hetzner.cloud/v1/servers/${server.id}`, config);
+        console.log(`Deleted server: ${server.name} (ID: ${server.id})`);
+      } catch (deleteError) {
+        console.error(`Failed to delete server: ${server.name} (ID: ${server.id})`, deleteError);
+      }
+    }
+
+    if (integrationTestServers.length === 0) {
+      console.log("No integration-test servers found.");
+    }
+  } catch (error) {
+    console.error("Error fetching servers:", error);
+  }
+
+  // Cleanup SSH keys with names containing "integration-test"
+  try {
+    const response = await axios.get("https://api.hetzner.cloud/v1/ssh_keys", config);
+    const sshKeys = response.data.ssh_keys;
+    let currentPage = response.data.meta.pagination.page + 1;
+    const lastPage = response.data.meta.pagination.last_page;
+
+    while (currentPage <= lastPage) {
+      const pagedResponse = await axios.get(`https://api.hetzner.cloud/v1/ssh_keys?page=${currentPage}`, config);
+      sshKeys.push(...pagedResponse.data.ssh_keys);
+      currentPage++;
+    }
+
+    const integrationTestKeys = sshKeys.filter((key) => key.name.includes("integration-test"));
+
+    for (const key of integrationTestKeys) {
+      try {
+        await axios.delete(`https://api.hetzner.cloud/v1/ssh_keys/${key.id}`, config);
+        console.log(`Deleted SSH key: ${key.name} (ID: ${key.id})`);
+      } catch (deleteError) {
+        console.error(`Failed to delete SSH key: ${key.name} (ID: ${key.id})`, deleteError);
+      }
+    }
+
+    if (integrationTestKeys.length === 0) {
+      console.log("No integration-test SSH keys found.");
+    }
+  } catch (error) {
+    console.error("Error fetching SSH keys:", error);
+  }
+}
+
+cleanup();

launcher/src/backend/tests/integration/L2GethService.int.js

@@ -11,10 +11,10 @@ jest.setTimeout(500000);
 
 test("l2geth installation", async () => {
   const testServer = new HetznerServer();
-  const keyResponse = await testServer.createSSHKey("L2Geth--integration-test--ubuntu-2204");
+  const keyResponse = await testServer.createSSHKey("Layer2-Geth--integration-test--ubuntu-2204");
 
   const serverSettings = {
-    name: "L2Geth--integration-test--ubuntu-2204",
+    name: "Layer2-Geth--integration-test--ubuntu-2204",
     image: "ubuntu-22.04",
     server_type: "cpx21",
     start_after_create: true,

launcher/src/backend/tests/integration/NethermindService.int.js

@@ -72,9 +72,11 @@ test("nethermind installationm", async () => {
     await testServer.Sleep(30000);
     status = await nodeConnection.sshService.exec(`docker logs stereum-${executionClient.id}`);
     if (
-      /Nethermind initialization completed/.test(status.stdout) &&
+      /Initialization Completed/.test(status.stdout) &&
       /Peers/.test(status.stdout) &&
-      /http:\/\/0\.0\.0\.0:8545 ; http:\/\/0\.0\.0\.0:8546 ; http:\/\/0\.0\.0\.0:8551/.test(status.stdout)
+      /http:\/\/0\.0\.0\.0:8545/.test(status.stdout) &&
+      /http:\/\/0\.0\.0\.0:8546/.test(status.stdout) &&
+      /http:\/\/0\.0\.0\.0:8551/.test(status.stdout)
     ) {
       condition = true;
     }
@@ -98,7 +100,9 @@ test("nethermind installationm", async () => {
     expect((docker.stdout.match(new RegExp("Up", "g")) || []).length).toBe(1);
   }
 
-  expect(status.stdout).toMatch(/Nethermind initialization completed/);
+  expect(status.stdout).toMatch(/Initialization Completed/);
   expect(status.stdout).toMatch(/Peers/);
-  expect(status.stdout).toMatch(/http:\/\/0\.0\.0\.0:8545 ; http:\/\/0\.0\.0\.0:8546 ; http:\/\/0\.0\.0\.0:8551/);
+  expect(status.stdout).toMatch(/http:\/\/0\.0\.0\.0:8545/);
+  expect(status.stdout).toMatch(/http:\/\/0\.0\.0\.0:8546/);
+  expect(status.stdout).toMatch(/http:\/\/0\.0\.0\.0:8551/);
 });

launcher/src/backend/tests/integration/OpErigonService.int.js

@@ -11,10 +11,10 @@ jest.setTimeout(1200000);
 
 test("op-erigon installation", async () => {
   const testServer = new HetznerServer();
-  const keyResponse = await testServer.createSSHKey("OpErigon--integration-test--ubuntu-2204");
+  const keyResponse = await testServer.createSSHKey("Optimism-Erigon--integration-test--ubuntu-2204");
 
   const serverSettings = {
-    name: "OpErigon--integration-test--ubuntu-2204",
+    name: "Optimism-Erigon--integration-test--ubuntu-2204",
     image: "ubuntu-22.04",
     server_type: "cpx21",
     start_after_create: true,

launcher/src/backend/tests/integration/OpGethService.int.js

@@ -11,10 +11,10 @@ jest.setTimeout(500000);
 
 test("op-geth installation", async () => {
   const testServer = new HetznerServer();
-  const keyResponse = await testServer.createSSHKey("OpGeth--integration-test--ubuntu-2204");
+  const keyResponse = await testServer.createSSHKey("Optimism-Geth--integration-test--ubuntu-2204");
 
   const serverSettings = {
-    name: "OpGeth--integration-test--ubuntu-2204",
+    name: "Optimism-Geth--integration-test--ubuntu-2204",
     image: "ubuntu-22.04",
     server_type: "cpx21",
     start_after_create: true,

launcher/src/backend/tests/integration/OpRethService.int.js

@@ -11,10 +11,10 @@ jest.setTimeout(500000);
 
 test("op-reth installation", async () => {
   const testServer = new HetznerServer();
-  const keyResponse = await testServer.createSSHKey("OpReth--integration-test--ubuntu-2204");
+  const keyResponse = await testServer.createSSHKey("Optimism-Reth--integration-test--ubuntu-2204");
 
   const serverSettings = {
-    name: "OpReth--integration-test--ubuntu-2204",
+    name: "Optimism-Reth--integration-test--ubuntu-2204",
     image: "ubuntu-22.04",
     server_type: "cpx21",
     start_after_create: true,