Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 104 additions & 0 deletions .github/scripts/setup-arm64-runner.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
#!/usr/bin/env bash
#
# Setup script for an ARM64 self-hosted GitHub Actions runner.
# Run this on a fresh ARM64 Ubuntu 22.04/24.04 machine with KVM support.
#
# Prerequisites:
# - ARM64 Linux host (Graviton, Ampere, etc.)
# - KVM enabled (/dev/kvm accessible)
# - At least 8GB RAM (for hugepage allocation)
# - Root access
#
# Usage:
# sudo ./setup-arm64-runner.sh
#
# After running this script, register the machine as a GitHub Actions
# self-hosted runner with the label: infra-tests-arm64
# https://github.com/e2b-dev/infra/settings/actions/runners/new

set -euo pipefail

PS4='[\D{%Y-%m-%d %H:%M:%S}] '
set -x

if [ "$(id -u)" -ne 0 ]; then
echo "ERROR: This script must be run as root" >&2
exit 1
fi

ARCH=$(dpkg --print-architecture)
if [ "$ARCH" != "arm64" ]; then
echo "ERROR: This script is for ARM64 hosts (detected: $ARCH)" >&2
exit 1
fi

echo "=== Setting up ARM64 GitHub Actions runner ==="

# KVM check
if [ ! -e /dev/kvm ]; then
echo "ERROR: /dev/kvm not found. KVM support is required." >&2
exit 1
fi

# Install base dependencies
apt-get update
apt-get install -y --no-install-recommends \
build-essential \
curl \
git \
jq \
nbd-client \
nbd-server

# Enable unprivileged userfaultfd
echo 1 > /proc/sys/vm/unprivileged_userfaultfd

# Hugepages
mkdir -p /mnt/hugepages
mount -t hugetlbfs none /mnt/hugepages 2>/dev/null || true
echo 2000 > /proc/sys/vm/nr_hugepages

grep -qF 'hugetlbfs /mnt/hugepages' /etc/fstab || \
echo "hugetlbfs /mnt/hugepages hugetlbfs defaults 0 0" >> /etc/fstab

# Sysctl — write once (idempotent)
cat <<'EOF' > /etc/sysctl.d/99-e2b.conf
vm.unprivileged_userfaultfd=1
vm.nr_hugepages=2000
net.core.somaxconn=65535
net.core.netdev_max_backlog=65535
net.ipv4.tcp_max_syn_backlog=65535
vm.max_map_count=1048576
EOF
sysctl --system

# NBD
modprobe nbd nbds_max=256
echo "nbd" > /etc/modules-load.d/e2b.conf
echo "options nbd nbds_max=256" > /etc/modprobe.d/e2b-nbd.conf

# Disable inotify for NBD devices
cat <<'EOF' > /etc/udev/rules.d/97-nbd-device.rules
ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"
EOF
udevadm control --reload-rules
udevadm trigger

# File descriptor limits
cat <<'EOF' > /etc/security/limits.d/99-e2b.conf
* soft nofile 1048576
* hard nofile 1048576
EOF

echo ""
echo "=== ARM64 runner setup complete ==="
echo ""
echo "Verify:"
echo " uname -m → aarch64"
echo " ls /dev/kvm → exists"
echo " cat /proc/meminfo | grep HugePages_Total"
echo " lsmod | grep nbd"
echo ""
echo "Next: register this machine as a GitHub Actions self-hosted runner"
echo " Label: infra-tests-arm64"
echo " https://github.com/e2b-dev/infra/settings/actions/runners/new"
115 changes: 115 additions & 0 deletions .github/workflows/pr-tests-arm64.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
name: ARM64 tests on PRs

on: [workflow_call]

permissions:
contents: read

jobs:
cross-compile:
name: Cross-compile all packages for ARM64
runs-on: ubuntu-24.04
steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Setup Go
uses: ./.github/actions/go-setup-cache

- name: Install ARM64 cross-compiler
run: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu

- name: Build and vet packages (pure Go)
run: |
for pkg in api client-proxy envd shared db docker-reverse-proxy; do
echo "::group::packages/$pkg"
pushd "packages/$pkg" > /dev/null
GOARCH=arm64 go build ./...
GOARCH=arm64 go vet ./...
popd > /dev/null
echo "::endgroup::"
done

- name: Build and vet orchestrator (CGO)
run: |
CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=arm64 go build ./...
CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=arm64 go vet ./...
working-directory: packages/orchestrator

arm64-unit-tests:
name: ARM64 tests for ${{ matrix.package }}
runs-on: ubuntu-24.04-arm
timeout-minutes: 30
strategy:
matrix:
include:
- package: packages/api
test_path: ./...
sudo: false
- package: packages/client-proxy
test_path: ./...
sudo: false
- package: packages/db
test_path: ./...
sudo: false
- package: packages/docker-reverse-proxy
test_path: ./...
sudo: false
- package: packages/envd
test_path: ./...
sudo: true
- package: packages/orchestrator
test_path: ./...
sudo: true
- package: packages/shared
test_path: ./pkg/...
sudo: false
fail-fast: false
steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Setup Go
uses: ./.github/actions/go-setup-cache
with:
cache-dependency-paths: |
go.work
${{ matrix.package }}/go.mod
${{ matrix.package }}/go.sum

- name: Setup envd tests
run: |
sudo apt-get update && sudo apt-get install -y bindfs
if: matrix.package == 'packages/envd'

- name: Setup orchestrator tests
run: |
# Enable unprivileged uffd (Ubuntu defaults to 0)
echo 1 | sudo tee /proc/sys/vm/unprivileged_userfaultfd

# Enable hugepages (256 × 2MB = 512MB).
# Tests that need more hugepages than available will skip gracefully.
sudo mkdir -p /mnt/hugepages
sudo mount -t hugetlbfs none /mnt/hugepages
echo 256 | sudo tee /proc/sys/vm/nr_hugepages

# Install extra kernel modules (nbd is not in base modules on GitHub-hosted runners)
sudo apt-get update
sudo apt-get install -y linux-modules-extra-$(uname -r)
sudo modprobe nbd nbds_max=256

# Disable inotify watching of change events for NBD devices
echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules
sudo udevadm control --reload-rules
sudo udevadm trigger
if: matrix.package == 'packages/orchestrator'

- name: Run tests that require sudo
working-directory: ${{ matrix.package }}
run: sudo -E `which go` test -race -v ${{ matrix.test_path }}
if: matrix.sudo == true

- name: Run tests
working-directory: ${{ matrix.package }}
run: go test -race -v ${{ matrix.test_path }}
if: matrix.sudo == false
2 changes: 2 additions & 0 deletions .github/workflows/pull-request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ jobs:
uses: ./.github/workflows/out-of-order-migrations.yml
unit-tests:
uses: ./.github/workflows/pr-tests.yml
arm64-tests:
uses: ./.github/workflows/pr-tests-arm64.yml
integration-tests:
needs: [out-of-order-migrations]
uses: ./.github/workflows/integration_tests.yml
Expand Down
7 changes: 7 additions & 0 deletions packages/api/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@ PREFIX := $(strip $(subst ",,$(PREFIX)))
HOSTNAME := $(shell hostname 2> /dev/null || hostnamectl hostname 2> /dev/null)
$(if $(HOSTNAME),,$(error Failed to determine hostname: both 'hostname' and 'hostnamectl' failed))

# Architecture for builds. Defaults to local arch; override for cross-compilation
# (e.g., BUILD_ARCH=amd64 make build-and-upload from an ARM64 host).
BUILD_ARCH ?= $(shell go env GOARCH)
# Docker platform string. Override for multi-arch builds:
# BUILD_PLATFORM=linux/amd64,linux/arm64 make build-and-upload
BUILD_PLATFORM ?= linux/$(BUILD_ARCH)

expectedMigration := $(shell ./../../scripts/get-latest-migration.sh)

ifeq ($(PROVIDER),aws)
Expand Down
11 changes: 9 additions & 2 deletions packages/client-proxy/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@ PREFIX := $(strip $(subst ",,$(PREFIX)))
HOSTNAME := $(shell hostname 2> /dev/null || hostnamectl hostname 2> /dev/null)
$(if $(HOSTNAME),,$(error Failed to determine hostname: both 'hostname' and 'hostnamectl' failed))

# Architecture for builds. Defaults to local arch; override for cross-compilation
# (e.g., BUILD_ARCH=amd64 make build-and-upload from an ARM64 host).
BUILD_ARCH ?= $(shell go env GOARCH)
# Docker platform string. Override for multi-arch builds:
# BUILD_PLATFORM=linux/amd64,linux/arm64 make build-and-upload
BUILD_PLATFORM ?= linux/$(BUILD_ARCH)

ifeq ($(PROVIDER),aws)
IMAGE_REGISTRY := $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(PREFIX)core/client-proxy
else
Expand All @@ -16,7 +23,7 @@ endif
build:
# Allow for passing commit sha directly for docker builds
$(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD))
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/client-proxy -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" .
CGO_ENABLED=0 GOOS=linux GOARCH=$(BUILD_ARCH) go build -o bin/client-proxy -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" .

.PHONY: build-debug
build-debug:
Expand All @@ -26,7 +33,7 @@ build-debug:
.PHONY: build-and-upload
build-and-upload:
$(eval COMMIT_SHA := $(shell git rev-parse --short HEAD))
@docker buildx build --platform linux/amd64 --tag $(IMAGE_REGISTRY) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" -f ./Dockerfile ..
@docker buildx build --platform $(BUILD_PLATFORM) --tag $(IMAGE_REGISTRY) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" -f ./Dockerfile ..

.PHONY: run
run:
Expand Down
10 changes: 10 additions & 0 deletions packages/db/pkg/testutils/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"

Expand Down Expand Up @@ -106,6 +107,12 @@ func SetupDatabase(t *testing.T) *Database {
}
}

// gooseMu serializes goose operations across parallel tests.
// goose.OpenDBWithDriver calls goose.SetDialect which writes to package-level
// globals (dialect, store) without synchronization. Concurrent test goroutines
// race on these globals, triggering the race detector on ARM64.
var gooseMu sync.Mutex

// runDatabaseMigrations executes all required database migrations
func runDatabaseMigrations(t *testing.T, connStr string) {
t.Helper()
Expand All @@ -115,6 +122,9 @@ func runDatabaseMigrations(t *testing.T, connStr string) {
require.NoError(t, err, "Failed to find git root")
repoRoot := strings.TrimSpace(string(output))

gooseMu.Lock()
defer gooseMu.Unlock()

db, err := goose.OpenDBWithDriver("pgx", connStr)
require.NoError(t, err)
t.Cleanup(func() {
Expand Down
13 changes: 10 additions & 3 deletions packages/envd/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,13 @@ LDFLAGS=-ldflags "-X=main.commitSHA=$(BUILD)"
AWS_BUCKET_PREFIX ?= $(PREFIX)$(AWS_ACCOUNT_ID)-
GCP_BUCKET_PREFIX ?= $(GCP_PROJECT_ID)-

# Architecture for builds. Defaults to local arch; override for cross-compilation
# (e.g., BUILD_ARCH=amd64 make build from an ARM64 host).
BUILD_ARCH ?= $(shell go env GOARCH)
# Docker platform string. Override for multi-arch builds:
# BUILD_PLATFORM=linux/amd64,linux/arm64 make start-docker
BUILD_PLATFORM ?= linux/$(BUILD_ARCH)

.PHONY: init
init:
brew install protobuf
Expand All @@ -20,17 +27,17 @@ else
endif

build:
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o bin/envd ${LDFLAGS}
CGO_ENABLED=0 GOOS=linux GOARCH=$(BUILD_ARCH) go build -a -o bin/envd ${LDFLAGS}

build-debug:
CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/debug/envd ${LDFLAGS}

start-docker:
make build
DOCKER_BUILDKIT=1 docker build --platform linux/amd64 -t envd-debug . -f debug.Dockerfile
DOCKER_BUILDKIT=1 docker build --platform $(BUILD_PLATFORM) -t envd-debug . -f debug.Dockerfile
docker run \
--name envd \
--platform linux/amd64 \
--platform $(BUILD_PLATFORM) \
-p 49983:49983 \
-p 2345:2345 \
-p 9999:9999 \
Expand Down
18 changes: 13 additions & 5 deletions packages/envd/internal/services/legacy/conversion_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package legacy

import (
"bytes"
"context"
"io"
"net/http"
"net/http/httptest"
Expand All @@ -22,12 +23,19 @@ import (
func TestFilesystemClient_FieldFormatter(t *testing.T) {
t.Parallel()
fsh := filesystemconnectmocks.NewMockFilesystemHandler(t)
fsh.EXPECT().Move(mock.Anything, mock.Anything).Return(connect.NewResponse(&filesystem.MoveResponse{
Entry: &filesystem.EntryInfo{
Name: "test-name",
Owner: "new-extra-field",
// Use RunAndReturn to create a fresh response per call. Using Return()
// shares one Response across parallel subtests, causing a data race on
// the lazily-initialized header/trailer maps inside connect.Response.
fsh.EXPECT().Move(mock.Anything, mock.Anything).RunAndReturn(
func(_ context.Context, _ *connect.Request[filesystem.MoveRequest]) (*connect.Response[filesystem.MoveResponse], error) {
return connect.NewResponse(&filesystem.MoveResponse{
Entry: &filesystem.EntryInfo{
Name: "test-name",
Owner: "new-extra-field",
},
}), nil
},
}), nil)
)

_, handler := filesystemconnect.NewFilesystemHandler(fsh,
connect.WithInterceptors(
Expand Down
Loading
Loading