Skip to content

Commit 48ca3ad

Browse files
loxlox
committed
feat(firecracker): prepare runtime rootfs with injected guest agent
1 parent b959de1 commit 48ca3ad

File tree

2 files changed

+254
-7
lines changed

2 files changed

+254
-7
lines changed

‎internal/backend/firecracker/backend.go‎

Lines changed: 254 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"context"
55
cryptorand "crypto/rand"
66
"crypto/sha1"
7+
"crypto/sha256"
8+
"encoding/hex"
79
"encoding/json"
810
"errors"
911
"fmt"
@@ -37,10 +39,73 @@ type Adapter struct {
3739
imageManager imageEnsurer
3840
imageManagerErr error
3941
newImageManager imageManagerFactory
42+
43+
guestAgentOnce sync.Once
44+
guestAgentPath string
45+
guestAgentHash string
46+
guestAgentErr error
47+
48+
runtimeImageMu sync.Mutex
4049
}
4150

4251
const runObservabilityFile = "run-observability.json"
4352
const vsockDialRetryInterval = 50 * time.Millisecond
53+
const preparedRuntimeRootFSVersion = "v1"
54+
55+
const guestInitScriptTemplate = `#!/bin/sh
56+
set -eu
57+
58+
mount -t proc proc /proc 2>/dev/null || true
59+
mount -t sysfs sysfs /sys 2>/dev/null || true
60+
mount -t devtmpfs devtmpfs /dev 2>/dev/null || true
61+
mkdir -p /dev/pts /run /tmp
62+
mount -t devpts devpts /dev/pts 2>/dev/null || true
63+
mount -t tmpfs tmpfs /run 2>/dev/null || true
64+
mount -t tmpfs tmpfs /tmp 2>/dev/null || true
65+
66+
export HOME=/root
67+
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.local/bin
68+
69+
cmdline="$(cat /proc/cmdline 2>/dev/null || true)"
70+
arg_value() {
71+
key="$1"
72+
for token in $cmdline; do
73+
case "$token" in
74+
"$key"=*) echo "${token#*=}"; return 0 ;;
75+
esac
76+
done
77+
return 1
78+
}
79+
80+
GUEST_IP="$(arg_value cleanroom_guest_ip || true)"
81+
GUEST_GW="$(arg_value cleanroom_guest_gw || true)"
82+
GUEST_MASK="$(arg_value cleanroom_guest_mask || true)"
83+
GUEST_DNS="$(arg_value cleanroom_guest_dns || true)"
84+
GUEST_PORT="$(arg_value cleanroom_guest_port || true)"
85+
86+
if command -v ip >/dev/null 2>&1 && [ -n "$GUEST_IP" ]; then
87+
[ -n "$GUEST_MASK" ] || GUEST_MASK="24"
88+
ip link set dev eth0 up 2>/dev/null || true
89+
ip addr flush dev eth0 2>/dev/null || true
90+
ip addr add "$GUEST_IP/$GUEST_MASK" dev eth0 2>/dev/null || true
91+
if [ -n "$GUEST_GW" ]; then
92+
ip route add default via "$GUEST_GW" dev eth0 2>/dev/null || true
93+
fi
94+
if [ -n "$GUEST_DNS" ]; then
95+
printf 'nameserver %s\n' "$GUEST_DNS" > /etc/resolv.conf 2>/dev/null || true
96+
fi
97+
fi
98+
99+
if [ -z "$GUEST_PORT" ]; then
100+
GUEST_PORT="10700"
101+
fi
102+
export CLEANROOM_VSOCK_PORT="$GUEST_PORT"
103+
104+
while true; do
105+
/usr/local/bin/cleanroom-guest-agent || true
106+
sleep 1
107+
done
108+
`
44109

45110
func New() *Adapter {
46111
return &Adapter{newImageManager: defaultImageManagerFactory}
@@ -109,6 +174,11 @@ func (a *Adapter) Doctor(_ context.Context, req backend.DoctorRequest) (*backend
109174
} else {
110175
appendCheck("kernel_image", "pass", fmt.Sprintf("kernel image configured: %s", req.KernelImagePath))
111176
}
177+
if guestAgentPath, _, err := a.getGuestAgentBinary(); err != nil {
178+
appendCheck("guest_agent_binary", "fail", err.Error())
179+
} else {
180+
appendCheck("guest_agent_binary", "pass", fmt.Sprintf("found cleanroom guest agent %q", guestAgentPath))
181+
}
112182

113183
imageRefStatus := "warn"
114184
imageRefMessage := "policy not loaded; cannot validate sandbox.image.ref"
@@ -283,7 +353,12 @@ func (a *Adapter) run(ctx context.Context, req backend.RunRequest, stream backen
283353
observation.ImageDigest = imageArtifact.Digest
284354
observation.ImageCacheHit = imageArtifact.CacheHit
285355

286-
rootfsPath, err := filepath.Abs(imageArtifact.RootFSPath)
356+
preparedRootFSPath, err := a.ensurePreparedRuntimeRootFS(ctx, imageArtifact)
357+
if err != nil {
358+
return nil, err
359+
}
360+
361+
rootfsPath, err := filepath.Abs(preparedRootFSPath)
287362
if err != nil {
288363
return nil, err
289364
}
@@ -322,9 +397,10 @@ func (a *Adapter) run(ctx context.Context, req backend.RunRequest, stream backen
322397
BootSource: bootSource{
323398
KernelImagePath: kernelPath,
324399
BootArgs: fmt.Sprintf(
325-
"console=ttyS0 reboot=k panic=1 pci=off init=/sbin/cleanroom-init random.trust_cpu=on cleanroom_guest_ip=%s cleanroom_guest_gw=%s cleanroom_guest_mask=24 cleanroom_guest_dns=1.1.1.1",
400+
"console=ttyS0 reboot=k panic=1 pci=off init=/sbin/cleanroom-init random.trust_cpu=on cleanroom_guest_ip=%s cleanroom_guest_gw=%s cleanroom_guest_mask=24 cleanroom_guest_dns=1.1.1.1 cleanroom_guest_port=%d",
326401
networkCfg.GuestIP,
327402
networkCfg.HostIP,
403+
req.GuestPort,
328404
),
329405
},
330406
Drives: []drive{
@@ -589,6 +665,182 @@ func (a *Adapter) ensureImageArtifact(ctx context.Context, imageRef string) (ima
589665
}, nil
590666
}
591667

668+
func (a *Adapter) ensurePreparedRuntimeRootFS(ctx context.Context, image imageArtifact) (string, error) {
669+
sourcePath := strings.TrimSpace(image.RootFSPath)
670+
if sourcePath == "" {
671+
return "", errors.New("resolved image rootfs path is empty")
672+
}
673+
if _, err := os.Stat(sourcePath); err != nil {
674+
return "", fmt.Errorf("resolved image rootfs %q: %w", sourcePath, err)
675+
}
676+
677+
guestAgentPath, guestAgentHash, err := a.getGuestAgentBinary()
678+
if err != nil {
679+
return "", err
680+
}
681+
682+
preparedPath, err := preparedRuntimeRootFSPath(image.Digest, guestAgentHash)
683+
if err != nil {
684+
return "", err
685+
}
686+
if _, err := os.Stat(preparedPath); err == nil {
687+
return preparedPath, nil
688+
} else if !errors.Is(err, os.ErrNotExist) {
689+
return "", fmt.Errorf("inspect prepared runtime rootfs %q: %w", preparedPath, err)
690+
}
691+
692+
a.runtimeImageMu.Lock()
693+
defer a.runtimeImageMu.Unlock()
694+
695+
if _, err := os.Stat(preparedPath); err == nil {
696+
return preparedPath, nil
697+
} else if !errors.Is(err, os.ErrNotExist) {
698+
return "", fmt.Errorf("inspect prepared runtime rootfs %q: %w", preparedPath, err)
699+
}
700+
701+
preparedDir := filepath.Dir(preparedPath)
702+
if err := os.MkdirAll(preparedDir, 0o755); err != nil {
703+
return "", fmt.Errorf("create prepared rootfs cache directory %q: %w", preparedDir, err)
704+
}
705+
706+
tmpPath := preparedPath + fmt.Sprintf(".tmp-%d", time.Now().UnixNano())
707+
if err := copyFile(sourcePath, tmpPath); err != nil {
708+
return "", fmt.Errorf("copy rootfs image for runtime preparation: %w", err)
709+
}
710+
if err := a.installGuestRuntimeIntoRootFS(ctx, tmpPath, guestAgentPath); err != nil {
711+
_ = os.Remove(tmpPath)
712+
return "", err
713+
}
714+
if err := os.Rename(tmpPath, preparedPath); err != nil {
715+
_ = os.Remove(tmpPath)
716+
if _, statErr := os.Stat(preparedPath); statErr == nil {
717+
return preparedPath, nil
718+
}
719+
return "", fmt.Errorf("store prepared runtime rootfs %q: %w", preparedPath, err)
720+
}
721+
return preparedPath, nil
722+
}
723+
724+
func preparedRuntimeRootFSPath(imageDigest, guestAgentHash string) (string, error) {
725+
cacheBase, err := paths.CacheBaseDir()
726+
if err != nil {
727+
return "", fmt.Errorf("resolve cache base directory: %w", err)
728+
}
729+
key := runtimeRootFSCacheKey(imageDigest, guestAgentHash)
730+
return filepath.Join(cacheBase, "firecracker", "runtime-rootfs", key+".ext4"), nil
731+
}
732+
733+
func runtimeRootFSCacheKey(imageDigest, guestAgentHash string) string {
734+
sum := sha256.Sum256([]byte(strings.TrimSpace(imageDigest) + "|" + guestAgentHash + "|" + preparedRuntimeRootFSVersion + "|" + guestInitScriptTemplate))
735+
return hex.EncodeToString(sum[:])
736+
}
737+
738+
func (a *Adapter) installGuestRuntimeIntoRootFS(ctx context.Context, rootFSPath, guestAgentPath string) error {
739+
mountDir, err := os.MkdirTemp("", "cleanroom-firecracker-rootfs-*")
740+
if err != nil {
741+
return fmt.Errorf("create temporary rootfs mount directory: %w", err)
742+
}
743+
defer os.RemoveAll(mountDir)
744+
745+
if err := runRootCommand(ctx, "mount", "-o", "loop", rootFSPath, mountDir); err != nil {
746+
return fmt.Errorf("mount rootfs image for runtime preparation: %w", err)
747+
}
748+
mounted := true
749+
defer func() {
750+
if mounted {
751+
_ = runRootCommand(context.Background(), "umount", mountDir)
752+
}
753+
}()
754+
755+
initScriptPath, err := createGuestInitScript()
756+
if err != nil {
757+
return err
758+
}
759+
defer os.Remove(initScriptPath)
760+
761+
if err := runRootCommand(ctx, "mkdir", "-p", filepath.Join(mountDir, "usr/local/bin"), filepath.Join(mountDir, "sbin")); err != nil {
762+
return fmt.Errorf("prepare runtime directories in mounted rootfs: %w", err)
763+
}
764+
if err := runRootCommand(ctx, "install", "-m", "0755", guestAgentPath, filepath.Join(mountDir, "usr/local/bin/cleanroom-guest-agent")); err != nil {
765+
return fmt.Errorf("install guest agent into mounted rootfs: %w", err)
766+
}
767+
if err := runRootCommand(ctx, "install", "-m", "0755", initScriptPath, filepath.Join(mountDir, "sbin/cleanroom-init")); err != nil {
768+
return fmt.Errorf("install cleanroom init into mounted rootfs: %w", err)
769+
}
770+
if err := runRootCommand(ctx, "umount", mountDir); err != nil {
771+
return fmt.Errorf("unmount prepared rootfs image: %w", err)
772+
}
773+
mounted = false
774+
return nil
775+
}
776+
777+
func createGuestInitScript() (string, error) {
778+
f, err := os.CreateTemp("", "cleanroom-init-*.sh")
779+
if err != nil {
780+
return "", fmt.Errorf("create guest init script: %w", err)
781+
}
782+
if _, err := f.WriteString(guestInitScriptTemplate); err != nil {
783+
_ = f.Close()
784+
_ = os.Remove(f.Name())
785+
return "", fmt.Errorf("write guest init script: %w", err)
786+
}
787+
if err := f.Chmod(0o755); err != nil {
788+
_ = f.Close()
789+
_ = os.Remove(f.Name())
790+
return "", fmt.Errorf("chmod guest init script: %w", err)
791+
}
792+
if err := f.Close(); err != nil {
793+
_ = os.Remove(f.Name())
794+
return "", fmt.Errorf("close guest init script: %w", err)
795+
}
796+
return f.Name(), nil
797+
}
798+
799+
func (a *Adapter) getGuestAgentBinary() (string, string, error) {
800+
a.guestAgentOnce.Do(func() {
801+
a.guestAgentPath, a.guestAgentErr = discoverGuestAgentBinary()
802+
if a.guestAgentErr != nil {
803+
return
804+
}
805+
a.guestAgentHash, a.guestAgentErr = hashFileSHA256(a.guestAgentPath)
806+
})
807+
if a.guestAgentErr != nil {
808+
return "", "", a.guestAgentErr
809+
}
810+
if strings.TrimSpace(a.guestAgentPath) == "" || strings.TrimSpace(a.guestAgentHash) == "" {
811+
return "", "", errors.New("failed to resolve cleanroom guest agent binary")
812+
}
813+
return a.guestAgentPath, a.guestAgentHash, nil
814+
}
815+
816+
func discoverGuestAgentBinary() (string, error) {
817+
if p, err := exec.LookPath("cleanroom-guest-agent"); err == nil {
818+
return p, nil
819+
}
820+
self, err := os.Executable()
821+
if err == nil {
822+
candidate := filepath.Join(filepath.Dir(self), "cleanroom-guest-agent")
823+
if info, statErr := os.Stat(candidate); statErr == nil && !info.IsDir() {
824+
return candidate, nil
825+
}
826+
}
827+
return "", errors.New("cleanroom-guest-agent binary not found in PATH; run `mise run install` first")
828+
}
829+
830+
func hashFileSHA256(path string) (string, error) {
831+
f, err := os.Open(path)
832+
if err != nil {
833+
return "", fmt.Errorf("open %q for hashing: %w", path, err)
834+
}
835+
defer f.Close()
836+
837+
hash := sha256.New()
838+
if _, err := io.Copy(hash, f); err != nil {
839+
return "", fmt.Errorf("hash %q: %w", path, err)
840+
}
841+
return hex.EncodeToString(hash.Sum(nil)), nil
842+
}
843+
592844
func (a *Adapter) getImageManager() (imageEnsurer, error) {
593845
if a.newImageManager == nil {
594846
a.newImageManager = defaultImageManagerFactory
@@ -990,5 +1242,3 @@ func guestMACFromRunID(runID string) string {
9901242
sum := sha1.Sum([]byte(runID))
9911243
return fmt.Sprintf("02:fc:%02x:%02x:%02x:%02x", sum[0], sum[1], sum[2], sum[3])
9921244
}
993-
994-

‎internal/controlservice/service.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -860,9 +860,6 @@ func (s *Service) runExecution(sandboxID, executionID string) {
860860
firecrackerCfg.RunDir = filepath.Join(runBaseDir, ex.RunID)
861861
}
862862
}
863-
if ex.Options.ReadOnlyWorkspace {
864-
firecrackerCfg.WorkspaceAccess = "ro"
865-
}
866863
if ex.Options.LaunchSeconds != 0 {
867864
firecrackerCfg.LaunchSeconds = ex.Options.LaunchSeconds
868865
}

0 commit comments

Comments
 (0)