Merge pull request #6005 from oasisprotocol/kostko/feature/tdx-persistent-image

go/runtime/host/tdx: Add support for persistent image overlay

Author: kostko

.changelog/6005.feature.md

@@ -0,0 +1 @@
+go/runtime/host/tdx: Add support for persistent image overlay

go/oasis-test-runner/oasis/runtime.go

@@ -98,6 +98,7 @@ type DeploymentCfg struct {
 // ComponentCfg is a runtime component configuration.
 type ComponentCfg struct {
 	Kind     component.Kind              `json:"kind"`
+	Name     string                      `json:"name,omitempty"`
 	Version  version.Version             `json:"version"`
 	Binaries map[node.TEEHardware]string `json:"binaries"`
 }
@@ -281,6 +282,7 @@ func (rt *Runtime) toRuntimeBundle(deploymentIndex int) (*bundle.Bundle, error)
 
 		comp := &bundle.Component{
 			Kind:    compCfg.Kind,
+			Name:    compCfg.Name,
 			Version: compCfg.Version,
 			ELF: &bundle.ELFMetadata{
 				Executable: elfBin,

go/oasis-test-runner/scenario/e2e/runtime/rofl.go

@@ -41,6 +41,7 @@ func (sc *roflImpl) Fixture() (*oasis.NetworkFixture, error) {
 	// Add ROFL component.
 	f.Runtimes[1].Deployments[0].Components = append(f.Runtimes[1].Deployments[0].Components, oasis.ComponentCfg{
 		Kind:     component.ROFL,
+		Name:     "test-rofl",
 		Binaries: sc.ResolveRuntimeBinaries(ROFLComponentBinary),
 	})
 

go/runtime/bundle/bundle_test.go

@@ -140,6 +140,7 @@ func TestDetachedBundle(t *testing.T) {
 			// No RONL component in the manifest.
 			{
 				Kind: component.ROFL,
+				Name: "my-rofl-comp",
 				ELF: &ELFMetadata{
 					Executable: "runtime.bin",
 				},

go/runtime/bundle/component.go

@@ -3,13 +3,24 @@ package bundle
 import (
 	"fmt"
 	"path/filepath"
+	"regexp"
 
 	"github.com/oasisprotocol/oasis-core/go/common"
 	"github.com/oasisprotocol/oasis-core/go/common/sgx"
 	"github.com/oasisprotocol/oasis-core/go/common/version"
 	"github.com/oasisprotocol/oasis-core/go/runtime/bundle/component"
 )
 
+// componentNameRegexp is the regular expression for valid component names.
+var componentNameRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`)
+
+const (
+	// minComponentNameLen is the minimum length of a valid component name.
+	minComponentNameLen = 3
+	// maxComponentNameLen is the maximum length of a valid component name.
+	maxComponentNameLen = 128
+)
+
 // ExplodedComponent is an exploded runtime component ready for execution.
 type ExplodedComponent struct {
 	*Component
@@ -118,6 +129,15 @@ func (c *Component) Validate() error {
 			return fmt.Errorf("RONL component cannot be disabled")
 		}
 	case component.ROFL:
+		if len(c.Name) < minComponentNameLen {
+			return fmt.Errorf("ROFL component name must be at least %d characters long", minComponentNameLen)
+		}
+		if len(c.Name) > maxComponentNameLen {
+			return fmt.Errorf("ROFL component name must be at most %d characters long", maxComponentNameLen)
+		}
+		if !componentNameRegexp.MatchString(c.Name) {
+			return fmt.Errorf("ROFL component name is invalid (must satisfy: %s)", componentNameRegexp)
+		}
 	default:
 		return fmt.Errorf("unknown component kind: '%s'", c.Kind)
 	}

go/runtime/bundle/component_test.go

@@ -0,0 +1,40 @@
+package bundle
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/oasisprotocol/oasis-core/go/runtime/bundle/component"
+)
+
+func TestComponentValidation(t *testing.T) {
+	require := require.New(t)
+
+	// Test component names.
+	var comp Component
+	err := comp.Validate()
+	require.ErrorContains(err, "unknown component kind")
+	comp.Kind = component.ROFL
+
+	for _, tc := range []struct {
+		name string
+		err  string
+	}{
+		{"", "ROFL component name must be at least 3 characters long"},
+		{strings.Repeat("a", 129), "ROFL component name must be at most 128 characters long"},
+		{"my invalid component name", "ROFL component name is invalid"},
+		{"my.invalid.component.name", "ROFL component name is invalid"},
+		{"my:invalid:component:name", "ROFL component name is invalid"},
+		{"my-valid-component-name", ""},
+	} {
+		comp.Name = tc.name
+		err = comp.Validate()
+		if tc.err == "" {
+			require.NoError(err)
+		} else {
+			require.ErrorContains(err, tc.err)
+		}
+	}
+}

go/runtime/bundle/registry_test.go

@@ -31,6 +31,7 @@ func createSyntheticBundle(runtimeID common.Namespace, version version.Version,
 		case component.ROFL:
 			manifest.Components = append(manifest.Components, &Component{
 				Kind:    component.ROFL,
+				Name:    "my-rofl-comp",
 				Version: version,
 			})
 		default:

go/runtime/host/tdx/qemu.go

@@ -2,21 +2,27 @@ package tdx
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/mdlayher/vsock"
 
+	"github.com/oasisprotocol/oasis-core/go/common"
 	"github.com/oasisprotocol/oasis-core/go/common/identity"
 	"github.com/oasisprotocol/oasis-core/go/common/logging"
 	"github.com/oasisprotocol/oasis-core/go/common/node"
 	"github.com/oasisprotocol/oasis-core/go/common/persistent"
 	"github.com/oasisprotocol/oasis-core/go/common/sgx/pcs"
 	sgxQuote "github.com/oasisprotocol/oasis-core/go/common/sgx/quote"
 	consensus "github.com/oasisprotocol/oasis-core/go/consensus/api"
+	"github.com/oasisprotocol/oasis-core/go/runtime/bundle"
 	"github.com/oasisprotocol/oasis-core/go/runtime/bundle/component"
 	"github.com/oasisprotocol/oasis-core/go/runtime/host"
 	"github.com/oasisprotocol/oasis-core/go/runtime/host/protocol"
@@ -28,10 +34,15 @@ import (
 const (
 	// defaultQemuSystemPath is the default QEMU system binary path.
 	defaultQemuSystemPath = "/usr/bin/qemu-system-x86_64"
+	// defaultQemuImgPath is the default qemu-bin binary path.
+	defaultQemuImgPath = "/usr/bin/qemu-img"
 	// defaultStartCid is the default start CID.
 	defaultStartCid = 0xA5150000
 	// defaultRuntimeAttestInterval is the default runtime (re-)attestation interval.
 	defaultRuntimeAttestInterval = 2 * time.Hour
+	// persistentImageDir is the name of the directory within the runtime data directory
+	// where persistent overlay images can be stored.
+	persistentImageDir = "images"
 
 	// vsockPortRHP is the VSOCK port used for the Runtime-Host Protocol.
 	vsockPortRHP = 1
@@ -41,6 +52,8 @@ const (
 
 // QemuConfig is the configuration of the QEMU-based TDX runtime provisioner.
 type QemuConfig struct {
+	// DataDir is the runtime data directory.
+	DataDir string
 	// HostInfo provides information about the host environment.
 	HostInfo *protocol.HostInfo
 
@@ -166,9 +179,20 @@ func (q *qemuProvisioner) getSandboxConfig(rtCfg host.Config, _ sandbox.Connecto
 				return process.Config{}, fmt.Errorf("format '%s' is not supported", stage2Format)
 			}
 
+			// Set up a persistent overlay image when configured to do so.
+			snapshotMode := "on" // Default to ephemeral images.
+			if tdxCfg.Stage2Persist {
+				stage2Image, err = q.createPersistentOverlayImage(rtCfg, comp, stage2Image, stage2Format)
+				if err != nil {
+					return process.Config{}, err
+				}
+				stage2Format = "qcow2"
+				snapshotMode = "off"
+			}
+
 			cfg.Args = append(cfg.Args,
 				// Stage 2 drive.
-				"-drive", fmt.Sprintf("format=%s,file=%s,if=none,id=drive0,snapshot=on", stage2Format, stage2Image),
+				"-drive", fmt.Sprintf("format=%s,file=%s,if=none,id=drive0,snapshot=%s", stage2Format, stage2Image, snapshotMode),
 				"-device", "virtio-blk-pci,drive=drive0",
 			)
 		}
@@ -211,6 +235,65 @@ func (q *qemuProvisioner) getSandboxConfig(rtCfg host.Config, _ sandbox.Connecto
 	return cfg, nil
 }
 
+// createPersistentOverlayImage creates a persistent overlay image for the given backing image and
+// returns the full path to the overlay image. In case the image already exists, it is reused.
+//
+// The format of the resulting image is always qcow2.
+func (q *qemuProvisioner) createPersistentOverlayImage(
+	rtCfg host.Config,
+	comp *bundle.ExplodedComponent,
+	image string,
+	format string,
+) (string, error) {
+	compID, _ := comp.ID().MarshalText()
+	imageDir := filepath.Join(q.cfg.DataDir, persistentImageDir, rtCfg.ID.String(), string(compID))
+	imageFn := filepath.Join(imageDir, fmt.Sprintf("%s.overlay", filepath.Base(image)))
+	switch _, err := os.Stat(imageFn); {
+	case err == nil:
+		// Image already exists, perform a rebase operation to account for the backing file location
+		// changing (e.g. due to an upgrade).
+		cmd := exec.Command(
+			defaultQemuImgPath,
+			"rebase",
+			"-u",
+			"-f", "qcow2",
+			"-b", image,
+			"-F", format,
+			imageFn,
+		)
+		var out strings.Builder
+		cmd.Stderr = &out
+		cmd.Stdout = &out
+		if err := cmd.Run(); err != nil {
+			return "", fmt.Errorf("failed to rebase persistent overlay image: %s\n%w", out.String(), err)
+		}
+	case errors.Is(err, os.ErrNotExist):
+		// Create image directory if it doesn't yet exist.
+		if err := common.Mkdir(imageDir); err != nil {
+			return "", fmt.Errorf("failed to create persistent overlay image directory: %w", err)
+		}
+
+		// Create the persistent overlay image.
+		cmd := exec.Command(
+			defaultQemuImgPath,
+			"create",
+			"-f", "qcow2",
+			"-b", image,
+			"-F", format,
+			imageFn,
+		)
+		var out strings.Builder
+		cmd.Stderr = &out
+		cmd.Stdout = &out
+		if err := cmd.Run(); err != nil {
+			return "", fmt.Errorf("failed to create persistent overlay image: %s\n%w", out.String(), err)
+		}
+	default:
+		return "", fmt.Errorf("failed to stat persistent overlay image: %w", err)
+	}
+	return imageFn, nil
+}
+
 func (q *qemuProvisioner) updateCapabilityTEE(ctx context.Context, hp *sandbox.HostInitializerParams) (cap *node.CapabilityTEE, aerr error) {
 	defer func() {
 		sgxCommon.UpdateAttestationMetrics(hp.Runtime.ID(), component.TEEKindTDX, aerr)

go/runtime/registry/config.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"maps"
 	"os"
+	"path/filepath"
 	"slices"
 	"strings"
 	"time"
@@ -114,6 +115,7 @@ func createHostInfo(consensus consensus.Backend) (*hostProtocol.HostInfo, error)
 }
 
 func createProvisioner(
+	dataDir string,
 	commonStore *persistent.CommonStore,
 	identity *identity.Identity,
 	consensus consensus.Backend,
@@ -205,6 +207,7 @@ func createProvisioner(
 	// Configure TDX provisioner.
 	// TODO: Allow provisioner selection in the future, currently we only have QEMU.
 	provisioners[component.TEEKindTDX], err = hostTdx.NewQemu(hostTdx.QemuConfig{
+		DataDir:               filepath.Join(dataDir, RuntimesDir),
 		HostInfo:              hostInfo,
 		CommonStore:           commonStore,
 		PCS:                   qs,

go/runtime/registry/registry.go

@@ -725,7 +725,7 @@ func New(
 	}
 
 	// Create runtime provisioner.
-	provisioner, err := createProvisioner(commonStore, identity, consensus, hostInfo, ias, qs)
+	provisioner, err := createProvisioner(dataDir, commonStore, identity, consensus, hostInfo, ias, qs)
 	if err != nil {
 		return nil, err
 	}