integrate go-nvtrust to collect GPU attestation (#676)

Author: yawangwang

.github/workflows/ci.yml

@@ -92,15 +92,15 @@ jobs:
         run: go build -v ./keymanager/...
         if: runner.os == 'Linux' && matrix.architecture == 'x64'
       - name: Build launcher module
-        run: go build -v ./launcher/...
+        run: go build -v -ldflags="-extldflags=-Wl,-z,lazy" ./launcher/...
         if: runner.os == 'Linux'
       - name: Run specific tests under root permission
         run: |
           GO_EXECUTABLE_PATH=$(which go)
-          sudo $GO_EXECUTABLE_PATH test -v -run "TestFetchImageSignaturesDockerPublic" ./launcher
+          sudo $GO_EXECUTABLE_PATH test -v -ldflags="-extldflags=-Wl,-z,lazy" -run "TestFetchImageSignaturesDockerPublic" ./launcher
         if: runner.os == 'Linux'
       - name: Run all tests in launcher to capture potential data race
-        run: go test -v -race ./launcher/...
+        run: go test -v -ldflags="-extldflags=-Wl,-z,lazy" -race ./launcher/...
         if: (runner.os == 'Linux') && matrix.architecture == 'x64'
       - name: Test all modules except launcher and keymanager
         run: go test -v ./... ./cmd/... ./verifier/... -skip='TestCacheConcurrentSetGet|TestHwAttestationPass|TestHardwareAttestationPass'

go.work.sum

@@ -71,6 +71,14 @@ type attestRoot interface {
 	Attest(nonce []byte) (any, error)
 	// ComputeNonce hashes the challenge and extraData using the algorithm preferred by the attestation root.
 	ComputeNonce(challenge []byte, extraData []byte) []byte
+	// AddDeviceROTs adds detected device RoTs(root of trust).
+	AddDeviceROTs([]DeviceROT)
+}
+
+// DeviceROT defines an interface for all attached devices to collect attestation.
+type DeviceROT interface {
+	// Attest fetches an attestation from the attached device detected by launcher.
+	Attest(nonce []byte) (any, error)
 }
 
 // AttestAgentOpts contains user generated options when calling the
@@ -98,7 +106,7 @@ type agent struct {
 // - principalFetcher is a func to fetch GCE principal tokens for a given audience.
 // - signaturesFetcher is a func to fetch container image signatures associated with the running workload.
 // - logger will log any partial errors returned by VerifyAttestation.
-func CreateAttestationAgent(tpm io.ReadWriteCloser, akFetcher util.TpmKeyFetcher, verifierClient verifier.Client, principalFetcher principalIDTokenFetcher, sigsFetcher signaturediscovery.Fetcher, launchSpec spec.LaunchSpec, logger logging.Logger) (AttestationAgent, error) {
+func CreateAttestationAgent(tpm io.ReadWriteCloser, akFetcher util.TpmKeyFetcher, verifierClient verifier.Client, principalFetcher principalIDTokenFetcher, sigsFetcher signaturediscovery.Fetcher, launchSpec spec.LaunchSpec, logger logging.Logger, deviceROTs []DeviceROT) (AttestationAgent, error) {
 	// Fetched the AK and save it, so the agent doesn't need to create a new key everytime
 	ak, err := akFetcher(tpm)
 	if err != nil {
@@ -161,6 +169,8 @@ func CreateAttestationAgent(tpm io.ReadWriteCloser, akFetcher util.TpmKeyFetcher
 		attestAgent.avRot = tpmAR
 	}
 
+	// Add deviceRoTs to the CPU attestation root.
+	attestAgent.avRot.AddDeviceROTs(deviceROTs)
 	return attestAgent, nil
 }
 
@@ -359,11 +369,12 @@ func convertOCIToContainerSignature(ociSig oci.Signature) (*verifier.ContainerSi
 }
 
 type tpmAttestRoot struct {
-	tpmMu     sync.Mutex
-	fetchedAK *client.Key
-	tpm       io.ReadWriteCloser
-	cosCel    gecel.CEL
-	hashAlgos []crypto.Hash
+	tpmMu      sync.Mutex
+	fetchedAK  *client.Key
+	tpm        io.ReadWriteCloser
+	cosCel     gecel.CEL
+	hashAlgos  []crypto.Hash
+	deviceROTs []DeviceROT
 }
 
 func (t *tpmAttestRoot) GetCEL() gecel.CEL {
@@ -404,10 +415,15 @@ func (t *tpmAttestRoot) ComputeNonce(challenge []byte, extraData []byte) []byte
 	return finalNonce[:]
 }
 
+func (t *tpmAttestRoot) AddDeviceROTs(deviceROTs []DeviceROT) {
+	t.deviceROTs = append(t.deviceROTs, deviceROTs...)
+}
+
 type tdxAttestRoot struct {
-	tdxMu  sync.Mutex
-	qp     *tg.LinuxConfigFsQuoteProvider
-	cosCel gecel.CEL
+	tdxMu      sync.Mutex
+	qp         *tg.LinuxConfigFsQuoteProvider
+	cosCel     gecel.CEL
+	deviceROTs []DeviceROT
 }
 
 func (t *tdxAttestRoot) GetCEL() gecel.CEL {
@@ -441,10 +457,25 @@ func (t *tdxAttestRoot) Attest(nonce []byte) (any, error) {
 		return nil, err
 	}
 
+	var nvAtt *models.NvidiaAttestation
+	for _, deviceRoT := range t.deviceROTs {
+		att, err := deviceRoT.Attest(nonce)
+		if err != nil {
+			return nil, err
+		}
+		switch v := att.(type) {
+		case *models.NvidiaAttestation:
+			nvAtt = v
+		default:
+			return nil, fmt.Errorf("unknown device attestation type: %T", v)
+		}
+	}
+
 	return &verifier.TDCCELAttestation{
-		CcelAcpiTable: ccelTable,
-		CcelData:      ccelData,
-		TdQuote:       rawQuote,
+		CcelAcpiTable:     ccelTable,
+		CcelData:          ccelData,
+		TdQuote:           rawQuote,
+		NvidiaAttestation: nvAtt,
 	}, nil
 }
 
@@ -459,6 +490,10 @@ func (t *tdxAttestRoot) ComputeNonce(challenge []byte, extraData []byte) []byte
 	return finalNonce[:]
 }
 
+func (t *tdxAttestRoot) AddDeviceROTs(deviceROTs []DeviceROT) {
+	t.deviceROTs = append(t.deviceROTs, deviceROTs...)
+}
+
 // Refresh refreshes the internal state of the attestation agent.
 // It will reset the container image signatures for now.
 func (a *agent) Refresh(ctx context.Context) error {

launcher/agent/agent.go

@@ -32,6 +32,7 @@ import (
 	tpmpb "github.com/google/go-tpm-tools/proto/tpm"
 	"github.com/google/go-tpm-tools/verifier"
 	"github.com/google/go-tpm-tools/verifier/fake"
+	"github.com/google/go-tpm-tools/verifier/models"
 	"github.com/google/go-tpm-tools/verifier/oci"
 	"github.com/google/go-tpm-tools/verifier/oci/cosign"
 	"google.golang.org/protobuf/encoding/protojson"
@@ -60,7 +61,7 @@ func TestAttestRacing(t *testing.T) {
 	}
 
 	verifierClient := fake.NewClient(fakeSigner)
-	agent, err := CreateAttestationAgent(tpm, client.AttestationKeyECC, verifierClient, placeholderPrincipalFetcher, signaturediscovery.NewFakeClient(), spec.LaunchSpec{}, logging.SimpleLogger())
+	agent, err := CreateAttestationAgent(tpm, client.AttestationKeyECC, verifierClient, placeholderPrincipalFetcher, signaturediscovery.NewFakeClient(), spec.LaunchSpec{}, logging.SimpleLogger(), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -112,7 +113,7 @@ func TestAttest(t *testing.T) {
 
 			verifierClient := fake.NewClient(fakeSigner)
 
-			agent, err := CreateAttestationAgent(tpm, client.AttestationKeyECC, verifierClient, tc.principalIDTokenFetcher, tc.containerSignaturesFetcher, tc.launchSpec, logging.SimpleLogger())
+			agent, err := CreateAttestationAgent(tpm, client.AttestationKeyECC, verifierClient, tc.principalIDTokenFetcher, tc.containerSignaturesFetcher, tc.launchSpec, logging.SimpleLogger(), nil)
 			if err != nil {
 				t.Fatalf("failed to create an attestation agent %v", err)
 			}
@@ -651,6 +652,7 @@ func measureFakeEvents(attestAgent AttestationAgent) error {
 type fakeTdxAttestRoot struct {
 	cel           gecel.CEL
 	receivedNonce []byte
+	deviceRoTS    []DeviceROT
 }
 
 func (f *fakeTdxAttestRoot) Extend(c gecel.Content) error {
@@ -665,8 +667,23 @@ func (f *fakeTdxAttestRoot) GetCEL() gecel.CEL {
 
 func (f *fakeTdxAttestRoot) Attest(nonce []byte) (any, error) {
 	f.receivedNonce = nonce
+	var nvAtt *models.NvidiaAttestation
+	for _, deviceRoT := range f.deviceRoTS {
+		att, err := deviceRoT.Attest(nonce)
+		if err != nil {
+			return nil, err
+		}
+		switch v := att.(type) {
+		case *models.NvidiaAttestation:
+			nvAtt = v
+		default:
+			return nil, fmt.Errorf("unknown device attestation type: %T", v)
+		}
+	}
+
 	return &verifier.TDCCELAttestation{
-		TdQuote: []byte("fake-tdx-quote"),
+		TdQuote:           []byte("fake-tdx-quote"),
+		NvidiaAttestation: nvAtt,
 	}, nil
 }
 
@@ -681,6 +698,71 @@ func (f *fakeTdxAttestRoot) ComputeNonce(challenge []byte, extraData []byte) []b
 	return finalNonce[:]
 }
 
+func (f *fakeTdxAttestRoot) AddDeviceROTs(deviceRoTS []DeviceROT) {
+	f.deviceRoTS = append(f.deviceRoTS, deviceRoTS...)
+}
+
+type fakeGPURoT struct{}
+
+func (f *fakeGPURoT) Attest(nonce []byte) (any, error) {
+	if len(nonce) == 0 {
+		return nil, fmt.Errorf("fake GPU attestation failed")
+	}
+	return &models.NvidiaAttestation{
+		CCFeature: &models.NvidiaSinglePassthroughAttestation{
+			GPUInfo: models.GPUInfo{UUID: "fake-gpu-uuid"},
+		},
+	}, nil
+}
+func TestTdxAttestRoot(t *testing.T) {
+	testCases := []struct {
+		name          string
+		tdxAttestRoot *fakeTdxAttestRoot
+		nonce         []byte
+		wantGPU       bool
+		wantPass      bool
+	}{
+		{
+			name:          "success tdxAttestRoot w/o GPU device",
+			tdxAttestRoot: &fakeTdxAttestRoot{},
+			nonce:         []byte("test-nonce"),
+			wantPass:      true,
+		},
+		{
+			name: "success tdxAttestRoot w/ GPU device",
+			tdxAttestRoot: &fakeTdxAttestRoot{
+				deviceRoTS: []DeviceROT{&fakeGPURoT{}},
+			},
+			nonce:    []byte("test-nonce"),
+			wantGPU:  true,
+			wantPass: true,
+		},
+		{
+			name: "failed tdxAttestRoot w/ GPU device",
+			tdxAttestRoot: &fakeTdxAttestRoot{
+				deviceRoTS: []DeviceROT{&fakeGPURoT{}},
+			},
+			nonce:    []byte(""),
+			wantPass: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			attestation, err := tc.tdxAttestRoot.Attest(tc.nonce)
+			if gotPass := (err == nil); gotPass != tc.wantPass {
+				t.Errorf("tdxAttestRoot.Attest() did not return expected attestation result, got %v, want %v", gotPass, tc.wantPass)
+			}
+			if tc.wantPass && tc.wantGPU {
+				if att := attestation.(*verifier.TDCCELAttestation); att.NvidiaAttestation == nil {
+					t.Error("tdxAttestRoot.Attest() did not return expected GPU attestation, want GPU attestation, but got nil")
+				}
+			}
+		})
+	}
+
+}
+
 func TestAttestationEvidence_TDX_Success(t *testing.T) {
 	ctx := context.Background()
 	tpm := test.GetTPM(t)
@@ -749,7 +831,7 @@ func TestAttestationEvidence_TPM_Success(t *testing.T) {
 		Experiments: experiments.Experiments{
 			EnableAttestationEvidence: true,
 		},
-	}, logging.SimpleLogger())
+	}, logging.SimpleLogger(), nil)
 	if err != nil {
 		t.Fatalf("failed to create agent: %v", err)
 	}

launcher/agent/agent_test.go

@@ -70,7 +70,7 @@ steps:
       # Build the launcher with CGO enabled
       cd launcher/launcher
       export CGO_LDFLAGS="-L/workspace/keymanager/target/release"
-      CGO_ENABLED=1 go build -o ../image/launcher -ldflags="-X 'main.BuildCommit=${SHORT_SHA}'"
+      go build -o ../image/launcher -ldflags="-extldflags=-Wl,-z,lazy -X 'main.BuildCommit=${SHORT_SHA}'"
 
 - name: 'gcr.io/cloud-builders/gcloud'
   id: DownloadExpBinary

launcher/cloudbuild.yaml

@@ -194,6 +194,7 @@ func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.To
 	}
 	specOpts = append(specOpts, cgroupOpts...)
 
+	var deviceROTs []agent.DeviceROT
 	if launchSpec.InstallGpuDriver {
 		gpuMounts := []specs.Mount{
 			{
@@ -228,6 +229,7 @@ func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.To
 			logger.Info(fmt.Sprintf("Detected nvidia device : %s", deviceFile))
 			specOpts = append(specOpts, oci.WithDevices(deviceFile, deviceFile, "crw-rw-rw-"))
 		}
+		deviceROTs = append(deviceROTs, &gpu.NvidiaAttester{})
 	}
 
 	container, err = cdClient.NewContainer(
@@ -294,7 +296,7 @@ func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.To
 	// Create a new signaturediscovery client to fetch signatures.
 	sdClient := getSignatureDiscoveryClient(cdClient, mdsClient, image.Target())
 
-	attestAgent, err := agent.CreateAttestationAgent(tpm, client.GceAttestationKeyECC, verifierClient, principalFetcherWithImpersonate, sdClient, launchSpec, logger)
+	attestAgent, err := agent.CreateAttestationAgent(tpm, client.GceAttestationKeyECC, verifierClient, principalFetcherWithImpersonate, sdClient, launchSpec, logger, deviceROTs)
 	if err != nil {
 		return nil, err
 	}

launcher/container_runner.go

@@ -5,10 +5,12 @@ go 1.24.0
 toolchain go1.24.8
 
 require (
-	cloud.google.com/go/compute/metadata v0.8.0
-	cloud.google.com/go/logging v1.13.0
+	cloud.google.com/go/compute/metadata v0.9.0
+	cloud.google.com/go/logging v1.13.1
 	cos.googlesource.com/cos/tools.git v0.0.0-20250414225215-0cf736c0714c
+	github.com/NVIDIA/go-nvml v0.13.0-1
 	github.com/cenkalti/backoff/v4 v4.3.0
+	github.com/confidentsecurity/go-nvtrust v0.2.2
 	github.com/containerd/containerd v1.7.23
 	github.com/containerd/containerd/v2 v2.0.1
 	github.com/coreos/go-systemd/v22 v22.5.0
@@ -22,23 +24,24 @@ require (
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0
 	github.com/opencontainers/runtime-spec v1.2.0
-	golang.org/x/oauth2 v0.30.0
-	google.golang.org/api v0.247.0
-	google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c
-	google.golang.org/grpc v1.74.2
-	google.golang.org/protobuf v1.36.9
+	golang.org/x/oauth2 v0.34.0
+	google.golang.org/api v0.265.0
+	google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20
+	google.golang.org/grpc v1.78.0
+	google.golang.org/protobuf v1.36.11
 )
 
 require (
-	cloud.google.com/go v0.120.0 // indirect
-	cloud.google.com/go/auth v0.16.4 // indirect
+	cloud.google.com/go v0.123.0 // indirect
+	cloud.google.com/go/auth v0.18.1 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
-	cloud.google.com/go/confidentialcomputing v1.9.3-0.20250902151313-51583bd5c9b8 // indirect
-	cloud.google.com/go/longrunning v0.6.7 // indirect
+	cloud.google.com/go/confidentialcomputing v1.11.0 // indirect
+	cloud.google.com/go/longrunning v0.8.0 // indirect
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/Microsoft/hcsshim v0.12.9 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/cgroups/v3 v3.0.3 // indirect
 	github.com/containerd/containerd/api v1.8.0 // indirect
 	github.com/containerd/continuity v0.4.4 // indirect
@@ -65,8 +68,8 @@ require (
 	github.com/google/logger v1.1.1 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
-	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
+	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/sys/mountinfo v0.7.2 // indirect
@@ -78,21 +81,21 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
-	go.opentelemetry.io/otel v1.36.0 // indirect
-	go.opentelemetry.io/otel/metric v1.36.0 // indirect
-	go.opentelemetry.io/otel/trace v1.36.0 // indirect
+	go.opentelemetry.io/otel v1.39.0 // indirect
+	go.opentelemetry.io/otel/metric v1.39.0 // indirect
+	go.opentelemetry.io/otel/trace v1.39.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.45.0 // indirect
-	golang.org/x/net v0.47.0 // indirect
-	golang.org/x/sync v0.18.0 // indirect
-	golang.org/x/sys v0.38.0 // indirect
-	golang.org/x/text v0.31.0 // indirect
-	golang.org/x/time v0.12.0 // indirect
-	google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
+	golang.org/x/crypto v0.47.0 // indirect
+	golang.org/x/net v0.49.0 // indirect
+	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sys v0.40.0 // indirect
+	golang.org/x/text v0.33.0 // indirect
+	golang.org/x/time v0.14.0 // indirect
+	google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
 )
 
 replace (