Otel more (#742)

instrument `otelslog` to ship logs as well, this rejiggers some env vars
around so that things use the more generic
`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`.

also adds a few more spans/attrs for the various `imagetest_test` phases

Author: joshrwolf

go.mod

@@ -34,9 +34,12 @@ require (
 	github.com/moby/docker-image-spec v1.3.1
 	github.com/samber/slog-multi v1.7.1
 	github.com/stretchr/testify v1.11.1
+	go.opentelemetry.io/contrib/bridges/otelslog v0.15.0
 	go.opentelemetry.io/otel v1.40.0
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0
 	go.opentelemetry.io/otel/sdk v1.40.0
+	go.opentelemetry.io/otel/sdk/log v0.16.0
 	go.opentelemetry.io/otel/trace v1.40.0
 	golang.org/x/sync v0.19.0
 	k8s.io/api v0.35.1
@@ -173,6 +176,7 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
+	go.opentelemetry.io/otel/log v0.16.0 // indirect
 	go.opentelemetry.io/otel/metric v1.40.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.step.sm/crypto v0.76.0 // indirect

go.sum

@@ -500,20 +500,30 @@ go.lsp.dev/uri v0.3.0 h1:KcZJmh6nFIBeJzTugn5JTU6OOyG0lDOo3R9KwTxTYbo=
 go.lsp.dev/uri v0.3.0/go.mod h1:P5sbO1IQR+qySTWOCnhnK7phBx+W3zbLqSMDJNTw88I=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/contrib/bridges/otelslog v0.15.0 h1:yOYhGNPZseueTTvWp5iBD3/CthrmvayUXYEX862dDi4=
+go.opentelemetry.io/contrib/bridges/otelslog v0.15.0/go.mod h1:CvaNVqIfcybc+7xqZNubbE+26K6P7AKZF/l0lE2kdCk=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
 go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
 go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0 h1:djrxvDxAe44mJUrKataUbOhCKhR3F8QCyWucO16hTQs=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0/go.mod h1:dt3nxpQEiSoKvfTVxp3TUg5fHPLhKtbcnN3Z1I1ePD0=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40=
+go.opentelemetry.io/otel/log v0.16.0 h1:DeuBPqCi6pQwtCK0pO4fvMB5eBq6sNxEnuTs88pjsN4=
+go.opentelemetry.io/otel/log v0.16.0/go.mod h1:rWsmqNVTLIA8UnwYVOItjyEZDbKIkMxdQunsIhpUMes=
 go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
 go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
 go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
 go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
+go.opentelemetry.io/otel/sdk/log v0.16.0 h1:e/b4bdlQwC5fnGtG3dlXUrNOnP7c8YLVSpSfEBIkTnI=
+go.opentelemetry.io/otel/sdk/log v0.16.0/go.mod h1:JKfP3T6ycy7QEuv3Hj8oKDy7KItrEkus8XJE6EoSzw4=
+go.opentelemetry.io/otel/sdk/log/logtest v0.16.0 h1:/XVkpZ41rVRTP4DfMgYv1nEtNmf65XPPyAdqV90TMy4=
+go.opentelemetry.io/otel/sdk/log/logtest v0.16.0/go.mod h1:iOOPgQr5MY9oac/F5W86mXdeyWZGleIx3uXO98X2R6Y=
 go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
 go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
 go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=

internal/drivers/aks/driver.go

@@ -27,6 +27,7 @@ import (
 	"github.com/charmbracelet/log"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/uuid"
+	"go.opentelemetry.io/otel/trace"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -328,6 +329,7 @@ func (k *driver) setupCommonClients() error {
 
 func (k *driver) Setup(ctx context.Context) error {
 	log := clog.FromContext(ctx)
+	span := trace.SpanFromContext(ctx)
 
 	if err := k.setupCommonClients(); err != nil {
 		return err
@@ -367,6 +369,7 @@ func (k *driver) Setup(ctx context.Context) error {
 		if err = k.createCluster(ctx); err != nil {
 			return err
 		}
+		span.AddEvent("aks.cluster.created")
 	}
 
 	if err = k.createPodIdentityAssociation(ctx); err != nil {
@@ -376,10 +379,12 @@ func (k *driver) Setup(ctx context.Context) error {
 	if err = k.createClusterIdentityAssociation(ctx); err != nil {
 		return err
 	}
+	span.AddEvent("aks.identity.configured")
 
 	if err = k.attachACRs(ctx); err != nil {
 		return err
 	}
+	span.AddEvent("aks.acr.attached")
 
 	if err = k.writeKubeConfig(ctx); err != nil {
 		return err

internal/drivers/docker_in_docker/driver.go

@@ -25,6 +25,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/uuid"
 	v1 "github.com/moby/docker-image-spec/specs-go/v1"
+	"go.opentelemetry.io/otel/trace"
 )
 
 type driver struct {
@@ -106,6 +107,8 @@ func (d *driver) Teardown(ctx context.Context) error {
 // Run implements drivers.TestDriver.
 func (d *driver) Run(ctx context.Context, ref name.Reference) (*drivers.RunResult, error) {
 	// Build the driver image, uses the provided dind image appended with the ref
+	span := trace.SpanFromContext(ctx)
+
 	tref, err := bundler.Mutate(ctx, d.ImageRef, ref.Context(), bundler.MutateOpts{
 		RemoteOptions: d.ropts,
 		ImageMutators: []func(ggcrv1.Image) (ggcrv1.Image, error){
@@ -150,6 +153,7 @@ func (d *driver) Run(ctx context.Context, ref name.Reference) (*drivers.RunResul
 	if err != nil {
 		return nil, fmt.Errorf("failed to build driver image: %w", err)
 	}
+	span.AddEvent("dind.image.built")
 
 	nw, err := d.cli.CreateNetwork(ctx, &docker.NetworkRequest{})
 	if err != nil {
@@ -161,6 +165,7 @@ func (d *driver) Run(ctx context.Context, ref name.Reference) (*drivers.RunResul
 	}); err != nil {
 		return nil, err
 	}
+	span.AddEvent("dind.network.created")
 
 	cliCfg, err := d.cliCfg.Content()
 	if err != nil {
@@ -199,6 +204,7 @@ func (d *driver) Run(ctx context.Context, ref name.Reference) (*drivers.RunResul
 
 	cname := fmt.Sprintf("%s-%s", d.name, uuid.New().String()[:8])
 	clog.InfoContext(ctx, "running docker-in-docker test", "image_ref", tref.String(), "container_name", cname)
+	span.AddEvent("dind.container.started")
 	cid, err := d.cli.Run(ctx, &docker.Request{
 		Name:       cname,
 		Ref:        tref,
@@ -223,6 +229,7 @@ func (d *driver) Run(ctx context.Context, ref name.Reference) (*drivers.RunResul
 	})
 
 	result := &drivers.RunResult{}
+	span.AddEvent("dind.container.completed")
 
 	arc, aerr := docker.GetFile(ctx, d.cli, cid, entrypoint.ArtifactsPath)
 	if aerr != nil {

internal/drivers/ec2/driver.go

@@ -22,6 +22,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/uuid"
 	"github.com/kballard/go-shellquote"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/crypto/ssh"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
@@ -103,6 +104,7 @@ func (d *driver) setupExistingInstance(ctx context.Context) error {
 
 func (d *driver) setupNewInstance(ctx context.Context) error {
 	log := clog.FromContext(ctx)
+	span := trace.SpanFromContext(ctx)
 
 	d.name = "imagetest-ec2-" + uuid.New().String()[:8]
 	log.Info("setting up EC2 driver", "name", d.name, "vpc_id", d.cfg.VPCID)
@@ -167,18 +169,22 @@ func (d *driver) setupNewInstance(ctx context.Context) error {
 	if err := d.create(ctx, d.instance); err != nil {
 		return fmt.Errorf("creating instance: %w", err)
 	}
+	span.AddEvent("ec2.instance.created")
 
 	if err := d.instance.wait(ctx); err != nil {
 		return fmt.Errorf("waiting for instance: %w", err)
 	}
+	span.AddEvent("ec2.instance.running")
 
 	if err := d.prepareInstance(ctx); err != nil {
 		return fmt.Errorf("preparing instance: %w", err)
 	}
+	span.AddEvent("ec2.cloudinit.complete")
 
 	if err := d.runSetupCommands(ctx); err != nil {
 		return fmt.Errorf("running setup commands: %w", err)
 	}
+	span.AddEvent("ec2.setup.complete")
 
 	log.Info("EC2 driver setup complete", "instance_id", d.instance.id, "public_ip", d.instance.publicIP)
 

internal/drivers/eks_with_eksctl/driver.go

@@ -22,6 +22,7 @@ import (
 	"github.com/charmbracelet/log"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/uuid"
+	"go.opentelemetry.io/otel/trace"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -428,6 +429,7 @@ func (k *driver) deletePodIdentityAssociation(ctx context.Context) error {
 
 func (k *driver) Setup(ctx context.Context) error {
 	log := clog.FromContext(ctx)
+	span := trace.SpanFromContext(ctx)
 
 	if n, ok := os.LookupEnv("IMAGETEST_EKS_CLUSTER"); ok {
 		log.Infof("Using cluster name from IMAGETEST_EKS_CLUSTER: %s", n)
@@ -489,16 +491,19 @@ func (k *driver) Setup(ctx context.Context) error {
 			return fmt.Errorf("eksctl create cluster: %w", err)
 		}
 		log.Infof("Created cluster %s without nodegroups", k.clusterName)
+		span.AddEvent("eks.cluster.created")
 	}
 
 	if err := k.createNodeGroup(ctx); err != nil {
 		return err
 	}
+	span.AddEvent("eks.nodegroup.created")
 
 	if k.podIdentityAssociations != nil {
 		if err = k.createPodIdentityAssociation(ctx); err != nil {
 			return fmt.Errorf("creating pod identity association: %w", err)
 		}
+		span.AddEvent("eks.identity.configured")
 	}
 
 	config, err := clientcmd.BuildConfigFromFlags("", k.kubeconfig)

internal/drivers/k3s_in_docker/driver.go

@@ -19,6 +19,7 @@ import (
 	"github.com/docker/go-connections/nat"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/moby/docker-image-spec/specs-go/v1"
+	"go.opentelemetry.io/otel/trace"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -162,6 +163,7 @@ configs:
 	if err != nil {
 		return fmt.Errorf("creating docker network: %w", err)
 	}
+	trace.SpanFromContext(ctx).AddEvent("k3s.network.created")
 
 	if err := k.stack.Add(func(ctx context.Context) error {
 		return cli.RemoveNetwork(ctx, nw)
@@ -224,6 +226,7 @@ configs:
 	}); err != nil {
 		return err
 	}
+	trace.SpanFromContext(ctx).AddEvent("k3s.container.started")
 
 	kcfgraw, err := resp.ReadFile(ctx, "/etc/rancher/k3s/k3s.yaml")
 	if err != nil {
@@ -251,6 +254,7 @@ configs:
 	}
 
 	config.Host = fmt.Sprintf("https://%s:%s", binding.HostIP, binding.HostPort)
+	trace.SpanFromContext(ctx).AddEvent("k3s.kubeconfig.ready")
 
 	kcli, err := kubernetes.NewForConfig(config)
 	if err != nil {
@@ -288,6 +292,7 @@ configs:
 	if err := k.waitReady(ctx); err != nil {
 		return fmt.Errorf("waiting for k3s to be ready: %w", err)
 	}
+	trace.SpanFromContext(ctx).AddEvent("k3s.cluster.ready")
 
 	// Ensure some common mount propagation fixes are applied to make this feel
 	// more like a "real" cluster
@@ -315,6 +320,7 @@ configs:
 				return fmt.Errorf("running post start hook: %w", err)
 			}
 		}
+		trace.SpanFromContext(ctx).AddEvent("k3s.hooks.complete")
 	}
 
 	return nil

internal/drivers/lambda/driver.go

@@ -12,6 +12,7 @@ import (
 	"github.com/chainguard-dev/clog"
 	"github.com/chainguard-dev/terraform-provider-imagetest/internal/drivers"
 	"github.com/google/go-containerregistry/pkg/name"
+	"go.opentelemetry.io/otel/trace"
 )
 
 type driver struct {
@@ -74,6 +75,8 @@ func (k *driver) Run(ctx context.Context, ref name.Reference) (*drivers.RunResul
 		return nil, fmt.Errorf("creating Lambda function: %w", err)
 	}
 	clog.FromContext(ctx).Info("Created Lambda function", "name", k.functionName)
+	span := trace.SpanFromContext(ctx)
+	span.AddEvent("lambda.function.created")
 
 	var out *lambda.GetFunctionOutput
 L:
@@ -99,6 +102,7 @@ L:
 		return nil, fmt.Errorf("function state is %s: %s", out.Configuration.State, *out.Configuration.StateReason)
 	}
 	clog.FromContext(ctx).Info("Lambda function is active", "name", k.functionName)
+	span.AddEvent("lambda.function.active")
 
 	// Invoke the function to ensure it is ready.
 	if out, err := k.client.Invoke(ctx, &lambda.InvokeInput{FunctionName: &k.functionName}); err != nil {

internal/drivers/pod/pod.go

@@ -17,6 +17,8 @@ import (
 	"github.com/chainguard-dev/terraform-provider-imagetest/internal/drivers"
 	"github.com/chainguard-dev/terraform-provider-imagetest/internal/entrypoint"
 	"github.com/google/go-containerregistry/pkg/name"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	authv1 "k8s.io/api/authorization/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -85,15 +87,19 @@ func Run(ctx context.Context, kcfg *rest.Config, options ...RunOpts) (*drivers.R
 	}
 	o.client = kcli
 
+	span := trace.SpanFromContext(ctx)
+
 	if err := o.preflight(ctx); err != nil {
 		return nil, err
 	}
+	span.AddEvent("pod.preflight.complete")
 
 	pobj, err := o.client.CoreV1().Pods(o.Namespace).
 		Create(ctx, o.pod(), metav1.CreateOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("failed to create pod: %w", err)
 	}
+	span.AddEvent("pod.created")
 
 	pw, err := o.client.CoreV1().Pods(pobj.Namespace).Watch(ctx, metav1.ListOptions{
 		FieldSelector: fmt.Sprintf("metadata.name=%s", pobj.Name),
@@ -121,6 +127,8 @@ func Run(ctx context.Context, kcfg *rest.Config, options ...RunOpts) (*drivers.R
 	result := &drivers.RunResult{Artifact: &drivers.RunArtifactResult{}}
 	if err := o.getArtifact(ctx, pobj, result); err != nil {
 		clog.ErrorContext(ctx, "failed to get artifact", "error", err)
+	} else {
+		span.AddEvent("pod.artifact.retrieved")
 	}
 
 	return result, monitorErr
@@ -253,6 +261,7 @@ func monitor(ctx context.Context, cli kubernetes.Interface, pod *corev1.Pod) err
 
 			if !logStarted && p.Status.Phase == corev1.PodRunning {
 				logStarted = true
+				trace.SpanFromContext(ctx).AddEvent("pod.running")
 				clog.InfoContext(ctx, "starting log stream")
 				logch = startLogStream(ctx, cli, pod)
 			}
@@ -263,6 +272,9 @@ func monitor(ctx context.Context, cli kubernetes.Interface, pod *corev1.Pod) err
 
 			for _, cs := range p.Status.ContainerStatuses {
 				if cs.Name == SandboxContainerName && cs.State.Terminated != nil {
+					trace.SpanFromContext(ctx).AddEvent("pod.completed",
+						trace.WithAttributes(attribute.Int64("exit_code", int64(cs.State.Terminated.ExitCode))),
+					)
 					clog.InfoContext(ctx, "sandbox container terminated",
 						"exit_code", cs.State.Terminated.ExitCode,
 						"reason", cs.State.Terminated.Reason,