Merge pull request moby#3544 from gabriel-samfira/fix-tracing-listener

[Windows] Fix tracing listener

Author: tonistiigi

cmd/buildkitd/main.go

@@ -631,7 +631,7 @@ func newController(c *cli.Context, cfg *config.Config) (*control.Controller, err
 
 	var traceSocket string
 	if tc != nil {
-		traceSocket = filepath.Join(cfg.Root, "otel-grpc.sock")
+		traceSocket = traceSocketPath(cfg.Root)
 		if err := runTraceController(traceSocket, tc); err != nil {
 			return nil, err
 		}
@@ -814,14 +814,9 @@ func parseBoolOrAuto(s string) (*bool, error) {
 func runTraceController(p string, exp sdktrace.SpanExporter) error {
 	server := grpc.NewServer()
 	tracev1.RegisterTraceServiceServer(server, &traceCollector{exporter: exp})
-	uid := os.Getuid()
-	l, err := sys.GetLocalListener(p, uid, uid)
+	l, err := getLocalListener(p)
 	if err != nil {
-		return err
-	}
-	if err := os.Chmod(p, 0666); err != nil {
-		l.Close()
-		return err
+		return errors.Wrap(err, "creating trace controller listener")
 	}
 	go server.Serve(l)
 	return nil

cmd/buildkitd/main_unix.go

@@ -6,8 +6,11 @@ package main
 import (
 	"crypto/tls"
 	"net"
+	"os"
+	"path/filepath"
 	"syscall"
 
+	"github.com/containerd/containerd/sys"
 	"github.com/coreos/go-systemd/v22/activation"
 	"github.com/pkg/errors"
 )
@@ -43,3 +46,20 @@ func listenFD(addr string, tlsConfig *tls.Config) (net.Listener, error) {
 	//TODO: systemd fd selection (default is 3)
 	return nil, errors.New("not supported yet")
 }
+
+func traceSocketPath(root string) string {
+	return filepath.Join(root, "otel-grpc.sock")
+}
+
+func getLocalListener(listenerPath string) (net.Listener, error) {
+	uid := os.Getuid()
+	l, err := sys.GetLocalListener(listenerPath, uid, uid)
+	if err != nil {
+		return nil, err
+	}
+	if err := os.Chmod(listenerPath, 0666); err != nil {
+		l.Close()
+		return nil, err
+	}
+	return l, nil
+}

cmd/buildkitd/main_windows.go

@@ -7,11 +7,37 @@ import (
 	"crypto/tls"
 	"net"
 
+	"github.com/Microsoft/go-winio"
 	_ "github.com/moby/buildkit/solver/llbsolver/ops"
 	_ "github.com/moby/buildkit/util/system/getuserinfo"
 	"github.com/pkg/errors"
 )
 
+const (
+	defaultTraceSocketPath = `\\.\pipe\buildkit-otel-grpc`
+)
+
 func listenFD(addr string, tlsConfig *tls.Config) (net.Listener, error) {
 	return nil, errors.New("listening server on fd not supported on windows")
 }
+
+func traceSocketPath(root string) string {
+	return defaultTraceSocketPath
+}
+
+func getLocalListener(listenerPath string) (net.Listener, error) {
+	pc := &winio.PipeConfig{
+		// Allow generic read and generic write access to authenticated users
+		// and system users. On Linux, this pipe seems to be given rw access to
+		// user, group and others (666).
+		// TODO(gabriel-samfira): should we restrict access to this pipe to just
+		// authenticated users? Or Administrators group?
+		SecurityDescriptor: "D:P(A;;GRGW;;;AU)(A;;GRGW;;;SY)",
+	}
+
+	listener, err := winio.ListenPipe(listenerPath, pc)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating listener")
+	}
+	return listener, nil
+}

executor/oci/spec.go

@@ -35,6 +35,12 @@ const (
 	NoProcessSandbox
 )
 
+var tracingEnvVars = []string{
+	"OTEL_TRACES_EXPORTER=otlp",
+	"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=" + getTracingSocket(),
+	"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc",
+}
+
 func (pm ProcessMode) String() string {
 	switch pm {
 	case ProcessSandbox:
@@ -112,7 +118,7 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
 
 	if tracingSocket != "" {
 		// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md
-		meta.Env = append(meta.Env, "OTEL_TRACES_EXPORTER=otlp", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=unix:///dev/otel-grpc.sock", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc")
+		meta.Env = append(meta.Env, tracingEnvVars...)
 		meta.Env = append(meta.Env, traceexec.Environ(ctx)...)
 	}
 
@@ -183,12 +189,7 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
 	}
 
 	if tracingSocket != "" {
-		s.Mounts = append(s.Mounts, specs.Mount{
-			Destination: "/dev/otel-grpc.sock",
-			Type:        "bind",
-			Source:      tracingSocket,
-			Options:     []string{"ro", "rbind"},
-		})
+		s.Mounts = append(s.Mounts, getTracingSocketMount(tracingSocket))
 	}
 
 	s.Mounts = dedupMounts(s.Mounts)

executor/oci/spec_unix.go

@@ -21,6 +21,10 @@ import (
 	"github.com/pkg/errors"
 )
 
+const (
+	tracingSocketPath = "/dev/otel-grpc.sock"
+)
+
 func generateMountOpts(resolvConf, hostsFile string) ([]oci.SpecOpts, error) {
 	return []oci.SpecOpts{
 		// https://github.com/moby/buildkit/issues/429
@@ -122,3 +126,16 @@ func withDefaultProfile() oci.SpecOpts {
 		return err
 	}
 }
+
+func getTracingSocketMount(socket string) specs.Mount {
+	return specs.Mount{
+		Destination: tracingSocketPath,
+		Type:        "bind",
+		Source:      socket,
+		Options:     []string{"ro", "rbind"},
+	}
+}
+
+func getTracingSocket() string {
+	return fmt.Sprintf("unix://%s", tracingSocketPath)
+}

executor/oci/spec_windows.go

@@ -4,12 +4,20 @@
 package oci
 
 import (
+	"fmt"
+	"path/filepath"
+
 	"github.com/containerd/containerd/oci"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/moby/buildkit/solver/pb"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 )
 
+const (
+	tracingSocketPath = "//./pipe/otel-grpc"
+)
+
 func generateMountOpts(resolvConf, hostsFile string) ([]oci.SpecOpts, error) {
 	return nil, nil
 }
@@ -43,3 +51,15 @@ func generateRlimitOpts(ulimits []*pb.Ulimit) ([]oci.SpecOpts, error) {
 	}
 	return nil, errors.New("no support for POSIXRlimit on Windows")
 }
+
+func getTracingSocketMount(socket string) specs.Mount {
+	return specs.Mount{
+		Destination: filepath.FromSlash(tracingSocketPath),
+		Source:      socket,
+		Options:     []string{"ro"},
+	}
+}
+
+func getTracingSocket() string {
+	return fmt.Sprintf("npipe://%s", filepath.ToSlash(tracingSocketPath))
+}