Add support for target scraping with SPIFFE

This is the first step toward
#702

The feature is not available until the calling code invokes the
WithSpiffeSourceFactory HTTPClientOptions, so no behaviour is yet changed.
Use of that option will be the subject of a future change to
https://github.com/prometheus/prometheus

SPIFFE replaces the usual peer X.509 certificate verification
algorithm with its own that checks SPIFFE IDs in URI SANs and checks
against different sets of trust roots for different SPIFFE trust domains.
Accordingly, none of the other TLS configuration parameters are applicable
when SPIFFE is configured and vice versa and this mutual exclusivity is
enforced in tls_config.

There are two ways to set which SPIFFE ID should be expected from a
scrape endpoint that is using HTTPS and SPIFFE: in the tls_config, and
per-request. The former would be expected mainly on static scrape configs
that define a single (perhaps replicated) scrape endpoint. For scrape
configs with target discovery it is expected that different endpoints
would present certificates with different SPIFFE IDs and so the
per-request version would be used. It is intended that the peer's
expected SPIFFE ID should come from a new special label __spiffe_id__
which would be populated by target discovery. That too will be part of
the next change. For the purposes of this library, the per-request peer
SPIFFE ID is supplied in the Context accompanying the Request.

Signed-off-by: Kim Vandry <vandry@TZoNE.ORG>

Author: vandry

config/http_config.go

@@ -34,6 +34,11 @@ import (
 
 	"github.com/golang-jwt/jwt/v5"
 	"github.com/mwitkow/go-conntrack"
+	spiffebundle "github.com/spiffe/go-spiffe/v2/bundle/x509bundle"
+	"github.com/spiffe/go-spiffe/v2/spiffeid"
+	"github.com/spiffe/go-spiffe/v2/spiffetls"
+	spiffetlsconfig "github.com/spiffe/go-spiffe/v2/spiffetls/tlsconfig"
+	spiffesvid "github.com/spiffe/go-spiffe/v2/svid/x509svid"
 	"go.yaml.in/yaml/v2"
 	"golang.org/x/net/http/httpproxy"
 	"golang.org/x/net/http2"
@@ -495,6 +500,11 @@ type DialContextFunc func(context.Context, string, string) (net.Conn, error)
 // NewTLSConfigFunc returns tls.Config.
 type NewTLSConfigFunc func(context.Context, *TLSConfig, ...TLSConfigOption) (*tls.Config, error)
 
+type SpiffeSvidAndBundleSource interface {
+	spiffesvid.Source
+	spiffebundle.Source
+}
+
 type httpClientOptions struct {
 	dialContextFunc   DialContextFunc
 	newTLSConfigFunc  NewTLSConfigFunc
@@ -504,6 +514,7 @@ type httpClientOptions struct {
 	userAgent         string
 	host              string
 	secretManager     SecretManager
+	spiffeSourceFn    func() (SpiffeSvidAndBundleSource, error)
 }
 
 // HTTPClientOption defines an option that can be applied to the HTTP client.
@@ -569,6 +580,20 @@ func WithHost(host string) HTTPClientOption {
 	})
 }
 
+// WithSpiffeSourceFactory allows SPIFFE to be used with this HTTP client.
+// The provided function should return the same X509Source on every call
+// since all clients can share the same source. The source may either
+// already exist (in which case the function can just return a fixed value)
+// or be created on demand (in which case no X509Source will be created unless
+// SPIFFE is configured and used). The returned X509Source will not be closed
+// during the lifetime of the HTTPClient. The default is that there is no
+// factory function and SPIFFE is not available.
+func WithSpiffeSourceFactory(fn func() (SpiffeSvidAndBundleSource, error)) HTTPClientOption {
+	return httpClientOptionFunc(func(opts *httpClientOptions) {
+		opts.spiffeSourceFn = fn
+	})
+}
+
 type secretManagerOption struct {
 	secretManager SecretManager
 }
@@ -623,6 +648,28 @@ func NewRoundTripperFromConfig(cfg HTTPClientConfig, name string, optFuncs ...HT
 	return NewRoundTripperFromConfigWithContext(context.Background(), cfg, name, optFuncs...)
 }
 
+func makeSpiffeDialer(configuredSpiffeID string, getSource func() (SpiffeSvidAndBundleSource, error)) func(ctx context.Context, network, addr string) (net.Conn, error) {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		ids := configuredSpiffeID
+		if idc := ctx.Value(SpiffeIDContextValue); idc != nil {
+			ids = idc.(string)
+		}
+		peer, err := spiffeid.FromString(ids)
+		if err != nil {
+			return nil, fmt.Errorf("unparsable SPIFFE ID %q: %w", ids, err)
+		}
+		if getSource == nil {
+			return nil, errors.New("SPIFFE requested but not configured")
+		}
+		source, err := getSource()
+		if err != nil {
+			return nil, err
+		}
+		mode := spiffetls.MTLSClientWithRawConfig(spiffetlsconfig.AuthorizeID(peer), source, source)
+		return spiffetls.DialWithMode(ctx, network, addr, mode)
+	}
+}
+
 // NewRoundTripperFromConfigWithContext returns a new HTTP RoundTripper configured for the
 // given config.HTTPClientConfig and config.HTTPClientOption.
 // The name is used as go-conntrack metric label.
@@ -646,6 +693,11 @@ func NewRoundTripperFromConfigWithContext(ctx context.Context, cfg HTTPClientCon
 	}
 
 	newRT := func(tlsConfig *tls.Config) (http.RoundTripper, error) {
+		var dialTLS func(ctx context.Context, network, addr string) (net.Conn, error)
+		if tlsConfig == nil {
+			// Use SPIFFE
+			dialTLS = makeSpiffeDialer(cfg.TLSConfig.SpiffeID, opts.spiffeSourceFn)
+		}
 		// The only timeout we care about is the configured scrape timeout.
 		// It is applied on request. So we leave out any timings here.
 		var rt http.RoundTripper = &http.Transport{
@@ -660,6 +712,7 @@ func NewRoundTripperFromConfigWithContext(ctx context.Context, cfg HTTPClientCon
 			TLSHandshakeTimeout:   10 * time.Second,
 			ExpectContinueTimeout: 1 * time.Second,
 			DialContext:           dialContext,
+			DialTLSContext:        dialTLS,
 		}
 		if opts.http2Enabled && cfg.EnableHTTP2 {
 			http2t, err := http2.ConfigureTransports(rt.(*http.Transport))
@@ -736,6 +789,10 @@ func NewRoundTripperFromConfigWithContext(ctx context.Context, cfg HTTPClientCon
 		return rt, nil
 	}
 
+	if cfg.TLSConfig.SpiffeID != "" || cfg.TLSConfig.UseSpiffe {
+		return newRT(nil)
+	}
+
 	tlsConfig, err := opts.newTLSConfigFunc(ctx, &cfg.TLSConfig, WithSecretManager(opts.secretManager))
 	if err != nil {
 		return nil, err
@@ -1226,6 +1283,13 @@ type TLSConfig struct {
 	MinVersion TLSVersion `yaml:"min_version,omitempty" json:"min_version,omitempty"`
 	// Maximum TLS version.
 	MaxVersion TLSVersion `yaml:"max_version,omitempty" json:"max_version,omitempty"`
+	// Use SPIFFE to configure TLS. The special label `__spiffe_id__` on the
+	// scrape target configures the SPIFFE ID that must be presented.
+	UseSpiffe bool `yaml:"use_spiffe,omitempty" json:"use_spiffe,omitempty"`
+	// Use SPIFFE to configure TLS. The special label `__spiffe_id__` on the
+	// scrape target (first) or the value of this parameter (otherwise)
+	// configures the SPIFFE ID that must be presented.
+	SpiffeID string `yaml:"spiffe_id,omitempty" json:"spiffe_id,omitempty"`
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -1267,6 +1331,10 @@ func (c *TLSConfig) Validate() error {
 		return errors.New("exactly one of cert or cert_file must be configured when a client key is configured")
 	}
 
+	if (len(c.CA) > 0 || len(c.CAFile) > 0 || len(c.CARef) > 0 || len(c.Cert) > 0 || len(c.CertFile) > 0 || len(c.CertRef) > 0 || len(c.Key) > 0 || len(c.KeyFile) > 0 || len(c.KeyRef) > 0 || len(c.ServerName) > 0 || c.InsecureSkipVerify) && (len(c.SpiffeID) > 0 || c.UseSpiffe) {
+		return errors.New("either SPIFFE settings or other TLSConfig settings may be set but not both")
+	}
+
 	return nil
 }
 
@@ -1625,3 +1693,7 @@ func (c *ProxyConfig) Proxy() (fn func(*http.Request) (*url.URL, error)) {
 func (c *ProxyConfig) GetProxyConnectHeader() http.Header {
 	return c.ProxyConnectHeader.HTTPHeader()
 }
+
+type spiffeIDContextValue bool
+
+const SpiffeIDContextValue = spiffeIDContextValue(false)

config/http_config_test.go

@@ -15,10 +15,12 @@ package config
 
 import (
 	"context"
+	"crypto"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
@@ -36,6 +38,9 @@ import (
 	"testing"
 	"time"
 
+	spiffebundle "github.com/spiffe/go-spiffe/v2/bundle/x509bundle"
+	"github.com/spiffe/go-spiffe/v2/spiffeid"
+	spiffesvid "github.com/spiffe/go-spiffe/v2/svid/x509svid"
 	"github.com/stretchr/testify/require"
 	"go.yaml.in/yaml/v2"
 )
@@ -54,6 +59,12 @@ const (
 	MissingCert           = "missing/cert.crt"
 	MissingKey            = "missing/secret.key"
 
+	SpiffeWorkload1Cert = "testdata/spiffe.workload1.cert.pem"
+	SpiffeWorkload1Key  = "testdata/spiffe.workload1.key.pem"
+	SpiffeWorkload2Cert = "testdata/spiffe.workload2.cert.pem"
+	SpiffeWorkload2Key  = "testdata/spiffe.workload2.key.pem"
+	SpiffeBundle        = "testdata/spiffe.bundle.pem"
+
 	ExpectedMessage                   = "I'm here to serve you!!!"
 	ExpectedError                     = "expected error"
 	AuthorizationCredentials          = "theanswertothegreatquestionoflifetheuniverseandeverythingisfortytwo"
@@ -171,6 +182,37 @@ func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*httpt
 	return testServer, nil
 }
 
+func newSpiffeTestServer() (*httptest.Server, error) {
+	handler := func(w http.ResponseWriter, _ *http.Request) {
+		fmt.Fprint(w, ExpectedMessage)
+	}
+	testServer := httptest.NewUnstartedServer(http.HandlerFunc(handler))
+
+	tlsCAChain, err := os.ReadFile(SpiffeBundle)
+	if err != nil {
+		return nil, fmt.Errorf("Can't read %s", SpiffeBundle)
+	}
+	serverCertificate, err := tls.LoadX509KeyPair(SpiffeWorkload1Cert, SpiffeWorkload1Key)
+	if err != nil {
+		return nil, fmt.Errorf("Can't load X509 key pair %s - %s", SpiffeWorkload1Cert, SpiffeWorkload1Key)
+	}
+
+	rootCAs := x509.NewCertPool()
+	rootCAs.AppendCertsFromPEM(tlsCAChain)
+
+	testServer.TLS = &tls.Config{
+		Certificates: make([]tls.Certificate, 1),
+		RootCAs:      rootCAs,
+		ClientAuth:   tls.RequireAndVerifyClientCert,
+		ClientCAs:    rootCAs,
+	}
+	testServer.TLS.Certificates[0] = serverCertificate
+
+	testServer.StartTLS()
+
+	return testServer, nil
+}
+
 func TestNewClientFromConfig(t *testing.T) {
 	newClientValidConfig := []struct {
 		clientConfig HTTPClientConfig
@@ -1280,6 +1322,131 @@ func TestTLSRoundTripper_Inline(t *testing.T) {
 	}
 }
 
+type testSpiffeSource struct{}
+
+func (*testSpiffeSource) GetX509SVID() (*spiffesvid.SVID, error) {
+	cert, err := tls.LoadX509KeyPair(SpiffeWorkload2Cert, SpiffeWorkload2Key)
+	if err != nil {
+		return nil, fmt.Errorf("Can't load X509 key pair %s - %s", SpiffeWorkload2Cert, SpiffeWorkload2Key)
+	}
+	if signer, ok := cert.PrivateKey.(crypto.Signer); ok {
+		return &spiffesvid.SVID{
+			ID:           spiffeid.RequireFromString("spiffe://example.org/workload2"),
+			Certificates: []*x509.Certificate{cert.Leaf},
+			PrivateKey:   signer,
+		}, nil
+	}
+	return nil, errors.New("private key is not crypto.Signer")
+}
+
+func (*testSpiffeSource) GetX509BundleForTrustDomain(trustDomain spiffeid.TrustDomain) (*spiffebundle.Bundle, error) {
+	if trustDomain.Name() == "example.org" {
+		bundlePem, err := os.ReadFile(SpiffeBundle)
+		if err != nil {
+			return nil, fmt.Errorf("Can't read %s", SpiffeBundle)
+		}
+		var bundle []*x509.Certificate
+		for len(bundlePem) > 0 {
+			b, more := pem.Decode(bundlePem)
+			c, err := x509.ParseCertificate(b.Bytes)
+			if err != nil {
+				return nil, fmt.Errorf("Can't parse %s as a certificate", SpiffeBundle)
+			}
+			bundlePem = more
+			bundle = append(bundle, c)
+		}
+		return spiffebundle.FromX509Authorities(trustDomain, bundle), nil
+	}
+	return nil, fmt.Errorf("No bundle for trust domain %v", trustDomain)
+}
+
+func spiffeMaker() (SpiffeSvidAndBundleSource, error) {
+	return &testSpiffeSource{}, nil
+}
+
+func TestTLSRoundTripper_SPIFFE(t *testing.T) {
+	testServer, err := newSpiffeTestServer()
+	require.NoError(t, err)
+	defer testServer.Close()
+
+	testCases := []struct {
+		disabled  bool
+		useSpiffe bool
+		confID    string
+		ctxID     string
+
+		errMsg string
+	}{
+		{
+			disabled: true,
+			confID:   "spiffe://trust.domain/foo",
+			errMsg:   "SPIFFE requested but not configured",
+		},
+		{
+			confID: "spiffe://example.org/workload1",
+		},
+		{
+			confID: "spiffe://example.org/wrong",
+			errMsg: "unexpected ID",
+		},
+		{
+			confID: "unparsable",
+			errMsg: "unparsable",
+		},
+		{
+			useSpiffe: true,
+			ctxID:     "spiffe://example.org/workload1",
+		},
+		{
+			confID: "spiffe://overridden/ignored",
+			ctxID:  "spiffe://example.org/workload1",
+		},
+	}
+
+	for i, tc := range testCases {
+		tc := tc
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			cfg := HTTPClientConfig{
+				TLSConfig: TLSConfig{
+					UseSpiffe: tc.useSpiffe,
+					SpiffeID:  tc.confID,
+				},
+			}
+
+			var opts []HTTPClientOption
+			if !tc.disabled {
+				opts = append(opts, WithSpiffeSourceFactory(spiffeMaker))
+			}
+			c, err := NewClientFromConfig(cfg, "test", opts...)
+			require.NoErrorf(t, err, "Error creating HTTP client: %v", err)
+			req, err := http.NewRequest(http.MethodGet, testServer.URL, nil)
+			require.NoErrorf(t, err, "Error creating HTTP request: %v", err)
+			ctx := context.Background()
+			if tc.ctxID != "" {
+				ctx = context.WithValue(ctx, SpiffeIDContextValue, tc.ctxID)
+			}
+			r, err := c.Do(req.WithContext(ctx))
+			if tc.errMsg != "" {
+				require.ErrorContainsf(t, err, tc.errMsg, "Expected error message to contain %q, got %q", tc.errMsg, err)
+				return
+			} else if err != nil {
+				t.Fatalf("Error executing HTTP request: %v", err)
+			}
+
+			b, err := io.ReadAll(r.Body)
+			r.Body.Close()
+			if err != nil {
+				t.Errorf("Can't read the server response body")
+			}
+
+			got := strings.TrimSpace(string(b))
+			if ExpectedMessage != got {
+				t.Errorf("The expected message %q differs from the obtained message %q", ExpectedMessage, got)
+			}
+		})
+	}
+}
+
 func TestTLSRoundTripperRaces(t *testing.T) {
 	bs := getCertificateBlobs(t)
 

config/testdata/spiffe.bundle.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICBDCCAaqgAwIBAgIRAP6xM1DuTb3X0lAhRTLmNhMwCgYIKoZIzj0EAwIwUDEL
+MAkGA1UEBhMCVVMxDzANBgNVBAoTBlNQSUZGRTEwMC4GA1UEBRMnMzM4NTQzOTg4
+Mjg4MjIzNDk5NzE4MjE1ODYwMjk1NDUyNDcyODUxMCAXDTI1MDkwNzA5MjQzMVoY
+DzIyMjUwNzIxMDkyNDQxWjBQMQswCQYDVQQGEwJVUzEPMA0GA1UEChMGU1BJRkZF
+MTAwLgYDVQQFEyczMzg1NDM5ODgyODgyMjM0OTk3MTgyMTU4NjAyOTU0NTI0NzI4
+NTEwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQtQEBKDICx1ywQOjIiYGOO6DI8
+edAy3MxMQrWs737AgD45ixo1fNIGHfpD9v0WH3kq5C9ycB5YNpIaSdfXJiiJo2Mw
+YTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUZ5LV
+HKgEZ5Kvs50NaJboDBt+U/owHwYDVR0RBBgwFoYUc3BpZmZlOi8vZXhhbXBsZS5v
+cmcwCgYIKoZIzj0EAwIDSAAwRQIgOvUg85SrI2hmeCStTkCF9or0Vgcuzdifwfq5
+qQBJ9W8CIQD13tnaExaopel4BbweByAnXHqYEwomLlMbLs6ldz7a7w==
+-----END CERTIFICATE-----

config/testdata/spiffe.workload1.cert.pem

@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICGjCCAb+gAwIBAgIQS8qrtPjeVHsY0SXsfT8B/zAKBggqhkjOPQQDAjBQMQsw
+CQYDVQQGEwJVUzEPMA0GA1UEChMGU1BJRkZFMTAwLgYDVQQFEyczMzg1NDM5ODgy
+ODgyMjM0OTk3MTgyMTU4NjAyOTU0NTI0NzI4NTEwIBcNMjUwOTA3MDkyNjUzWhgP
+MjEwMDA4MjAwOTI3MDNaMB0xCzAJBgNVBAYTAlVTMQ4wDAYDVQQKEwVTUElSRTBZ
+MBMGByqGSM49AgEGCCqGSM49AwEHA0IABO+FhNA7IYyFNA/BDOovyNVlwZVesbfj
+knv/alVHrJ8Z5PPLv5RC4EqxDUCIdYH8IcfpXbRUsxV3Y8oIRkQFWpOjgaswgagw
+DgYDVR0PAQH/BAQDAgOoMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAM
+BgNVHRMBAf8EAjAAMB0GA1UdDgQWBBQJ/hirbRAZC32OWIHWBdVfOF67gzAfBgNV
+HSMEGDAWgBRnktUcqARnkq+znQ1olugMG35T+jApBgNVHREEIjAghh5zcGlmZmU6
+Ly9leGFtcGxlLm9yZy93b3JrbG9hZDEwCgYIKoZIzj0EAwIDSQAwRgIhAPIkKNdq
+SBb9ROIe2eo5ed1/0ay89UMjd+dxlkuyX8jAAiEAx1P1bBnzAVhuQx0YbzG2lJnK
+QBCtPcGZ/amdWZt9G0A=
+-----END CERTIFICATE-----

config/testdata/spiffe.workload1.key.pem

@@ -0,0 +1,5 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgorRBVyov8trytD8G
+v094aUhNkeOlBfV6mdZfC7rm66+hRANCAATvhYTQOyGMhTQPwQzqL8jVZcGVXrG3
+45J7/2pVR6yfGeTzy7+UQuBKsQ1AiHWB/CHH6V20VLMVd2PKCEZEBVqT
+-----END PRIVATE KEY-----