Merge pull request #696 from jetstack/VC-43403-client

[VC-43403] CyberArk(agent): add support for MachineHub output mode

Author: wallrj-cyberark

.envrc.template

@@ -0,0 +1,21 @@
+# Example .envrc file for use with direnv.
+# Copy this file to .envrc and edit the values as required.
+# Do not check in your .envrc file to source control as it may contain secrets.
+
+# The following variables are required by the E2E test script: ./hack/e2e/test.sh.
+export VEN_API_KEY=       # your Venafi Cloud API key with full permissions
+export VEN_API_KEY_PULL=  # your Venafi Cloud API key with pull-only permissions
+export VEN_ZONE=          # the Venafi Cloud zone to use for certificate requests
+export VEN_VCP_REGION=    # the Venafi Cloud region to use (us or eu)
+export VEN_API_HOST=      # the Venafi Cloud API host (usually api.venafi.cloud or api.venafi.eu)
+export OCI_BASE=          # the base URL for the OCI registry where the Agent chart and image will be pushed
+export CLOUDSDK_CORE_PROJECT= # the GCP project ID where a GKE cluster will be created.
+export CLOUDSDK_COMPUTE_ZONE= # the GCP zone where a GKE cluster will be created. E.g. europe-west2-b
+export CLUSTER_NAME=          # the name of the GKE cluster which will be created. E.g. cluster-1
+
+# The following variables are required for CyberArk / MachineHub integration tests.
+export ARK_SUBDOMAIN=      # your CyberArk tenant subdomain
+export ARK_USERNAME=       # your CyberArk username
+export ARK_SECRET=         # your CyberArk password
+# OPTIONAL: the URL for the CyberArk Discovery API if not using the production environment
+export ARK_DISCOVERY_API=https://platform-discovery.integration-cyberark.cloud/api/v2

.gitignore

@@ -14,3 +14,4 @@ predicate.json
 *.tgz
 
 _bin
+.envrc

examples/machinehub.yaml

@@ -0,0 +1,12 @@
+# An example agent config for MachineHub output mode.
+#
+# For example:
+#
+#  export ARK_SUBDOMAIN=      # your CyberArk tenant subdomain
+#  export ARK_USERNAME=       # your CyberArk username
+#  export ARK_SECRET=         # your CyberArk password
+#  go run . agent --one-shot --machine-hub -v 6 --agent-config-file ./examples/machinehub.yaml
+
+data-gatherers:
+  - kind: "dummy"
+    name: "dummy"

pkg/agent/config.go

@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"crypto/x509"
 	"fmt"
 	"io"
 	"net/url"
@@ -10,9 +11,11 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/hashicorp/go-multierror"
+	"github.com/jetstack/venafi-connection-lib/http_client"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/transport"
 
 	"github.com/jetstack/preflight/api"
 	"github.com/jetstack/preflight/pkg/client"
@@ -334,6 +337,7 @@ const (
 	VenafiCloudKeypair          OutputMode = "Venafi Cloud Key Pair Service Account"
 	VenafiCloudVenafiConnection OutputMode = "Venafi Cloud VenafiConnection"
 	LocalFile                   OutputMode = "Local File"
+	MachineHub                  OutputMode = "MachineHub"
 )
 
 // The command-line flags and the config file are combined into this struct by
@@ -420,6 +424,9 @@ func ValidateAndCombineConfig(log logr.Logger, cfg Config, flags AgentCmdFlags)
 		case !flags.VenafiCloudMode && flags.CredentialsPath != "":
 			mode = JetstackSecureOAuth
 			reason = "--credentials-file was specified without --venafi-cloud"
+		case flags.MachineHubMode:
+			mode = MachineHub
+			reason = "--machine-hub was specified"
 		case flags.OutputPath != "":
 			mode = LocalFile
 			reason = "--output-path was specified"
@@ -433,6 +440,7 @@ func ValidateAndCombineConfig(log logr.Logger, cfg Config, flags AgentCmdFlags)
 				" - Use --venafi-connection for the " + string(VenafiCloudVenafiConnection) + " mode.\n" +
 				" - Use --credentials-file alone if you want to use the " + string(JetstackSecureOAuth) + " mode.\n" +
 				" - Use --api-token if you want to use the " + string(JetstackSecureAPIToken) + " mode.\n" +
+				" - Use --machine-hub if you want to use the " + string(MachineHub) + " mode.\n" +
 				" - Use --output-path or output-path in the config file for " + string(LocalFile) + " mode.")
 		}
 
@@ -548,6 +556,13 @@ func ValidateAndCombineConfig(log logr.Logger, cfg Config, flags AgentCmdFlags)
 			}
 			organizationID = cfg.OrganizationID
 			clusterID = cfg.ClusterID
+		case MachineHub:
+			if cfg.ClusterID != "" {
+				log.Info(fmt.Sprintf(`Ignoring the cluster_id field in the config file. This field is not needed in %s mode.`, res.OutputMode))
+			}
+			if cfg.OrganizationID != "" {
+				log.Info(fmt.Sprintf(`Ignoring the organization_id field in the config file. This field is not needed in %s mode.`, res.OutputMode))
+			}
 		}
 		res.OrganizationID = organizationID
 		res.ClusterID = clusterID
@@ -762,6 +777,17 @@ func validateCredsAndCreateClient(log logr.Logger, flagCredentialsPath, flagClie
 		}
 	case LocalFile:
 		outputClient = client.NewFileClient(cfg.OutputPath)
+	case MachineHub:
+		var (
+			err     error
+			rootCAs *x509.CertPool
+		)
+		httpClient := http_client.NewDefaultClient(version.UserAgent(), rootCAs)
+		httpClient.Transport = transport.NewDebuggingRoundTripper(httpClient.Transport, transport.DebugByContext)
+		outputClient, err = client.NewCyberArk(httpClient)
+		if err != nil {
+			errs = multierror.Append(errs, err)
+		}
 	default:
 		panic(fmt.Errorf("programmer mistake: output mode not implemented: %s", cfg.OutputMode))
 	}

pkg/agent/config_test.go

@@ -199,6 +199,7 @@ func Test_ValidateAndCombineConfig(t *testing.T) {
 			 - Use --venafi-connection for the Venafi Cloud VenafiConnection mode.
 			 - Use --credentials-file alone if you want to use the Jetstack Secure OAuth mode.
 			 - Use --api-token if you want to use the Jetstack Secure API Token mode.
+			 - Use --machine-hub if you want to use the MachineHub mode.
 			 - Use --output-path or output-path in the config file for Local File mode.`))
 		assert.Nil(t, cl)
 	})
@@ -617,6 +618,38 @@ func Test_ValidateAndCombineConfig(t *testing.T) {
 		assert.Equal(t, VenafiCloudVenafiConnection, got.OutputMode)
 	})
 
+	t.Run("--machine-hub selects MachineHub mode", func(t *testing.T) {
+		t.Setenv("POD_NAMESPACE", "venafi")
+		t.Setenv("KUBECONFIG", withFile(t, fakeKubeconfig))
+		t.Setenv("ARK_SUBDOMAIN", "tlspk")
+		t.Setenv("ARK_USERNAME", "[email protected]")
+		t.Setenv("ARK_SECRET", "test-secret")
+		got, cl, err := ValidateAndCombineConfig(discardLogs(),
+			withConfig(""),
+			withCmdLineFlags("--period", "1m", "--machine-hub"))
+		require.NoError(t, err)
+		assert.Equal(t, MachineHub, got.OutputMode)
+		assert.IsType(t, &client.CyberArkClient{}, cl)
+	})
+
+	t.Run("--machine-hub without required environment variables", func(t *testing.T) {
+		t.Setenv("POD_NAMESPACE", "venafi")
+		t.Setenv("KUBECONFIG", withFile(t, fakeKubeconfig))
+		t.Setenv("ARK_SUBDOMAIN", "")
+		t.Setenv("ARK_USERNAME", "")
+		t.Setenv("ARK_SECRET", "")
+		got, cl, err := ValidateAndCombineConfig(discardLogs(),
+			withConfig(""),
+			withCmdLineFlags("--period", "1m", "--machine-hub"))
+		assert.Equal(t, CombinedConfig{}, got)
+		assert.Nil(t, cl)
+		assert.EqualError(t, err, testutil.Undent(`
+			validating creds: failed loading config using the MachineHub mode: 1 error occurred:
+				* missing environment variables: ARK_SUBDOMAIN, ARK_USERNAME, ARK_SECRET
+
+	   `))
+	})
+
 	t.Run("argument: --output-file selects local file mode", func(t *testing.T) {
 		log, gotLog := recordLogs(t)
 		got, outputClient, err := ValidateAndCombineConfig(log,

pkg/client/client_cyberark.go

@@ -1,9 +1,62 @@
 package client
 
 import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/jetstack/preflight/api"
+	"github.com/jetstack/preflight/pkg/internal/cyberark"
 	"github.com/jetstack/preflight/pkg/internal/cyberark/dataupload"
+	"github.com/jetstack/preflight/pkg/version"
 )
 
-type CyberArkClient = dataupload.CyberArkClient
+// CyberArkClient is a client for publishing data readings to CyberArk's discoverycontext API.
+type CyberArkClient struct {
+	configLoader cyberark.ClientConfigLoader
+	httpClient   *http.Client
+}
+
+var _ Client = &CyberArkClient{}
+
+// NewCyberArk initializes a CyberArk client using configuration from environment variables.
+// It requires an HTTP client to be provided, which will be used for making requests.
+// The environment variables ARK_SUBDOMAIN, ARK_USERNAME, and ARK_SECRET must be set for authentication.
+// If the configuration is invalid or missing, an error is returned.
+func NewCyberArk(httpClient *http.Client) (*CyberArkClient, error) {
+	configLoader := cyberark.LoadClientConfigFromEnvironment
+	_, err := configLoader()
+	if err != nil {
+		return nil, err
+	}
+	return &CyberArkClient{
+		configLoader: configLoader,
+		httpClient:   httpClient,
+	}, nil
+}
+
+// PostDataReadingsWithOptions uploads data readings to CyberArk.
+// It initializes a data upload client with the configured HTTP client and credentials,
+// then uploads a snapshot.
+// The supplied Options are not used by this publisher.
+func (o *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, readings []*api.DataReading, _ Options) error {
+	cfg, err := o.configLoader()
+	if err != nil {
+		return err
+	}
+	datauploadClient, err := cyberark.NewDatauploadClient(ctx, o.httpClient, cfg)
+	if err != nil {
+		return fmt.Errorf("while initializing data upload client: %s", err)
+	}
 
-var NewCyberArkClient = dataupload.New
+	err = datauploadClient.PutSnapshot(ctx, dataupload.Snapshot{
+		// Temporary hard coded cluster ID.
+		// TODO(wallrj): The clusterID will eventually be extracted from the supplied readings.
+		ClusterID:    "success-cluster-id",
+		AgentVersion: version.PreflightVersion,
+	})
+	if err != nil {
+		return fmt.Errorf("while uploading snapshot: %s", err)
+	}
+	return nil
+}

pkg/client/client_cyberark_test.go

@@ -0,0 +1,74 @@
+package client_test
+
+import (
+	"crypto/x509"
+	"errors"
+	"testing"
+
+	"github.com/jetstack/venafi-connection-lib/http_client"
+	"github.com/stretchr/testify/require"
+	"k8s.io/client-go/transport"
+	"k8s.io/klog/v2"
+	"k8s.io/klog/v2/ktesting"
+
+	"github.com/jetstack/preflight/api"
+	"github.com/jetstack/preflight/pkg/client"
+	"github.com/jetstack/preflight/pkg/internal/cyberark"
+	"github.com/jetstack/preflight/pkg/internal/cyberark/servicediscovery"
+	"github.com/jetstack/preflight/pkg/testutil"
+	"github.com/jetstack/preflight/pkg/version"
+
+	_ "k8s.io/klog/v2/ktesting/init"
+)
+
+// TestCyberArkClient_PostDataReadingsWithOptions_MockAPI demonstrates that the
+// dataupload code works with the mock CyberArk APIs.
+// The environment variables are chosen to match those expected by the mock
+// server.
+func TestCyberArkClient_PostDataReadingsWithOptions_MockAPI(t *testing.T) {
+	t.Setenv("ARK_SUBDOMAIN", servicediscovery.MockDiscoverySubdomain)
+	t.Setenv("ARK_USERNAME", "[email protected]")
+	t.Setenv("ARK_SECRET", "somepassword")
+	t.Run("success", func(t *testing.T) {
+		logger := ktesting.NewLogger(t, ktesting.DefaultConfig)
+		ctx := klog.NewContext(t.Context(), logger)
+
+		httpClient := testutil.FakeCyberArk(t)
+
+		c, err := client.NewCyberArk(httpClient)
+		require.NoError(t, err)
+
+		var readings []*api.DataReading
+		err = c.PostDataReadingsWithOptions(ctx, readings, client.Options{})
+		require.NoError(t, err)
+	})
+}
+
+// TestCyberArkClient_PostDataReadingsWithOptions_RealAPI demonstrates that the
+// dataupload code works with the real CyberArk APIs.
+//
+// To enable verbose request logging:
+//
+//	go test ./pkg/internal/cyberark/dataupload/... \
+//	  -v -count 1 -run TestCyberArkClient_PostDataReadingsWithOptions_RealAPI -args -testing.v 6
+func TestCyberArkClient_PostDataReadingsWithOptions_RealAPI(t *testing.T) {
+	t.Run("success", func(t *testing.T) {
+		logger := ktesting.NewLogger(t, ktesting.DefaultConfig)
+		ctx := klog.NewContext(t.Context(), logger)
+
+		var rootCAs *x509.CertPool
+		httpClient := http_client.NewDefaultClient(version.UserAgent(), rootCAs)
+		httpClient.Transport = transport.NewDebuggingRoundTripper(httpClient.Transport, transport.DebugByContext)
+
+		c, err := client.NewCyberArk(httpClient)
+		if err != nil {
+			if errors.Is(err, cyberark.ErrMissingEnvironmentVariables) {
+				t.Skipf("Skipping: %s", err)
+			}
+			require.NoError(t, err)
+		}
+		var readings []*api.DataReading
+		err = c.PostDataReadingsWithOptions(ctx, readings, client.Options{})
+		require.NoError(t, err)
+	})
+}

pkg/internal/cyberark/client.go

@@ -0,0 +1,72 @@
+package cyberark
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"os"
+
+	"github.com/jetstack/preflight/pkg/internal/cyberark/dataupload"
+	"github.com/jetstack/preflight/pkg/internal/cyberark/identity"
+	"github.com/jetstack/preflight/pkg/internal/cyberark/servicediscovery"
+)
+
+// ClientConfig holds the configuration needed to initialize a CyberArk client.
+type ClientConfig struct {
+	Subdomain string
+	Username  string
+	Secret    string
+}
+
+// ClientConfigLoader is a function type that loads and returns a ClientConfig.
+type ClientConfigLoader func() (ClientConfig, error)
+
+// ErrMissingEnvironmentVariables is returned when required environment variables are not set.
+var ErrMissingEnvironmentVariables = errors.New("missing environment variables: ARK_SUBDOMAIN, ARK_USERNAME, ARK_SECRET")
+
+// LoadClientConfigFromEnvironment loads the CyberArk client configuration from environment variables.
+// It expects the following environment variables to be set:
+// - ARK_SUBDOMAIN: The CyberArk subdomain to use.
+// - ARK_USERNAME: The username for authentication.
+// - ARK_SECRET: The secret for authentication.
+func LoadClientConfigFromEnvironment() (ClientConfig, error) {
+	subdomain := os.Getenv("ARK_SUBDOMAIN")
+	username := os.Getenv("ARK_USERNAME")
+	secret := os.Getenv("ARK_SECRET")
+
+	if subdomain == "" || username == "" || secret == "" {
+		return ClientConfig{}, ErrMissingEnvironmentVariables
+	}
+
+	return ClientConfig{
+		Subdomain: subdomain,
+		Username:  username,
+		Secret:    secret,
+	}, nil
+
+}
+
+// NewDatauploadClient initializes and returns a new CyberArk Data Upload client.
+// It performs service discovery to find the necessary API endpoints and authenticates
+// using the provided client configuration.
+func NewDatauploadClient(ctx context.Context, httpClient *http.Client, cfg ClientConfig) (*dataupload.CyberArkClient, error) {
+	discoveryClient := servicediscovery.New(httpClient)
+	serviceMap, err := discoveryClient.DiscoverServices(ctx, cfg.Subdomain)
+	if err != nil {
+		return nil, err
+	}
+	identityAPI := serviceMap.Identity.API
+	if identityAPI == "" {
+		return nil, errors.New("service discovery returned an empty identity API")
+	}
+	identityClient := identity.New(httpClient, identityAPI, cfg.Subdomain)
+	err = identityClient.LoginUsernamePassword(ctx, cfg.Username, []byte(cfg.Secret))
+	if err != nil {
+		return nil, err
+	}
+	discoveryAPI := serviceMap.DiscoveryContext.API
+	if discoveryAPI == "" {
+		return nil, errors.New("service discovery returned an empty discovery API")
+	}
+	return dataupload.New(httpClient, discoveryAPI, identityClient.AuthenticateRequest), nil
+}