Merge pull request #257 from fluxcd/gotk-standard

Conform with GitOps Toolkit standards

Author: stefanprodan

api/v1beta1/artifactgenerator_types.go

@@ -22,7 +22,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	"github.com/fluxcd/pkg/apis/meta"
+	gotkmeta "github.com/fluxcd/pkg/apis/meta"
 )
 
 const (
@@ -157,7 +157,7 @@ type CopyOperation struct {
 
 // ArtifactGeneratorStatus defines the observed state of ArtifactGenerator.
 type ArtifactGeneratorStatus struct {
-	meta.ReconcileRequestStatus `json:",inline"`
+	gotkmeta.ReconcileRequestStatus `json:",inline"`
 
 	// Conditions holds the conditions for the ArtifactGenerator.
 	// +optional

cmd/main.go

@@ -29,23 +29,24 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/utils/ptr"
 	ctrlruntime "sigs.k8s.io/controller-runtime"
+	ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlcfg "sigs.k8s.io/controller-runtime/pkg/config"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
-	"github.com/fluxcd/pkg/artifact/config"
-	"github.com/fluxcd/pkg/artifact/digest"
-	"github.com/fluxcd/pkg/artifact/server"
-	"github.com/fluxcd/pkg/artifact/storage"
-	"github.com/fluxcd/pkg/runtime/acl"
-	"github.com/fluxcd/pkg/runtime/client"
-	ctrl "github.com/fluxcd/pkg/runtime/controller"
-	"github.com/fluxcd/pkg/runtime/features"
-	"github.com/fluxcd/pkg/runtime/jitter"
-	"github.com/fluxcd/pkg/runtime/leaderelection"
-	"github.com/fluxcd/pkg/runtime/logger"
-	"github.com/fluxcd/pkg/runtime/pprof"
-	"github.com/fluxcd/pkg/runtime/probes"
+	gotkconfig "github.com/fluxcd/pkg/artifact/config"
+	gotkdigest "github.com/fluxcd/pkg/artifact/digest"
+	gotkserver "github.com/fluxcd/pkg/artifact/server"
+	gotkstorage "github.com/fluxcd/pkg/artifact/storage"
+	gotkacl "github.com/fluxcd/pkg/runtime/acl"
+	gotkclient "github.com/fluxcd/pkg/runtime/client"
+	gotkctrl "github.com/fluxcd/pkg/runtime/controller"
+	gotkfeatures "github.com/fluxcd/pkg/runtime/features"
+	gotkjitter "github.com/fluxcd/pkg/runtime/jitter"
+	gotkelection "github.com/fluxcd/pkg/runtime/leaderelection"
+	gotklogger "github.com/fluxcd/pkg/runtime/logger"
+	gotkpprof "github.com/fluxcd/pkg/runtime/pprof"
+	gotkprobes "github.com/fluxcd/pkg/runtime/probes"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 
 	swapi "github.com/fluxcd/source-watcher/api/v1beta1"
@@ -71,51 +72,59 @@ func main() {
 	var (
 		metricsAddr           string
 		healthAddr            string
+		eventsAddr            string
 		concurrent            int
 		httpRetry             int
 		reconciliationTimeout time.Duration
 		requeueDependency     time.Duration
-		artifactOptions       config.Options
-		aclOptions            acl.Options
-		clientOptions         client.Options
-		logOptions            logger.Options
-		leaderElectionOptions leaderelection.Options
-		rateLimiterOptions    ctrl.RateLimiterOptions
-		intervalJitterOptions jitter.IntervalOptions
-		featureGates          features.FeatureGates
+
+		// GitOps Toolkit (gotk) runtime options.
+		// https://pkg.go.dev/github.com/fluxcd/pkg/runtime
+
+		aclOptions            gotkacl.Options
+		artifactOptions       gotkconfig.Options
+		clientOptions         gotkclient.Options
+		featureGates          gotkfeatures.FeatureGates
+		intervalJitterOptions gotkjitter.IntervalOptions
+		leaderElectionOptions gotkelection.Options
+		logOptions            gotklogger.Options
+		rateLimiterOptions    gotkctrl.RateLimiterOptions
+		watchOptions          gotkctrl.WatchOptions
 	)
 
 	flag.IntVar(&concurrent, "concurrent", 10, "The number of concurrent resource reconciles.")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.")
+	flag.StringVar(&eventsAddr, "events-addr", "", "The address of the events receiver.")
 	flag.IntVar(&httpRetry, "http-retry", 9,
 		"The maximum number of retries when failing to fetch artifacts over HTTP.")
 	flag.DurationVar(&reconciliationTimeout, "reconciliation-timeout", 10*time.Minute,
 		"The maximum duration of a reconciliation.")
 	flag.DurationVar(&requeueDependency, "requeue-dependency", 5*time.Second,
 		"The interval at which failing dependencies are reevaluated.")
 
-	artifactOptions.BindFlags(flag.CommandLine)
 	aclOptions.BindFlags(flag.CommandLine)
+	artifactOptions.BindFlags(flag.CommandLine)
 	clientOptions.BindFlags(flag.CommandLine)
-	logOptions.BindFlags(flag.CommandLine)
+	featureGates.BindFlags(flag.CommandLine)
+	intervalJitterOptions.BindFlags(flag.CommandLine)
 	leaderElectionOptions.BindFlags(flag.CommandLine)
+	logOptions.BindFlags(flag.CommandLine)
 	rateLimiterOptions.BindFlags(flag.CommandLine)
-	intervalJitterOptions.BindFlags(flag.CommandLine)
-	featureGates.BindFlags(flag.CommandLine)
+	watchOptions.BindFlags(flag.CommandLine)
 
 	flag.Parse()
 
-	ctrlruntime.SetLogger(logger.NewLogger(logOptions))
+	ctrlruntime.SetLogger(gotklogger.NewLogger(logOptions))
 
-	digestAlgo, err := digest.AlgorithmForName(artifactOptions.ArtifactDigestAlgo)
+	digestAlgo, err := gotkdigest.AlgorithmForName(artifactOptions.ArtifactDigestAlgo)
 	if err != nil {
 		setupLog.Error(err, "unable to configure canonical digest algorithm")
 		os.Exit(1)
 	}
-	digest.Canonical = digestAlgo
+	gotkdigest.Canonical = digestAlgo
 
-	artifactStorage, err := storage.New(&artifactOptions)
+	artifactStorage, err := gotkstorage.New(&artifactOptions)
 	if err != nil {
 		setupLog.Error(err, "unable to configure artifact storage")
 		os.Exit(1)
@@ -127,13 +136,14 @@ func main() {
 		os.Exit(1)
 	}
 
-	ctx := ctrlruntime.SetupSignalHandler()
 	leaderElectionID := fmt.Sprintf("%s-%s", controllerName, "leader-election")
-	mgr, err := ctrlruntime.NewManager(ctrlruntime.GetConfigOrDie(), ctrlruntime.Options{
+
+	// Configure the manager with the GitOps Toolkit runtime options.
+	mgrConfig := ctrlruntime.Options{
 		Scheme: scheme,
 		Metrics: metricsserver.Options{
 			BindAddress:   metricsAddr,
-			ExtraHandlers: pprof.GetHandlers(),
+			ExtraHandlers: gotkpprof.GetHandlers(),
 		},
 		HealthProbeBindAddress:        healthAddr,
 		LeaderElection:                leaderElectionOptions.Enable,
@@ -149,19 +159,32 @@ func main() {
 		},
 		Client: ctrlclient.Options{
 			Cache: &ctrlclient.CacheOptions{
+				// We don't cache Secrets and ConfigMaps
+				// as it would lead to unnecessary memory consumption.
 				DisableFor: []ctrlclient.Object{&corev1.Secret{}, &corev1.ConfigMap{}},
 			},
 		},
-	})
+	}
+
+	// Limit the watch scope to the runtime namespace if specified.
+	if !watchOptions.AllNamespaces {
+		mgrConfig.Cache.DefaultNamespaces = map[string]ctrlcache.Config{
+			os.Getenv(gotkctrl.EnvRuntimeNamespace): ctrlcache.Config{},
+		}
+	}
+
+	ctx := ctrlruntime.SetupSignalHandler()
+	mgr, err := ctrlruntime.NewManager(ctrlruntime.GetConfigOrDie(), mgrConfig)
 	if err != nil {
-		setupLog.Error(err, "unable to start manager")
+		setupLog.Error(err, "unable to create manager")
 		os.Exit(1)
 	}
 
 	// Note that the liveness check will pass beyond this point, but the readiness
 	// check will continue to fail until this controller instance is elected leader.
-	probes.SetupChecks(mgr, setupLog)
+	gotkprobes.SetupChecks(mgr, setupLog)
 
+	// Register the ArtifactGenerator controller with the manager.
 	if err = (&controller.ArtifactGeneratorReconciler{
 		ControllerName:            controllerName,
 		Client:                    mgr.GetClient(),
@@ -173,7 +196,7 @@ func main() {
 		DependencyRequeueInterval: requeueDependency,
 		NoCrossNamespaceRefs:      aclOptions.NoCrossNamespaceRefs,
 	}).SetupWithManager(ctx, mgr, controller.ArtifactGeneratorReconcilerOptions{
-		RateLimiter: ctrl.GetRateLimiter(rateLimiterOptions),
+		RateLimiter: gotkctrl.GetRateLimiter(rateLimiterOptions),
 	}); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", swapi.ArtifactGeneratorKind)
 		os.Exit(1)
@@ -190,7 +213,7 @@ func main() {
 		// This will make the readiness check pass and Kubernetes will start
 		// routing traffic from kustomize-controller and helm-controller to this instance.
 		setupLog.Info("starting storage server on " + artifactOptions.StorageAddress)
-		if err := server.Start(ctx, &artifactOptions); err != nil {
+		if err := gotkserver.Start(ctx, &artifactOptions); err != nil {
 			setupLog.Error(err, "artifact server error")
 			os.Exit(1)
 		}

config/manager/deployment.yaml

@@ -48,6 +48,7 @@ spec:
                 fieldRef:
                   fieldPath: metadata.namespace
           args:
+            - --watch-all-namespaces
             - --log-level=info
             - --log-encoding=json
             - --enable-leader-election

docs/README.md

@@ -21,6 +21,7 @@ with advanced source composition and decomposition patterns.
 | `--artifact-retention-ttl`            | duration      | The duration of time that artifacts from previous reconciliations will be kept in storage before being garbage collected. (default 1m0s)                                                 |
 | `--concurrent`                        | int           | The number of concurrent reconciles per controller. (default 10)                                                                                                                         |
 | `--enable-leader-election`            | boolean       | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.                                                                    |
+| `--events-addr`                       | string        | The address of the events receiver.                                                                                                                                                      |
 | `--health-addr`                       | string        | The address the health endpoint binds to. (default ":9440")                                                                                                                              |
 | `--http-retry`                        | int           | The maximum number of retries when failing to fetch artifacts over HTTP. (default 9)                                                                                                     |
 | `--interval-jitter-percentage`        | uint8         | Percentage of jitter to apply to interval durations. A value of 10 will apply a jitter of +/-10% to the interval duration. It cannot be negative, and must be less than 100. (default 5) |
@@ -39,6 +40,7 @@ with advanced source composition and decomposition patterns.
 | `--storage-addr`                      | string        | The address the static file server binds to. (default ":9090")                                                                                                                           |
 | `--storage-adv-addr`                  | string        | The advertised address of the static file server.                                                                                                                                        |
 | `--storage-path`                      | string        | The local storage path. (default "/data")                                                                                                                                                |
+| `--watch-all-namespaces`              | boolean       | Watch for resources in all namespaces, if set to false it will only watch the runtime namespace. (default true)                                                                          |
 | `--feature-gates`                     | mapStringBool | A comma separated list of key=value pairs defining the state of experimental features.                                                                                                   |
 
 ### Feature Gates

internal/builder/builder.go

@@ -28,8 +28,8 @@ import (
 	"golang.org/x/mod/sumdb/dirhash"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	"github.com/fluxcd/pkg/apis/meta"
-	"github.com/fluxcd/pkg/artifact/storage"
+	gotkmeta "github.com/fluxcd/pkg/apis/meta"
+	gotkstorage "github.com/fluxcd/pkg/artifact/storage"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 
 	swapi "github.com/fluxcd/source-watcher/api/v1beta1"
@@ -38,11 +38,11 @@ import (
 // ArtifactBuilder is responsible for building and storing artifacts
 // based on a given specification and source files.
 type ArtifactBuilder struct {
-	Storage *storage.Storage
+	Storage *gotkstorage.Storage
 }
 
 // New creates a new ArtifactBuilder with the given storage backend.
-func New(storage *storage.Storage) *ArtifactBuilder {
+func New(storage *gotkstorage.Storage) *ArtifactBuilder {
 	return &ArtifactBuilder{
 		Storage: storage,
 	}
@@ -58,7 +58,7 @@ func (r *ArtifactBuilder) Build(ctx context.Context,
 	spec *swapi.OutputArtifact,
 	sources map[string]string,
 	namespace string,
-	workspace string) (*meta.Artifact, error) {
+	workspace string) (*gotkmeta.Artifact, error) {
 	// Create a dir to stage the artifact files.
 	stagingDir := filepath.Join(workspace, spec.Name)
 	if err := os.MkdirAll(stagingDir, 0o755); err != nil {
@@ -100,7 +100,7 @@ func (r *ArtifactBuilder) Build(ctx context.Context,
 	defer unlock()
 
 	// Create the artifact tarball from the staging dir.
-	if err := r.Storage.Archive(&artifact, stagingDir, storage.SourceIgnoreFilter(nil, nil)); err != nil {
+	if err := r.Storage.Archive(&artifact, stagingDir, gotkstorage.SourceIgnoreFilter(nil, nil)); err != nil {
 		return nil, fmt.Errorf("failed to create artifact: %w", err)
 	}