Initial version of the Datum API server and controller manager, based on the generic API server capabilities of the Kubernetes project. (#33)

This work is largely based on the work done in
kubernetes/kubernetes#126260 as part of
kubernetes/enhancements#4080.

## API Server
One key difference in the API server is overriding the building of the
server handler chain in order to insert custom middleware into the
request path to enable a better user experience for end users listing
projects, and support authorization by injecting organization metadata
into the request.

Very few native Kubernetes types have been included in the API server -
Datum API types will be defined via CRDs coming from our own as well as
third party operators.

## Controller Manager

Only a handful of controllers have been registered to support garbage
collection, namespace lifecycle needs, and quota enforcement.

## Other

Future work will include investigating the [integration test
framework](https://github.com/kubernetes/kubernetes/tree/master/test/integration)
available in the Kubernetes project to determine if we can leverage it
here.

Author: scotwells

LICENSE

@@ -0,0 +1,26 @@
+FROM golang:1.23 AS build-stage
+
+WORKDIR /workspace
+
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+
+COPY cmd/datum-apiserver/ cmd/datum-apiserver/
+COPY pkg pkg/
+
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+  --mount=type=cache,target="/root/.cache/go-build" \
+  CGO_ENABLED=0 GOOS=linux \
+  go build \
+  -ldflags="-X 'k8s.io/component-base/version/verflag.programName=Datum'" \
+  -o /datum-apiserver cmd/datum-apiserver/main.go
+
+FROM gcr.io/distroless/base-debian11
+ARG cmd
+
+WORKDIR /
+
+COPY --from=build-stage /datum-apiserver /datum-apiserver
+
+ENTRYPOINT ["/datum-apiserver"]

cmd/datum-apiserver/Dockerfile

@@ -0,0 +1,27 @@
+package app
+
+import (
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
+	mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
+	validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
+	"k8s.io/kubernetes/pkg/kubeapiserver/options"
+)
+
+// DefaultOffAdmissionPlugins get admission plugins off by default for kube-apiserver.
+func DefaultOffAdmissionPlugins() sets.Set[string] {
+	defaultOnPlugins := sets.New[string](
+		lifecycle.PluginName, // NamespaceLifecycle
+		// defaulttolerationseconds.PluginName, // DefaultTolerationSeconds
+		mutatingwebhook.PluginName,   // MutatingAdmissionWebhook
+		validatingwebhook.PluginName, // ValidatingAdmissionWebhook
+	// resourcequota.PluginName,            // ResourceQuota
+	// certapproval.PluginName,              // CertificateApproval
+	// certsigning.PluginName,               // CertificateSigning
+	// ctbattest.PluginName,                 // ClusterTrustBundleAttest
+	// certsubjectrestriction.PluginName,    // CertificateSubjectRestriction
+	// validatingadmissionpolicy.PluginName, // ValidatingAdmissionPolicy, only active when feature gate ValidatingAdmissionPolicy is enabled
+	)
+
+	return sets.New(options.AllOrderedPlugins...).Difference(defaultOnPlugins)
+}

cmd/datum-apiserver/app/admission.go

@@ -0,0 +1,240 @@
+package app
+
+import (
+	"net/http"
+
+	corev1 "k8s.io/api/core/v1"
+	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/endpoints/filterlatency"
+	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
+	genericfeatures "k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/server"
+	genericfilters "k8s.io/apiserver/pkg/server/filters"
+	"k8s.io/apiserver/pkg/server/routine"
+	flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
+	"k8s.io/apiserver/pkg/util/webhook"
+	"k8s.io/client-go/discovery"
+	utilversion "k8s.io/component-base/version"
+	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
+	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/pkg/controlplane"
+	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
+	"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
+	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
+	admissionregistrationrest "k8s.io/kubernetes/pkg/registry/admissionregistration/rest"
+	apiserverinternalrest "k8s.io/kubernetes/pkg/registry/apiserverinternal/rest"
+	authenticationrest "k8s.io/kubernetes/pkg/registry/authentication/rest"
+	authorizationrest "k8s.io/kubernetes/pkg/registry/authorization/rest"
+	coordinationrest "k8s.io/kubernetes/pkg/registry/coordination/rest"
+	discoveryrest "k8s.io/kubernetes/pkg/registry/discovery/rest"
+	eventsrest "k8s.io/kubernetes/pkg/registry/events/rest"
+	flowcontrolrest "k8s.io/kubernetes/pkg/registry/flowcontrol/rest"
+	rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
+	svmrest "k8s.io/kubernetes/pkg/registry/storagemigration/rest"
+
+	datumfilters "go.datumapis.com/datum/pkg/server/filters"
+)
+
+type Config struct {
+	Options options.CompletedOptions
+
+	Aggregator    *aggregatorapiserver.Config
+	ControlPlane  *controlplaneapiserver.Config
+	APIExtensions *apiextensionsapiserver.Config
+
+	ExtraConfig
+}
+
+type ExtraConfig struct {
+}
+
+type completedConfig struct {
+	Options options.CompletedOptions
+
+	Aggregator    aggregatorapiserver.CompletedConfig
+	ControlPlane  controlplaneapiserver.CompletedConfig
+	APIExtensions apiextensionsapiserver.CompletedConfig
+
+	ExtraConfig
+}
+
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+func (c *CompletedConfig) GenericStorageProviders(discovery discovery.DiscoveryInterface) ([]controlplaneapiserver.RESTStorageProvider, error) {
+	return []controlplaneapiserver.RESTStorageProvider{
+		c.ControlPlane.NewCoreGenericConfig(),
+		apiserverinternalrest.StorageProvider{},
+		authenticationrest.RESTStorageProvider{Authenticator: c.ControlPlane.Generic.Authentication.Authenticator, APIAudiences: c.ControlPlane.Generic.Authentication.APIAudiences},
+		authorizationrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer, RuleResolver: c.ControlPlane.Generic.RuleResolver},
+		coordinationrest.RESTStorageProvider{},
+		rbacrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer},
+		svmrest.RESTStorageProvider{},
+		flowcontrolrest.RESTStorageProvider{InformerFactory: c.ControlPlane.Generic.SharedInformerFactory},
+		admissionregistrationrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer, DiscoveryClient: discovery},
+		eventsrest.RESTStorageProvider{TTL: c.ControlPlane.EventTTL},
+		discoveryrest.StorageProvider{},
+	}, nil
+}
+
+func (c *Config) Complete() (CompletedConfig, error) {
+	return CompletedConfig{&completedConfig{
+		Options: c.Options,
+
+		Aggregator:    c.Aggregator.Complete(),
+		ControlPlane:  c.ControlPlane.Complete(),
+		APIExtensions: c.APIExtensions.Complete(),
+
+		ExtraConfig: c.ExtraConfig,
+	}}, nil
+}
+
+func NewConfig(opts options.CompletedOptions) (*Config, error) {
+	c := &Config{
+		Options: opts,
+	}
+
+	apiResourceConfigSource := controlplane.DefaultAPIResourceConfigSource()
+	apiResourceConfigSource.DisableResources(corev1.SchemeGroupVersion.WithResource("serviceaccounts"))
+
+	genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig(
+		opts,
+		[]*runtime.Scheme{legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme},
+		apiResourceConfigSource,
+		generatedopenapi.GetOpenAPIDefinitions,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	genericConfig.BuildHandlerChainFunc = DefaultBuildHandlerChain
+
+	serviceResolver := webhook.NewDefaultServiceResolver()
+	kubeAPIs, pluginInitializer, err := controlplaneapiserver.CreateConfig(opts, genericConfig, versionedInformers, storageFactory, serviceResolver, nil)
+	if err != nil {
+		return nil, err
+	}
+	c.ControlPlane = kubeAPIs
+	c.ControlPlane.Generic.EffectiveVersion = utilversion.DefaultKubeEffectiveVersion()
+
+	authInfoResolver := webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIs.ProxyTransport, kubeAPIs.Generic.EgressSelector, kubeAPIs.Generic.LoopbackClientConfig, kubeAPIs.Generic.TracerProvider)
+	apiExtensions, err := controlplaneapiserver.CreateAPIExtensionsConfig(*kubeAPIs.Generic, kubeAPIs.VersionedInformers, pluginInitializer, opts, 3, serviceResolver, authInfoResolver)
+	if err != nil {
+		return nil, err
+	}
+	c.APIExtensions = apiExtensions
+
+	// TODO(jreese) create an admission plugin that will prohibit the creation of
+	// a Secret with a type of `kubernetes.io/service-account-token`
+	c.APIExtensions.GenericConfig.DisabledPostStartHooks.Insert("start-legacy-token-tracking-controller")
+
+	aggregator, err := controlplaneapiserver.CreateAggregatorConfig(*kubeAPIs.Generic, opts, kubeAPIs.VersionedInformers, serviceResolver, kubeAPIs.ProxyTransport, kubeAPIs.Extra.PeerProxy, pluginInitializer)
+	if err != nil {
+		return nil, err
+	}
+	c.Aggregator = aggregator
+	c.Aggregator.ExtraConfig.DisableRemoteAvailableConditionController = true
+	// TODO(jreese) better version handling
+	c.Aggregator.GenericConfig.EffectiveVersion = utilversion.DefaultKubeEffectiveVersion()
+
+	return c, nil
+}
+
+// Taken from https://github.com/kubernetes/kubernetes/blob/50fc400f178d2078d0ca46aee955ee26375fc437/staging/src/k8s.io/apiserver/pkg/server/config.go#L1004
+//
+// Modified to inject the following filters at the necessary locations:
+//   - datumfilters.OrganizationContextAuthorizationDecorator
+//   - datumfilters.ProjectListOrganizationConstraintDecorator
+//   - datumfilters.OrganizationContextHandler
+//
+// This is done to improve the UX that customers will experience while
+// interacting with the Datum API server.
+//
+// Some handlers have not been added as a result of not having access to
+// lifecycleSignals in server.Config. TODO(jreese) need to look into this more
+func DefaultBuildHandlerChain(apiHandler http.Handler, c *server.Config) http.Handler {
+	handler := apiHandler
+
+	handler = filterlatency.TrackCompleted(handler)
+	handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
+	handler = datumfilters.OrganizationContextAuthorizationDecorator(handler)
+	handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authorization")
+
+	if c.FlowControl != nil {
+		workEstimatorCfg := flowcontrolrequest.DefaultWorkEstimatorConfig()
+		requestWorkEstimator := flowcontrolrequest.NewWorkEstimator(
+			c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg, c.FlowControl.GetMaxSeats)
+		handler = filterlatency.TrackCompleted(handler)
+		handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator, c.RequestTimeout/4)
+		handler = filterlatency.TrackStarted(handler, c.TracerProvider, "priorityandfairness")
+	} else {
+		handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
+	}
+
+	handler = filterlatency.TrackCompleted(handler)
+	handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
+	handler = filterlatency.TrackStarted(handler, c.TracerProvider, "impersonation")
+
+	handler = filterlatency.TrackCompleted(handler)
+	handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator, c.LongRunningFunc)
+	handler = filterlatency.TrackStarted(handler, c.TracerProvider, "audit")
+
+	failedHandler := genericapifilters.Unauthorized(c.Serializer)
+	failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyRuleEvaluator)
+
+	failedHandler = filterlatency.TrackCompleted(failedHandler)
+	handler = filterlatency.TrackCompleted(handler)
+	handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences, c.Authentication.RequestHeaderConfig)
+	handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authentication")
+
+	handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")
+
+	// WithWarningRecorder must be wrapped by the timeout handler
+	// to make the addition of warning headers threadsafe
+	handler = genericapifilters.WithWarningRecorder(handler)
+
+	// WithTimeoutForNonLongRunningRequests will call the rest of the request handling in a go-routine with the
+	// context with deadline. The go-routine can keep running, while the timeout logic will return a timeout to the client.
+	handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc)
+
+	handler = genericapifilters.WithRequestDeadline(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator,
+		c.LongRunningFunc, c.Serializer, c.RequestTimeout)
+	handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.NonLongRunningRequestWaitGroup)
+	// if c.ShutdownWatchTerminationGracePeriod > 0 {
+	// 	handler = genericfilters.WithWatchTerminationDuringShutdown(handler, c.lifecycleSignals, c.WatchRequestWaitGroup)
+	// }
+	if c.SecureServing != nil && !c.SecureServing.DisableHTTP2 && c.GoawayChance > 0 {
+		handler = genericfilters.WithProbabilisticGoaway(handler, c.GoawayChance)
+	}
+	handler = genericapifilters.WithCacheControl(handler)
+	handler = genericfilters.WithHSTS(handler, c.HSTSDirectives)
+	// if c.ShutdownSendRetryAfter {
+	// 	handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled())
+	// }
+	handler = genericfilters.WithHTTPLogging(handler)
+	if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) {
+		handler = genericapifilters.WithTracing(handler, c.TracerProvider)
+	}
+	handler = genericapifilters.WithLatencyTrackers(handler)
+	// WithRoutine will execute future handlers in a separate goroutine and serving
+	// handler in current goroutine to minimize the stack memory usage. It must be
+	// after WithPanicRecover() to be protected from panics.
+	if c.FeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
+		handler = routine.WithRoutine(handler, c.LongRunningFunc)
+	}
+
+	handler = datumfilters.OrganizationProjectListConstraintDecorator(handler)
+	handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
+	handler = genericapifilters.WithRequestReceivedTimestamp(handler)
+	// handler = genericapifilters.WithMuxAndDiscoveryComplete(handler, c.lifecycleSignals.MuxAndDiscoveryComplete.Signaled())
+	handler = genericfilters.WithPanicRecovery(handler, c.RequestInfoResolver)
+	handler = genericapifilters.WithAuditInit(handler)
+
+	handler = datumfilters.OrganizationContextHandler(handler, c.Serializer)
+
+	return handler
+}