feat: add onex-apiserver code chassis

Author: colin404

OWNERS

@@ -1,4 +1,4 @@
-# See the OWNERS docs at https://go.k8s.io/owners
+KubeVersionedInformers# See the OWNERS docs at https://go.k8s.io/owners
 
 reviewers:
   - colin404

cmd/onex-apiserver/apiserver.go

@@ -9,19 +9,76 @@
 package main
 
 import (
+	"context"
 	"os"
 
 	_ "go.uber.org/automaxprocs/maxprocs"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apiserver/pkg/admission"
+	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/component-base/cli"
 	_ "k8s.io/component-base/logs/json/register"          // for JSON log format registration
 	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
 	_ "k8s.io/component-base/metrics/prometheus/version"  // for version metric registration
+	"k8s.io/klog/v2"
 
 	"github.com/superproj/onex/cmd/onex-apiserver/app"
+	"github.com/superproj/onex/internal/apiserver/admission/plugin/minerset"
+	"github.com/superproj/onex/internal/controlplane/admission/initializer"
+	"github.com/superproj/onex/internal/pkg/config/minerprofile"
+	//"github.com/superproj/onex/internal/pkg/options"
+	appsrest "github.com/superproj/onex/internal/apiserver/registry/apps/rest"
+	"github.com/superproj/onex/pkg/apis/apps/v1beta1"
+	"github.com/superproj/onex/pkg/generated/clientset/versioned"
+	"github.com/superproj/onex/pkg/generated/informers"
 )
 
 func main() {
-	command := app.NewAPIServerCommand()
+	// Please note that the following WithOptions are all required.
+	command := app.NewAPIServerCommand(
+		// Add custom etcd options.
+		app.WithEtcdOptions("/registry/onex.io", v1beta1.SchemeGroupVersion),
+		// Add custom resource storage.
+		app.WithRESTStorageProviders(appsrest.RESTStorageProvider{}),
+		// Add custom dns address.
+		app.WithAlternateDNS("onex.io"),
+		// Add custom admission plugins.
+		app.WithAdmissionPlugin(minerset.PluginName, minerset.Register),
+		// Add custom admission plugins initializer.
+		app.WithAdmissionInitializers(func(c *genericapiserver.RecommendedConfig) ([]admission.PluginInitializer, error) {
+			client, err := versioned.NewForConfig(c.LoopbackClientConfig)
+			if err != nil {
+				return nil, err
+			}
+			informerFactory := informers.NewSharedInformerFactory(client, c.LoopbackClientConfig.Timeout)
+			// NOTICE: As we create a shared informer, we need to start it later.
+			app.WithSharedInformerFactory(informerFactory)
+			return []admission.PluginInitializer{initializer.New(informerFactory, client)}, nil
+		}),
+		app.WithPostStartHook(
+			"initialize-instance-config-client",
+			func(ctx genericapiserver.PostStartHookContext) error {
+				client, err := versioned.NewForConfig(ctx.LoopbackClientConfig)
+				if err != nil {
+					return err
+				}
+
+				if err := minerprofile.Init(context.Background(), client); err != nil {
+					// When returning 'NotFound' error, we should not report an error, otherwise we can not
+					// create 'MinerTypesConfigMapName' configmap via onex-apiserver
+					if apierrors.IsNotFound(err) {
+						return nil
+					}
+
+					klog.ErrorS(err, "Failed to init miner type cache")
+					return err
+				}
+
+				return nil
+			},
+		),
+	)
+
 	code := cli.Run(command)
 	os.Exit(code)
 }

cmd/onex-apiserver/app/import_known_versions.go

@@ -0,0 +1,13 @@
+// Copyright 2022 Lingfei Kong <[email protected]>. All rights reserved.
+// Use of this source code is governed by a MIT style
+// license that can be found in the LICENSE file. The original repo for
+// this file is https://github.com/superproj/onex.
+//
+
+package app
+
+// UPDATEME: When add new api group.
+import (
+	// These imports are the API groups the API server will support.
+	_ "github.com/superproj/onex/pkg/apis/apps/install"
+)

cmd/onex-apiserver/app/options/options.go

@@ -8,10 +8,14 @@
 package options
 
 import (
-	cliflag "k8s.io/component-base/cli/flag"
 	"net"
 
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	cliflag "k8s.io/component-base/cli/flag"
+
+	"github.com/superproj/onex/internal/controlplane"
 	controlplaneoptions "github.com/superproj/onex/internal/controlplane/apiserver/options"
+	"github.com/superproj/onex/internal/controlplane/storage"
 )
 
 const defaultEtcdPathPrefix = "/registry/onex.io"
@@ -29,14 +33,20 @@ type Extra struct {
 	// OnexletConfig onexletclient.OnexletClientConfig
 	APIServerServiceIP     net.IP
 	EndpointReconcilerType string
+
+	// For external resources
+	ExternalRESTStorageProviders []storage.RESTStorageProvider
+	ExternalVersionedInformers   controlplane.ExternalSharedInformerFactory
+	ExternalPostStartHooks       map[string]genericapiserver.PostStartHookFunc
 }
 
 // NewServerRunOptions returns a new ServerRunOptions.
 func NewServerRunOptions() *ServerRunOptions {
 	o := &ServerRunOptions{
 		Options: controlplaneoptions.NewOptions(),
 		Extra: Extra{
-			MasterCount: 1,
+			MasterCount:            1,
+			ExternalPostStartHooks: make(map[string]genericapiserver.PostStartHookFunc),
 		},
 	}
 

cmd/onex-apiserver/app/server.go

@@ -21,11 +21,15 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/admission"
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	genericapiserver "k8s.io/apiserver/pkg/server"
+	genericoptions "k8s.io/apiserver/pkg/server/options"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/notfoundhandler"
 	"k8s.io/apiserver/pkg/util/webhook"
@@ -45,6 +49,7 @@ import (
 	"github.com/superproj/onex/cmd/onex-apiserver/app/options"
 	"github.com/superproj/onex/internal/controlplane"
 	controlplaneapiserver "github.com/superproj/onex/internal/controlplane/apiserver"
+	"github.com/superproj/onex/internal/controlplane/storage"
 	generatedopenapi "github.com/superproj/onex/pkg/generated/openapi"
 	"github.com/superproj/onex/pkg/version"
 )
@@ -55,9 +60,80 @@ func init() {
 	utilruntime.Must(logsapi.AddFeatureGates(utilfeature.DefaultMutableFeatureGate))
 }
 
+type Option func(*options.ServerRunOptions)
+type RegisterFunc func(plugins *admission.Plugins)
+
+// WithLegacyCode returns an Option that sets the external group versions in the ServerRunOptions.
+func WithEtcdOptions(prefix string, versions ...schema.GroupVersion) Option {
+	return func(s *options.ServerRunOptions) {
+		codec := legacyscheme.Codecs.LegacyCodec(versions...)
+		s.RecommendedOptions.Etcd = genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(prefix, codec))
+		// Note: DefaultStorageMediaType needs to be reset here.
+		s.RecommendedOptions.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
+		/*
+			s.RecommendedOptions.Etcd.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(
+				v1beta1.SchemeGroupVersion,
+				schema.GroupKind{Group: v1beta1.GroupName},
+			)
+		*/
+		for _, version := range versions {
+			controlplane.AddStableAPIGroupVersionsEnabledByDefault(version)
+		}
+	}
+}
+
+// WithAdmissionPlugin returns an Option function that adds an admission plugin to the recommended plugin order list
+// and registers the plugin using the provided RegisterFunc in the ServerRunOptions.
+func WithAdmissionPlugin(name string, registerFunc RegisterFunc) Option {
+	return func(s *options.ServerRunOptions) {
+		// Note: Need to add this to the RecommendedPluginOrder list.
+		s.RecommendedOptions.Admission.RecommendedPluginOrder = append(s.RecommendedOptions.Admission.RecommendedPluginOrder, name)
+		registerFunc(s.RecommendedOptions.Admission.Plugins)
+	}
+}
+
+// WithAdmissionInitializers returns an Option function that sets the external admission initializers in the ServerRunOptions.
+func WithAdmissionInitializers(initializer func(c *genericapiserver.RecommendedConfig) ([]admission.PluginInitializer, error)) Option {
+	return func(s *options.ServerRunOptions) {
+		s.RecommendedOptions.ExternalAdmissionInitializers = initializer
+	}
+}
+
+// WithPostStartHook returns an Option function that sets the external post-start hook with the given name in the ServerRunOptions.
+func WithPostStartHook(name string, hook genericapiserver.PostStartHookFunc) Option {
+	return func(s *options.ServerRunOptions) {
+		s.ExternalPostStartHooks[name] = hook
+	}
+}
+
+// WithSharedInformerFactory returns an Option function that sets the external SharedInformerFactory in the ServerRunOptions.
+func WithSharedInformerFactory(informers controlplane.ExternalSharedInformerFactory) Option {
+	return func(s *options.ServerRunOptions) {
+		s.ExternalVersionedInformers = informers
+	}
+}
+
+// WithRESTStorageProviders returns an Option function that sets the external REST storage providers in the ServerRunOptions.
+func WithRESTStorageProviders(providers ...storage.RESTStorageProvider) Option {
+	return func(s *options.ServerRunOptions) {
+		s.ExternalRESTStorageProviders = providers
+	}
+}
+
+// WithAlternateDNS returns an Option function that sets the alternate DNS configurations in the ServerRunOptions.
+func WithAlternateDNS(dns ...string) Option {
+	return func(s *options.ServerRunOptions) {
+		s.AlternateDNS = dns
+	}
+}
+
 // NewAPIServerCommand creates a *cobra.Command object with default parameters.
-func NewAPIServerCommand() *cobra.Command {
+func NewAPIServerCommand(serverRunOptions ...Option) *cobra.Command {
 	s := options.NewServerRunOptions()
+	for _, opt := range serverRunOptions {
+		opt(s)
+	}
+
 	cmd := &cobra.Command{
 		Use:   appName,
 		Short: "Launch a onex API server",
@@ -198,7 +274,7 @@ func CreateOneXAPIServerConfig(opts options.CompletedOptions) (
 ) {
 	proxyTransport := CreateProxyTransport()
 
-	genericConfig, _, kubeVersionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig(
+	genericConfig, _, kubeSharedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig(
 		opts.CompletedOptions,
 		[]*runtime.Scheme{legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme},
 		generatedopenapi.GetOpenAPIDefinitions,
@@ -217,10 +293,15 @@ func CreateOneXAPIServerConfig(opts options.CompletedOptions) (
 			EventTTL:                opts.EventTTL,
 			EnableLogsSupport:       opts.EnableLogsHandler,
 			ProxyTransport:          proxyTransport,
-			MasterCount:             opts.MasterCount,
-			VersionedInformers:      opts.SharedInformerFactory,
+			//ExternalGroupResources: opts.ExternalGroupResources,
+			ExternalRESTStorageProviders: opts.ExternalRESTStorageProviders,
+			MasterCount:                  opts.MasterCount,
+			//VersionedInformers:           opts.SharedInformerFactory,
 			// Here we will use the config file of "onex" to create a client-go informers.
-			KubeVersionedInformers: kubeVersionedInformers,
+			KubeVersionedInformers:     kubeSharedInformers,
+			InternalVersionedInformers: opts.InternalVersionedInformers,
+			ExternalVersionedInformers: opts.ExternalVersionedInformers,
+			ExternalPostStartHooks:     opts.ExternalPostStartHooks,
 		},
 	}
 
@@ -232,7 +313,7 @@ func CreateOneXAPIServerConfig(opts options.CompletedOptions) (
 		// build peer proxy config only if peer ca file exists
 		if opts.PeerCAFile != "" {
 			config.ExtraConfig.PeerProxy, err = controlplaneapiserver.BuildPeerProxy(
-				kubeVersionedInformers,
+				kubeSharedInformers,
 				genericConfig.StorageVersionManager,
 				opts.ProxyClientCertFile,
 				opts.ProxyClientKeyFile,
@@ -268,7 +349,7 @@ func CreateOneXAPIServerConfig(opts options.CompletedOptions) (
 	}
 	*/
 
-	serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, kubeVersionedInformers)
+	serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, kubeSharedInformers)
 
 	return config, serviceResolver, nil
 }

internal/apiserver/README.md

@@ -0,0 +1,5 @@
+# apiserver 目录
+
+该目录存放 onex-apiserver 业务的逻辑，包括：
+- registry: CURD 逻辑
+- admission: 动态准入控制

internal/apiserver/admission/initializer/doc.go

@@ -0,0 +1,7 @@
+// Copyright 2022 Lingfei Kong <[email protected]>. All rights reserved.
+// Use of this source code is governed by a MIT style
+// license that can be found in the LICENSE file. The original repo for
+// this file is https://github.com/superproj/onex.
+//
+
+package initializer // import "github.com/superproj/onex/internal/controlplane/admission/initializer"

internal/apiserver/admission/initializer/initializer.go

@@ -0,0 +1,47 @@
+// Copyright 2022 Lingfei Kong <[email protected]>. All rights reserved.
+// Use of this source code is governed by a MIT style
+// license that can be found in the LICENSE file. The original repo for
+// this file is https://github.com/superproj/onex.
+//
+
+package initializer
+
+import (
+	"k8s.io/apiserver/pkg/admission"
+
+	clientset "github.com/superproj/onex/pkg/generated/clientset/versioned"
+	"github.com/superproj/onex/pkg/generated/informers"
+)
+
+type pluginInitializer struct {
+	informers      informers.SharedInformerFactory
+	externalClient clientset.Interface
+	//authorizer        authorizer.Authorizer
+	//featureGates      featuregate.FeatureGate
+	stopCh <-chan struct{}
+}
+
+var _ admission.PluginInitializer = pluginInitializer{}
+
+// New creates an instance of node admission plugins initializer.
+func New(
+	informers informers.SharedInformerFactory,
+	extClientset clientset.Interface,
+) pluginInitializer {
+	return pluginInitializer{
+		informers:      informers,
+		externalClient: extClientset,
+	}
+}
+
+// Initialize checks the initialization interfaces implemented by a plugin
+// and provide the appropriate initialization data.
+func (i pluginInitializer) Initialize(plugin admission.Interface) {
+	if wants, ok := plugin.(WantsExternalInformerFactory); ok {
+		wants.SetInternalInformerFactory(i.informers)
+	}
+
+	if wants, ok := plugin.(WantsExternalClientSet); ok {
+		wants.SetExternalClientSet(i.externalClient)
+	}
+}

internal/apiserver/admission/initializer/interfaces.go

@@ -0,0 +1,26 @@
+// Copyright 2022 Lingfei Kong <[email protected]>. All rights reserved.
+// Use of this source code is governed by a MIT style
+// license that can be found in the LICENSE file. The original repo for
+// this file is https://github.com/superproj/onex.
+//
+
+package initializer
+
+import (
+	"k8s.io/apiserver/pkg/admission"
+
+	clientset "github.com/superproj/onex/pkg/generated/clientset/versioned"
+	"github.com/superproj/onex/pkg/generated/informers"
+)
+
+// WantsExternalInformerFactory defines a function which sets InformerFactory for admission plugins that need it.
+type WantsExternalInformerFactory interface {
+	admission.InitializationValidator
+	SetInternalInformerFactory(informers.SharedInformerFactory)
+}
+
+// WantsExternalClientSet defines a function which sets external ClientSet for admission plugins that need it.
+type WantsExternalClientSet interface {
+	admission.InitializationValidator
+	SetExternalClientSet(clientset.Interface)
+}

internal/apiserver/admission/plugin/doc.go

@@ -0,0 +1,7 @@
+// Copyright 2022 Lingfei Kong <[email protected]>. All rights reserved.
+// Use of this source code is governed by a MIT style
+// license that can be found in the LICENSE file. The original repo for
+// this file is https://github.com/superproj/onex.
+//
+
+package plugin // import "github.com/superproj/onex/internal/controlplane/admission/plugin"