feat: support cronjob & custom job

Author: colin404

README.md

@@ -1,4 +1,3 @@
-[项目网站]: https://konglingfei.com
 [项目文档]: https://konglingfei.com/onex/
 [安装指南]: https://konglingfei.com/onex/installation/
 [开发指南]: https://konglingfei.com/onex/devel/devel.html
@@ -42,8 +41,6 @@
 
 OneX 云原生实战平台是一个 Go 语言 + 云原生实战平台，也是一个企业级的云平台，整个项目遵循最佳实践的思路去构建。
 
-OneX 项目通过提供功能完备、规范、高质量的 Go 源码来实现代码即学习的理念，通过配套的 Go 开发课程，来降低学习难度、扩展学习深度和广度。通过配套的云原生开发社群 [孔令飞的云原生实战营](https://konglingfei.com)，来提高云原生技术学习的效果。
-
 > OneX 寓意该实战项目具有功能全、扩展性高等特点。未来 OneX 会继续集成进很多跟 Go、云原生相关的功能。因为 OneX在项目设计的时候，就考虑到了未来功能和架构的扩展，所以 OneX 未来可以无缝、丝滑的扩展很多新的功能。
 
 ## Features
@@ -131,15 +128,6 @@ OneX [项目文档] 包含了 OneX 项目介绍、OneX 部署、OneX 项目测
 
 请遵守本项目的 行为准则 [code of conduct](./CODE_OF_CONDUCT.md)。
 
-## Contacts
-
-- 关注公众号【[令飞编程](https://mp.weixin.qq.com/s/a8efr1hZ6bZHOqH3ITj9yA)】，干货不错过：
-  - 回复【资料】获取全套 Go + 云原生学习资料；
-  - 回复【好友】可以添加我的微信，免费一对一技术交流指导；
-  - 回复【星球】加入 [孔令飞的云原生实战营](https://konglingfei.com)，带你进阶 Go+云原生高级开发工程师。
-
-![令飞编程](./docs/images/令飞编程.png)
-
 ## License
 
 [MIT](https://choosealicense.com/licenses/mit/)

cmd/gen-man/gen_onex_man.go

@@ -21,10 +21,9 @@ import (
 
 	apiservapp "github.com/onexstack/onex/cmd/onex-apiserver/app"
 	ctrlmgrapp "github.com/onexstack/onex/cmd/onex-controller-manager/app"
-	//fakeserverapp "github.com/onexstack/onex/cmd/onex-fakeserver/app"
+
+	// fakeserverapp "github.com/onexstack/onex/cmd/onex-fakeserver/app"
 	gwapp "github.com/onexstack/onex/cmd/onex-gateway/app"
-	minerctrlapp "github.com/onexstack/onex/cmd/onex-miner-controller/app"
-	minersetctrlapp "github.com/onexstack/onex/cmd/onex-minerset-controller/app"
 	nwapp "github.com/onexstack/onex/cmd/onex-nightwatch/app"
 	pumpapp "github.com/onexstack/onex/cmd/onex-pump/app"
 	toyblcapp "github.com/onexstack/onex/cmd/onex-toyblc/app"
@@ -113,20 +112,6 @@ func main() {
 		for _, c := range ctrlmgr.Commands() {
 			genMarkdown(c, "onex-controller-manager", outDir)
 		}
-	case "onex-minerset-controller":
-		// generate manpage for onex-minerset-controller
-		minersetctrl := minersetctrlapp.NewControllerCommand()
-		genMarkdown(minersetctrl, "", outDir)
-		for _, c := range minersetctrl.Commands() {
-			genMarkdown(c, "onex-minerset-controller", outDir)
-		}
-	case "onex-miner-controller":
-		// generate manpage for onex-miner-controller
-		minerctrl := minerctrlapp.NewControllerCommand()
-		genMarkdown(minerctrl, "", outDir)
-		for _, c := range minerctrl.Commands() {
-			genMarkdown(c, "onex-miner-controller", outDir)
-		}
 	case "onexctl":
 		// generate manpage for onexctl
 		// TODO os.Stdin should really be something like ioutil.Discard, but a Reader

cmd/gen-onex-docs/gen_onex_docs.go

@@ -15,10 +15,8 @@ import (
 
 	apiservapp "github.com/onexstack/onex/cmd/onex-apiserver/app"
 	ctrlmgrapp "github.com/onexstack/onex/cmd/onex-controller-manager/app"
-	//fakeserverapp "github.com/onexstack/onex/cmd/onex-fakeserver/app"
+	// fakeserverapp "github.com/onexstack/onex/cmd/onex-fakeserver/app"
 	gwapp "github.com/onexstack/onex/cmd/onex-gateway/app"
-	minerctrlapp "github.com/onexstack/onex/cmd/onex-miner-controller/app"
-	minersetctrlapp "github.com/onexstack/onex/cmd/onex-minerset-controller/app"
 	watchapp "github.com/onexstack/onex/cmd/onex-nightwatch/app"
 	pumpapp "github.com/onexstack/onex/cmd/onex-pump/app"
 	toyblcapp "github.com/onexstack/onex/cmd/onex-toyblc/app"
@@ -78,14 +76,6 @@ func main() {
 		// generate docs for onex-controller-manager
 		ctrlmgr := ctrlmgrapp.NewControllerManagerCommand()
 		_ = doc.GenMarkdownTree(ctrlmgr, outDir)
-	case "onex-minerset-controller":
-		// generate docs for onex-minerset-controller
-		minersetctrl := minersetctrlapp.NewControllerCommand()
-		_ = doc.GenMarkdownTree(minersetctrl, outDir)
-	case "onex-miner-controller":
-		// generate docs for onex-miner-controller
-		minerctrl := minerctrlapp.NewControllerCommand()
-		_ = doc.GenMarkdownTree(minerctrl, outDir)
 	case "onexctl":
 		// generate docs for onexctl
 		onexctl := onexctlcmd.NewDefaultOneXCtlCommand()

cmd/onex-apiserver/apiserver.go

@@ -24,24 +24,27 @@ import (
 
 	"github.com/onexstack/onex/cmd/onex-apiserver/app"
 	"github.com/onexstack/onex/internal/apiserver/admission/plugin/minerset"
-	//"github.com/onexstack/onex/internal/controlplane/admission/initializer"
-	"github.com/onexstack/onex/internal/pkg/config/minerprofile"
-	//"github.com/onexstack/onex/internal/pkg/options"
+
 	"github.com/onexstack/onex/internal/apiserver/admission/initializer"
 	appsrest "github.com/onexstack/onex/internal/apiserver/registry/apps/rest"
-	"github.com/onexstack/onex/pkg/apis/apps/v1beta1"
+	batchrest "github.com/onexstack/onex/internal/apiserver/registry/batch/rest"
+	"github.com/onexstack/onex/internal/pkg/config/minerprofile"
+	appsv1beta1 "github.com/onexstack/onex/pkg/apis/apps/v1beta1"
+	batchv1beta1 "github.com/onexstack/onex/pkg/apis/batch/v1beta1"
 	"github.com/onexstack/onex/pkg/generated/clientset/versioned"
 	"github.com/onexstack/onex/pkg/generated/informers"
 	generatedopenapi "github.com/onexstack/onex/pkg/generated/openapi"
 )
 
 func main() {
+	var informerFactory informers.SharedInformerFactory
+
 	// Please note that the following WithOptions are all required.
 	command := app.NewAPIServerCommand(
 		// Add custom etcd options.
-		app.WithEtcdOptions("/registry/onex.io", v1beta1.SchemeGroupVersion),
+		app.WithEtcdOptions("/registry/onex.io", appsv1beta1.SchemeGroupVersion, batchv1beta1.SchemeGroupVersion),
 		// Add custom resource storage.
-		app.WithRESTStorageProviders(appsrest.RESTStorageProvider{}),
+		app.WithRESTStorageProviders(appsrest.RESTStorageProvider{}, batchrest.RESTStorageProvider{}),
 		// Add custom dns address.
 		app.WithAlternateDNS("onex.io"),
 		// Add custom admission plugins.
@@ -53,11 +56,19 @@ func main() {
 			if err != nil {
 				return nil, err
 			}
-			informerFactory := informers.NewSharedInformerFactory(client, c.LoopbackClientConfig.Timeout)
+			informerFactory = informers.NewSharedInformerFactory(client, c.LoopbackClientConfig.Timeout)
 			// NOTICE: As we create a shared informer, we need to start it later.
-			app.WithSharedInformerFactory(informerFactory)
+			// We can usually start it by adding a PostStartHook.
 			return []admission.PluginInitializer{initializer.New(informerFactory, client)}, nil
 		}),
+		app.WithPostStartHook(
+			"start-external-informers",
+			func(ctx genericapiserver.PostStartHookContext) error {
+				if informerFactory != nil {
+					informerFactory.Start(ctx.Done())
+				}
+				return nil
+			}),
 		app.WithPostStartHook(
 			"initialize-instance-config-client",
 			func(ctx genericapiserver.PostStartHookContext) error {

cmd/onex-apiserver/app/config.go

@@ -68,17 +68,17 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) {
 	c.KubeAPIs = controlPlane
 
 	apiExtensions, err := apiserver.CreateAPIExtensionsConfig(
-		controlPlane.GenericConfig.Config,
-		controlPlane.ExtraConfig.InternalVersionedInformers,
+		controlPlane.Generic.Config,
+		controlPlane.Extra.InternalVersionedInformers,
 		nil,
 		opts.CompletedOptions,
 		3,
 		serviceResolver,
 		webhook.NewDefaultAuthenticationInfoResolverWrapper(
-			controlPlane.ExtraConfig.ProxyTransport,
-			controlPlane.GenericConfig.EgressSelector,
-			controlPlane.GenericConfig.LoopbackClientConfig,
-			controlPlane.GenericConfig.TracerProvider,
+			controlPlane.Extra.ProxyTransport,
+			controlPlane.Generic.EgressSelector,
+			controlPlane.Generic.LoopbackClientConfig,
+			controlPlane.Generic.TracerProvider,
 		),
 	)
 	if err != nil {
@@ -87,12 +87,12 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) {
 	c.ApiExtensions = apiExtensions
 
 	aggregator, err := createAggregatorConfig(
-		controlPlane.GenericConfig.Config,
+		controlPlane.Generic.Config,
 		opts.CompletedOptions,
-		controlPlane.ExtraConfig.InternalVersionedInformers,
+		controlPlane.Extra.InternalVersionedInformers,
 		serviceResolver,
-		controlPlane.ExtraConfig.ProxyTransport,
-		controlPlane.ExtraConfig.PeerProxy,
+		controlPlane.Extra.ProxyTransport,
+		controlPlane.Extra.PeerProxy,
 		nil,
 	)
 	if err != nil {

cmd/onex-apiserver/app/options/options.go

@@ -14,7 +14,6 @@ import (
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/kube-openapi/pkg/common"
 
-	"github.com/onexstack/onex/internal/controlplane"
 	controlplaneoptions "github.com/onexstack/onex/internal/controlplane/apiserver/options"
 	"github.com/onexstack/onex/pkg/apiserver/storage"
 )
@@ -37,7 +36,6 @@ type Extra struct {
 
 	// For external resources
 	ExternalRESTStorageProviders []storage.RESTStorageProvider
-	ExternalVersionedInformers   controlplane.ExternalSharedInformerFactory
 	ExternalPostStartHooks       map[string]genericapiserver.PostStartHookFunc
 	GetOpenAPIDefinitions        common.GetOpenAPIDefinitions
 }

cmd/onex-apiserver/app/server.go

@@ -28,8 +28,10 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/admission"
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
+	"k8s.io/apiserver/pkg/reconcilers"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
+	serverstorage "k8s.io/apiserver/pkg/server/storage"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/notfoundhandler"
@@ -46,10 +48,12 @@ import (
 	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
 	"k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/features"
 
 	"github.com/onexstack/onex/cmd/onex-apiserver/app/options"
 	"github.com/onexstack/onex/internal/controlplane"
+	"github.com/onexstack/onex/internal/controlplane/apiserver"
 	controlplaneapiserver "github.com/onexstack/onex/internal/controlplane/apiserver"
 	"github.com/onexstack/onex/pkg/apiserver/storage"
 	"github.com/onexstack/onexstack/pkg/version"
@@ -59,8 +63,10 @@ func init() {
 	utilruntime.Must(logsapi.AddFeatureGates(utilfeature.DefaultMutableFeatureGate))
 }
 
-type Option func(*options.ServerRunOptions)
-type RegisterFunc func(plugins *admission.Plugins)
+type (
+	Option       func(*options.ServerRunOptions)
+	RegisterFunc func(plugins *admission.Plugins)
+)
 
 // WithLegacyCode returns an Option that sets the external group versions in the ServerRunOptions.
 func WithEtcdOptions(prefix string, versions ...schema.GroupVersion) Option {
@@ -112,13 +118,6 @@ func WithPostStartHook(name string, hook genericapiserver.PostStartHookFunc) Opt
 	}
 }
 
-// WithSharedInformerFactory returns an Option function that sets the external SharedInformerFactory in the ServerRunOptions.
-func WithSharedInformerFactory(informers controlplane.ExternalSharedInformerFactory) Option {
-	return func(s *options.ServerRunOptions) {
-		s.ExternalVersionedInformers = informers
-	}
-}
-
 // WithRESTStorageProviders returns an Option function that sets the external REST storage providers in the ServerRunOptions.
 func WithRESTStorageProviders(providers ...storage.RESTStorageProvider) Option {
 	return func(s *options.ServerRunOptions) {
@@ -238,7 +237,7 @@ func Run(ctx context.Context, opts options.CompletedOptions) error {
 
 // CreateServerChain creates the apiservers connected via delegation.
 func CreateServerChain(config CompletedConfig) (*aggregatorapiserver.APIAggregator, error) {
-	notFoundHandler := notfoundhandler.New(config.KubeAPIs.GenericConfig.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey)
+	notFoundHandler := notfoundhandler.New(config.KubeAPIs.Generic.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey)
 	apiExtensionsServer, err := config.ApiExtensions.New(genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler))
 	if err != nil {
 		return nil, err
@@ -292,43 +291,41 @@ func CreateOneXAPIServerConfig(opts options.CompletedOptions) (
 	opts.Metrics.Apply()
 
 	config := &controlplane.Config{
-		GenericConfig: genericConfig,
-		ExtraConfig: controlplane.ExtraConfig{
+		Generic: genericConfig,
+		Extra: controlplane.Extra{
 			APIResourceConfigSource: storageFactory.APIResourceConfigSource,
 			StorageFactory:          storageFactory,
 			EventTTL:                opts.EventTTL,
 			EnableLogsSupport:       opts.EnableLogsHandler,
 			ProxyTransport:          proxyTransport,
-			//ExternalGroupResources: opts.ExternalGroupResources,
+			// ExternalGroupResources: opts.ExternalGroupResources,
 			ExternalRESTStorageProviders: opts.ExternalRESTStorageProviders,
 			MasterCount:                  opts.MasterCount,
-			//VersionedInformers:           opts.SharedInformerFactory,
+			// VersionedInformers:           opts.SharedInformerFactory,
 			// Here we will use the config file of "onex" to create a client-go informers.
-			//KubeVersionedInformers:     kubeSharedInformers,
+			// KubeVersionedInformers:     kubeSharedInformers,
 			InternalVersionedInformers: opts.InternalVersionedInformers,
-			ExternalVersionedInformers: opts.ExternalVersionedInformers,
 			ExternalPostStartHooks:     opts.ExternalPostStartHooks,
 		},
 	}
 
 	if utilfeature.DefaultFeatureGate.Enabled(features.UnknownVersionInteroperabilityProxy) {
-		config.ExtraConfig.PeerEndpointLeaseReconciler, err = controlplaneapiserver.CreatePeerEndpointLeaseReconciler(genericConfig.Config, storageFactory)
+		var err error
+		config.PeerEndpointLeaseReconciler, err = CreatePeerEndpointLeaseReconciler(*genericConfig, storageFactory)
 		if err != nil {
 			return nil, nil, err
 		}
-		// build peer proxy config only if peer ca file exists
 		if opts.PeerCAFile != "" {
-			config.ExtraConfig.PeerProxy, err = controlplaneapiserver.BuildPeerProxy(
-				kubeSharedInformers,
-				genericConfig.StorageVersionManager,
+			leaseInformer := kubeSharedInformers.Coordination().V1().Leases()
+			config.PeerProxy, err = apiserver.BuildPeerProxy(
+				leaseInformer,
+				genericConfig.LoopbackClientConfig,
 				opts.ProxyClientCertFile,
-				opts.ProxyClientKeyFile,
-				opts.PeerCAFile,
+				opts.ProxyClientKeyFile, opts.PeerCAFile,
 				opts.PeerAdvertiseAddress,
 				genericConfig.APIServerID,
-				config.ExtraConfig.PeerEndpointLeaseReconciler,
-				config.GenericConfig.Serializer,
-			)
+				config.Extra.PeerEndpointLeaseReconciler,
+				config.Generic.Serializer)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -385,3 +382,15 @@ func buildServiceResolver(enabledAggregatorRouting bool, hostname string, inform
 	}
 	return serviceResolver
 }
+
+// CreatePeerEndpointLeaseReconciler creates a apiserver endpoint lease reconciliation loop
+// The peer endpoint leases are used to find network locations of apiservers for peer proxy
+func CreatePeerEndpointLeaseReconciler(c genericapiserver.RecommendedConfig, storageFactory serverstorage.StorageFactory) (reconcilers.PeerEndpointLeaseReconciler, error) {
+	ttl := apiserver.DefaultPeerEndpointReconcilerTTL
+	config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo"), &api.Endpoints{})
+	if err != nil {
+		return nil, fmt.Errorf("error creating storage factory config: %w", err)
+	}
+	reconciler, err := reconcilers.NewPeerEndpointLeaseReconciler(config, "/peerserverleases/", ttl)
+	return reconciler, err
+}

cmd/onex-apiserver/import_known_versions.go

@@ -10,4 +10,5 @@ package main
 import (
 	// These imports are the API groups the API server will support.
 	_ "github.com/onexstack/onex/pkg/apis/apps/install"
+	_ "github.com/onexstack/onex/pkg/apis/batch/install"
 )

cmd/onex-blockchain-controller/README.md

@@ -0,0 +1,12 @@
+# onex-blockchain-controller
+
+onex-blockchain-controller 用来实现私有链业务，私有链业务囊括向下承载 2 大技术板块：
+- Kubernetes
+- Web3
+
+onex-blockchain-controller 包括的控制器如下：
+- 
+
+onex-blockchain-controller 的开发方式，是 controller-runtime 方式
+
+将所有跟区块链相关的功能聚合在一个 controller 核心优点是：可以复用公共部分的源码

cmd/onex-miner-controller/app/config/config.go renamed to cmd/onex-blockchain-controller/app/config/config.go

@@ -11,12 +11,12 @@ import (
 	restclient "k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
 
-	"github.com/onexstack/onex/internal/controller/miner/apis/config"
+	"github.com/onexstack/onex/internal/controller/blockchain/apis/config"
 )
 
 // Config is the main context object for the controller.
 type Config struct {
-	ComponentConfig *config.MinerControllerConfiguration
+	ComponentConfig *config.BlockchainControllerConfiguration
 
 	// the rest config for the master
 	Kubeconfig *restclient.Config
@@ -26,13 +26,3 @@ type Config struct {
 
 	ProviderCluster cluster.Cluster
 }
-
-// CompletedConfig same as Config, just to swap private object.
-type CompletedConfig struct {
-	*Config
-}
-
-// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
-func (c *Config) Complete() *CompletedConfig {
-	return &CompletedConfig{c}
-}