cleanup externalSchedulerEnable (#382)

* cleanup

* update md doc

* update doc

* doc: add the second step to rebuild the scheduler

---------

Co-authored-by: Kensei Nakada <[email protected]>

Author: YamasouA

simulator/cmd/simulator/simulator.go

@@ -94,7 +94,7 @@ func startSimulator() error {
 		return xerrors.Errorf("kubeapi-server is not ready: %w", err)
 	}
 
-	dic, err := di.NewDIContainer(client, dynamicClient, restMapper, etcdclient, restCfg, cfg.InitialSchedulerCfg, cfg.ExternalImportEnabled, cfg.ResourceSyncEnabled, importClusterResourceClient, importClusterDynamicClient, cfg.ExternalSchedulerEnabled, cfg.Port, syncer.Options{})
+	dic, err := di.NewDIContainer(client, dynamicClient, restMapper, etcdclient, restCfg, cfg.InitialSchedulerCfg, cfg.ExternalImportEnabled, cfg.ResourceSyncEnabled, importClusterResourceClient, importClusterDynamicClient, cfg.Port, syncer.Options{})
 	if err != nil {
 		return xerrors.Errorf("create di container: %w", err)
 	}
@@ -110,9 +110,7 @@ func startSimulator() error {
 		}
 	}
 
-	if !cfg.ExternalSchedulerEnabled {
-		dic.SchedulerService().SetSchedulerConfig(cfg.InitialSchedulerCfg)
-	}
+	dic.SchedulerService().SetSchedulerConfig(cfg.InitialSchedulerCfg)
 
 	if cfg.ResourceSyncEnabled {
 		// Start the resource syncer to sync resources from the target cluster.

simulator/config.yaml

@@ -47,7 +47,3 @@ externalImportEnabled: false
 # You cannot make both externalImportEnabled and resourceSyncEnabled true because those features would be conflicted.
 # Note, this is still a beta feature.
 resourceSyncEnabled: false
-
-# This variable indicates whether an external scheduler
-# is used.
-externalSchedulerEnabled: false

simulator/config/config.go

@@ -43,8 +43,6 @@ type Config struct {
 	// This field should be set when ExternalImportEnabled == true or ResourceSyncEnabled == true.
 	ExternalKubeClientCfg *rest.Config
 	InitialSchedulerCfg   *configv1.KubeSchedulerConfiguration
-	// ExternalSchedulerEnabled indicates whether an external scheduler is enabled.
-	ExternalSchedulerEnabled bool
 }
 
 const (
@@ -99,18 +97,15 @@ func NewConfig() (*Config, error) {
 		return nil, xerrors.Errorf("get SchedulerCfg: %w", err)
 	}
 
-	externalSchedEnabled := getExternalSchedulerEnabled()
-
 	return &Config{
-		Port:                     port,
-		KubeAPIServerURL:         apiurl,
-		EtcdURL:                  etcdurl,
-		CorsAllowedOriginList:    corsAllowedOriginList,
-		InitialSchedulerCfg:      initialschedulerCfg,
-		ExternalImportEnabled:    externalimportenabled,
-		ExternalKubeClientCfg:    externalKubeClientCfg,
-		ExternalSchedulerEnabled: externalSchedEnabled,
-		ResourceSyncEnabled:      resourceSyncEnabled,
+		Port:                  port,
+		KubeAPIServerURL:      apiurl,
+		EtcdURL:               etcdurl,
+		CorsAllowedOriginList: corsAllowedOriginList,
+		InitialSchedulerCfg:   initialschedulerCfg,
+		ExternalImportEnabled: externalimportenabled,
+		ExternalKubeClientCfg: externalKubeClientCfg,
+		ResourceSyncEnabled:   resourceSyncEnabled,
 	}, nil
 }
 
@@ -163,17 +158,6 @@ func getKubeAPIServerURL() (string, error) {
 	return url, nil
 }
 
-// getExternalSchedulerEnabled gets ExternalSchedulerEnabled from environment variable first,
-// if empty from the config file.
-func getExternalSchedulerEnabled() bool {
-	e := os.Getenv("EXTERNAL_SCHEDULER_ENABLED")
-	b, err := strconv.ParseBool(e)
-	if e == "" || err != nil {
-		b = configYaml.ExternalSchedulerEnabled
-	}
-	return b
-}
-
 // getEtcdURL gets EtcdURL from environment variable first,
 // if empty from the config file.
 func getEtcdURL() (string, error) {

simulator/docs/custom-plugin.md

@@ -1,11 +1,7 @@
 ## Debuggable scheduler
 
-The "debuggable scheduler" is the core concept of the simulator.
-
-### What's the "debuggable scheduler"?
-
-The debuggable scheduler is the scheduler that schedules Pods like other schedulers, 
-but it exposes the scheduling results from each scheduling plugin on Pod annotation.
+The "debuggable scheduler" is the core component of the simulator,
+which is responsible for scheduling Pods like the normal scheduler, but also recording all the scheduling details to the Pod annotations.
 
 ```yaml
 kind: Pod
@@ -35,60 +31,45 @@ metadata:
     scheduler-simulator/selected-node: node-282x7
 ```
 
-The simulator runs the debuggable scheduler built from the upstream scheduler by default,
-that's why we can see the scheduling results like the above in the simulator.
-
-## Build a debuggable scheduler from your scheduler
-
-You can use the [`debuggablescheduler` package](../pkg/debuggablescheduler) to transform your scheduler into a debuggable scheduler.
+The simulator works with the debuggable scheduler built from the upstream scheduler by default,
+and the web UI just visualizes those scheduling details on the annotations.
 
-The debuggable scheduler could work anywhere, in the simulator or even in your cluster.
+## Integrate your plugins to the simulator
 
-### Change your scheduler
-
-Here, we assume you're registering your custom plugins in your scheduler like this:
-
-```go
-// your scheduler's main package
-func main() {
-	command := app.NewSchedulerCommand(
-		app.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New),
-	)
-
-	code := cli.Run(command)
-	os.Exit(code)
-}
-```
+You can integrate your plugins to the simulator, that is, to the debuggable scheduler working within the simulator, by following these steps:
 
-Then, you need to replace few lines to use the [`debuggablescheduler` package](../pkg/debuggablescheduler).
+1. Register your plugins [here](../cmd/scheduler/scheduler.go#L17) in a very similar way you do with the normal scheduler.
 
 ```go
-func main() {
-	command, cancelFn, err := debuggablescheduler.NewSchedulerCommand(
+    command, cancelFn, err := debuggablescheduler.NewSchedulerCommand(
         debuggablescheduler.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New),
         debuggablescheduler.WithPluginExtenders(noderesources.Name, extender.New), // [optional] see plugin-extender.md about PluginExtender.
     )
-    if err != nil {
-        klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err))
-        os.Exit(1)
-    }
-    code := cli.Run(command)
-    cancelFn()
-    os.Exit(code)
-}
 ```
 
-As you see, `debuggablescheduler.NewSchedulerCommand` has much similar interface as the `app.NewSchedulerCommand`.
-You can register your plugins by `debuggablescheduler.WithPlugin` option.
+2. Rebuild the scheduler and restart.
+
+```sh
+make docker_build docker_up_local
+```
 
 ### The plugin extender
 
-We have the plugin extender feature to expand the debuggable scheduler more.
-See [plugin-extender.md](./plugin-extender.md) to know more about it.
+We have the plugin extender feature to provide more debuggability from the debuggable scheduler.
+See [plugin-extender.md](./plugin-extender.md).
 
-### The example debuggable scheduler
+### Use the debuggable scheduler in your dev cluster
+
+The debuggable scheduler can work outside the simulator, that is, in your clusters too.
+which allows you to have the same debuggability with your custom scheduler in real clusters.
 
-We have the sample implementation in [./sample/debuggable-scheduler](./sample/debuggable-scheduler).
+It's just a single binary that doesn't contain anything but the debuggability features that we described so far.
+
+> [!NOTE]
+> We do not recommend using it in the production cluster because the debuggable scheduler contains the overhead.
+> Also, you may want to pay attention to the fact that the annotations would be visible to all cluster users,
+> which could leak the hints of other tenants to cluster users.
+
+### The example debuggable scheduler
 
-You can try to run it in the simulator via [external scheduler](./external-scheduler.md) feature.
-See [external-scheduler.md#the-example-external-scheduler](./external-scheduler.md#the-example-external-scheduler).
+We have the sample to show how to implement the debuggable scheduler in [./sample/debuggable-scheduler](./sample/debuggable-scheduler).

simulator/docs/debuggable-scheduler.md

@@ -1,40 +1,3 @@
-## External scheduler
-
-The external scheduler is the feature of the simulator 
-to allow you to run any of your scheduler in the simulator instead of the default one.
-
-Before diving into the description, you need to know [debuggable scheduler](./debuggable-scheduler.md).
-
-### connect your scheduler to the simulator
-
-Let's connect your scheduler to the simulator.
-
-First, you need to set `externalSchedulerEnabled: true` on [the simulator config](../config.yaml)
-so that the scheduler, running in the simulator by default, won't get started.
-
-Next, you need to connect your scheduler into the simulator's kube-apiserver via KubeSchedulerConfig:
-
-```yaml
-kind: KubeSchedulerConfiguration
-apiVersion: kubescheduler.config.k8s.io/v1
-clientConnection:
-  kubeconfig: ./path/to/kubeconfig.yaml
-```
-
-You can use this [kubeconfig.yaml](./kubeconfig.yaml) to communicate with the simulator's kube-apiserver.
-
-### The example external scheduler
-
-You can see how the external scheduler can be set up 
-with the sample debuggable scheduler implementation in [./sample/debuggable-scheduler](./sample/debuggable-scheduler).
-
-prerequisite:
-1. set `externalSchedulerEnabled: true` on [the simulator config](../config.yaml)
-2. run the simulator 
-
-```shell
-cd sample/debuggable-scheduler
-go run main.go --config scheduler.yaml
-```
-
-You'll see the simulator is working with the external scheduler.
+> [!CAUTION]
+We previously had a feature called the external scheduler, but we removed it in favor of [debuggable scheduler](../debuggable-scheduler.md).
+To maintain compatibility, the externalSchedulerEnabled setting remains for now though, will be removed in the near future.

simulator/docs/external-scheduler.md

@@ -1,13 +1,8 @@
 ## Integrate your scheduler 
 
-There are several ways to integrate your scheduler into the simulator.
+You can integrate your scheduler into the simulator.
 
-Basically you have two options on the table:
-- Integrate your scheduler plugins into the scheduler running in the simulator
-  - Check out [custom-plugin.md](./custom-plugin.md) 
-- Disable the scheduler running in the simulator and use your scheduler instead
-  - We call this feature "external scheduler"
-  - Check out [external-scheduler.md](./external-scheduler.md)
+To add your custom scheduler plugins, see [debuggable-scheduler.md](./debuggable-scheduler.md#integrate-your-plugins-to-the-simulator).
 
 Also, if you just want to use your `KubeSchedulerConfig` while using default plugins,
 you don't need to follow this page. Check out [simulator-server-config.md](./simulator-server-config.md) instead.

simulator/docs/integrate-your-scheduler.md

@@ -51,8 +51,4 @@ externalImportEnabled: false
 # You cannot make both externalImportEnabled and resourceSyncEnabled true because those features would be conflicted.
 # Note, this is still a beta feature.
 resourceSyncEnabled: false
-
-# This variable indicates whether an external scheduler
-# is used.
-externalSchedulerEnabled: false
 ```

simulator/docs/simulator-server-config.md

@@ -28,11 +28,6 @@ type Service struct {
 	// function to shutdown scheduler.
 	shutdownfn func()
 
-	// disabled represents if this Service is disabled.
-	// If externalSchedulerEnabled, it'll be true
-	// because we don't need to start scheduler, and we cannot change an external scheduler's config in that case.
-	disabled bool
-
 	clientset           clientset.Interface
 	restclientCfg       *restclient.Config
 	initialSchedulerCfg *configv1.KubeSchedulerConfiguration
@@ -52,11 +47,7 @@ type ExtenderService interface {
 var ErrServiceDisabled = errors.New("scheduler service is disabled")
 
 // NewSchedulerService starts scheduler and return *Service.
-func NewSchedulerService(client clientset.Interface, restclientCfg *restclient.Config, initialSchedulerCfg *configv1.KubeSchedulerConfiguration, externalSchedulerEnabled bool, simulatorPort int) *Service {
-	if externalSchedulerEnabled {
-		return &Service{disabled: true}
-	}
-
+func NewSchedulerService(client clientset.Interface, restclientCfg *restclient.Config, initialSchedulerCfg *configv1.KubeSchedulerConfiguration, simulatorPort int) *Service {
 	// sharedStore has some resultstores which are referenced by Registry of Plugins and Extenders.
 	sharedStore := storereflector.New()
 
@@ -131,10 +122,6 @@ func (s *Service) ShutdownScheduler() {
 }
 
 func (s *Service) GetSchedulerConfig() (*configv1.KubeSchedulerConfiguration, error) {
-	if s.disabled {
-		return nil, xerrors.Errorf("an external scheduler is enabled: %w", ErrServiceDisabled)
-	}
-
 	return s.currentSchedulerCfg, nil
 }
 

simulator/scheduler/scheduler.go

@@ -44,14 +44,13 @@ func NewDIContainer(
 	resourceSyncEnabled bool,
 	externalClient clientset.Interface,
 	externalDynamicClient dynamic.Interface,
-	externalSchedulerEnabled bool,
 	simulatorPort int,
 	syncerOptions syncer.Options,
 ) (*Container, error) {
 	c := &Container{}
 
 	// initializes each service
-	c.schedulerService = scheduler.NewSchedulerService(client, restclientCfg, initialSchedulerCfg, externalSchedulerEnabled, simulatorPort)
+	c.schedulerService = scheduler.NewSchedulerService(client, restclientCfg, initialSchedulerCfg, simulatorPort)
 	var err error
 	c.resetService, err = reset.NewResetService(etcdclient, client, c.schedulerService)
 	if err != nil {