Skip to content

Commit e8e1cc9

Browse files
author
louisgong
committed
extract PreInitRuntimeService from NewMainKubelet
1 parent 0dd4680 commit e8e1cc9

File tree

2 files changed

+102
-96
lines changed

2 files changed

+102
-96
lines changed

‎cmd/kubelet/app/server.go‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -757,6 +757,17 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate f
757757
klog.Warning(err)
758758
}
759759

760+
err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration,
761+
kubeDeps, &s.ContainerRuntimeOptions,
762+
s.ContainerRuntime,
763+
s.RuntimeCgroups,
764+
s.RemoteRuntimeEndpoint,
765+
s.RemoteImageEndpoint,
766+
s.NonMasqueradeCIDR)
767+
if err != nil {
768+
return err
769+
}
770+
760771
if err := RunKubelet(s, kubeDeps, s.RunOnce); err != nil {
761772
return err
762773
}
@@ -1062,7 +1073,6 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie
10621073
kubeDeps,
10631074
&kubeServer.ContainerRuntimeOptions,
10641075
kubeServer.ContainerRuntime,
1065-
kubeServer.RuntimeCgroups,
10661076
kubeServer.HostnameOverride,
10671077
kubeServer.NodeIP,
10681078
kubeServer.ProviderID,
@@ -1072,8 +1082,6 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie
10721082
kubeServer.RegisterNode,
10731083
kubeServer.RegisterWithTaints,
10741084
kubeServer.AllowedUnsafeSysctls,
1075-
kubeServer.RemoteRuntimeEndpoint,
1076-
kubeServer.RemoteImageEndpoint,
10771085
kubeServer.ExperimentalMounterPath,
10781086
kubeServer.ExperimentalKernelMemcgNotification,
10791087
kubeServer.ExperimentalCheckNodeCapabilitiesBeforeMount,
@@ -1083,7 +1091,6 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie
10831091
kubeServer.MaxContainerCount,
10841092
kubeServer.MasterServiceNamespace,
10851093
kubeServer.RegisterSchedulable,
1086-
kubeServer.NonMasqueradeCIDR,
10871094
kubeServer.KeepTerminatedPodVolumes,
10881095
kubeServer.NodeLabels,
10891096
kubeServer.SeccompProfileRoot,
@@ -1138,7 +1145,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
11381145
kubeDeps *kubelet.Dependencies,
11391146
crOptions *config.ContainerRuntimeOptions,
11401147
containerRuntime string,
1141-
runtimeCgroups string,
11421148
hostnameOverride string,
11431149
nodeIP string,
11441150
providerID string,
@@ -1148,8 +1154,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
11481154
registerNode bool,
11491155
registerWithTaints []api.Taint,
11501156
allowedUnsafeSysctls []string,
1151-
remoteRuntimeEndpoint string,
1152-
remoteImageEndpoint string,
11531157
experimentalMounterPath string,
11541158
experimentalKernelMemcgNotification bool,
11551159
experimentalCheckNodeCapabilitiesBeforeMount bool,
@@ -1159,7 +1163,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
11591163
maxContainerCount int32,
11601164
masterServiceNamespace string,
11611165
registerSchedulable bool,
1162-
nonMasqueradeCIDR string,
11631166
keepTerminatedPodVolumes bool,
11641167
nodeLabels map[string]string,
11651168
seccompProfileRoot string,
@@ -1172,7 +1175,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
11721175
kubeDeps,
11731176
crOptions,
11741177
containerRuntime,
1175-
runtimeCgroups,
11761178
hostnameOverride,
11771179
nodeIP,
11781180
providerID,
@@ -1182,8 +1184,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
11821184
registerNode,
11831185
registerWithTaints,
11841186
allowedUnsafeSysctls,
1185-
remoteRuntimeEndpoint,
1186-
remoteImageEndpoint,
11871187
experimentalMounterPath,
11881188
experimentalKernelMemcgNotification,
11891189
experimentalCheckNodeCapabilitiesBeforeMount,
@@ -1193,7 +1193,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
11931193
maxContainerCount,
11941194
masterServiceNamespace,
11951195
registerSchedulable,
1196-
nonMasqueradeCIDR,
11971196
keepTerminatedPodVolumes,
11981197
nodeLabels,
11991198
seccompProfileRoot,

‎pkg/kubelet/kubelet.go‎

Lines changed: 91 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,10 @@ type Dependencies struct {
266266
KubeletConfigController *kubeletconfig.Controller
267267
RemoteRuntimeService internalapi.RuntimeService
268268
RemoteImageService internalapi.ImageManagerService
269+
criHandler http.Handler
270+
dockerLegacyService dockershim.DockerLegacyService
271+
// remove it after cadvisor.UsingLegacyCadvisorStats dropped.
272+
useLegacyCadvisorStats bool
269273
}
270274

271275
// makePodSourceConfig creates a config.PodConfig from the given
@@ -319,13 +323,90 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku
319323
return cfg, nil
320324
}
321325

326+
// PreInitRuntimeService will init runtime service before RunKubelet.
327+
func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
328+
kubeDeps *Dependencies,
329+
crOptions *config.ContainerRuntimeOptions,
330+
containerRuntime string,
331+
runtimeCgroups string,
332+
remoteRuntimeEndpoint string,
333+
remoteImageEndpoint string,
334+
nonMasqueradeCIDR string) error {
335+
if remoteRuntimeEndpoint != "" {
336+
// remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified
337+
if remoteImageEndpoint == "" {
338+
remoteImageEndpoint = remoteRuntimeEndpoint
339+
}
340+
}
341+
342+
switch containerRuntime {
343+
case kubetypes.DockerContainerRuntime:
344+
// TODO: These need to become arguments to a standalone docker shim.
345+
pluginSettings := dockershim.NetworkPluginSettings{
346+
HairpinMode: kubeletconfiginternal.HairpinMode(kubeCfg.HairpinMode),
347+
NonMasqueradeCIDR: nonMasqueradeCIDR,
348+
PluginName: crOptions.NetworkPluginName,
349+
PluginConfDir: crOptions.CNIConfDir,
350+
PluginBinDirString: crOptions.CNIBinDir,
351+
PluginCacheDir: crOptions.CNICacheDir,
352+
MTU: int(crOptions.NetworkPluginMTU),
353+
}
354+
355+
// Create and start the CRI shim running as a grpc server.
356+
streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions)
357+
ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig,
358+
&pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming)
359+
if err != nil {
360+
return err
361+
}
362+
if crOptions.RedirectContainerStreaming {
363+
kubeDeps.criHandler = ds
364+
}
365+
366+
// The unix socket for kubelet <-> dockershim communication, dockershim start before runtime service init.
367+
klog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q",
368+
remoteRuntimeEndpoint,
369+
remoteImageEndpoint)
370+
klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.")
371+
dockerServer := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds)
372+
if err := dockerServer.Start(); err != nil {
373+
return err
374+
}
375+
376+
// Create dockerLegacyService when the logging driver is not supported.
377+
supported, err := ds.IsCRISupportedLogDriver()
378+
if err != nil {
379+
return err
380+
}
381+
if !supported {
382+
kubeDeps.dockerLegacyService = ds
383+
}
384+
case kubetypes.RemoteContainerRuntime:
385+
// No-op.
386+
break
387+
default:
388+
return fmt.Errorf("unsupported CRI runtime: %q", containerRuntime)
389+
}
390+
391+
var err error
392+
if kubeDeps.RemoteRuntimeService, err = remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration); err != nil {
393+
return err
394+
}
395+
if kubeDeps.RemoteImageService, err = remote.NewRemoteImageService(remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout.Duration); err != nil {
396+
return err
397+
}
398+
399+
kubeDeps.useLegacyCadvisorStats = cadvisor.UsingLegacyCadvisorStats(containerRuntime, remoteRuntimeEndpoint)
400+
401+
return nil
402+
}
403+
322404
// NewMainKubelet instantiates a new Kubelet object along with all the required internal modules.
323405
// No initialization of Kubelet and its modules should happen here.
324406
func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
325407
kubeDeps *Dependencies,
326408
crOptions *config.ContainerRuntimeOptions,
327409
containerRuntime string,
328-
runtimeCgroups string,
329410
hostnameOverride string,
330411
nodeIP string,
331412
providerID string,
@@ -335,8 +416,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
335416
registerNode bool,
336417
registerWithTaints []api.Taint,
337418
allowedUnsafeSysctls []string,
338-
remoteRuntimeEndpoint string,
339-
remoteImageEndpoint string,
340419
experimentalMounterPath string,
341420
experimentalKernelMemcgNotification bool,
342421
experimentalCheckNodeCapabilitiesBeforeMount bool,
@@ -346,7 +425,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
346425
maxContainerCount int32,
347426
masterServiceNamespace string,
348427
registerSchedulable bool,
349-
nonMasqueradeCIDR string,
350428
keepTerminatedPodVolumes bool,
351429
nodeLabels map[string]string,
352430
seccompProfileRoot string,
@@ -589,82 +667,11 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
589667

590668
klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet)
591669

592-
if remoteRuntimeEndpoint != "" {
593-
// remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified
594-
if remoteImageEndpoint == "" {
595-
remoteImageEndpoint = remoteRuntimeEndpoint
596-
}
597-
}
598-
599-
// TODO: These need to become arguments to a standalone docker shim.
600-
pluginSettings := dockershim.NetworkPluginSettings{
601-
HairpinMode: kubeletconfiginternal.HairpinMode(kubeCfg.HairpinMode),
602-
NonMasqueradeCIDR: nonMasqueradeCIDR,
603-
PluginName: crOptions.NetworkPluginName,
604-
PluginConfDir: crOptions.CNIConfDir,
605-
PluginBinDirString: crOptions.CNIBinDir,
606-
PluginCacheDir: crOptions.CNICacheDir,
607-
MTU: int(crOptions.NetworkPluginMTU),
608-
}
609-
610670
klet.resourceAnalyzer = serverstats.NewResourceAnalyzer(klet, kubeCfg.VolumeStatsAggPeriod.Duration)
611671

612-
// if left at nil, that means it is unneeded
613-
var legacyLogProvider kuberuntime.LegacyLogProvider
614-
615-
switch containerRuntime {
616-
case kubetypes.DockerContainerRuntime:
617-
// Create and start the CRI shim running as a grpc server.
618-
streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions)
619-
ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig,
620-
&pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming)
621-
if err != nil {
622-
return nil, err
623-
}
624-
if crOptions.RedirectContainerStreaming {
625-
klet.criHandler = ds
626-
}
627-
628-
// The unix socket for kubelet <-> dockershim communication.
629-
klog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q",
630-
remoteRuntimeEndpoint,
631-
remoteImageEndpoint)
632-
klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.")
633-
server := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds)
634-
if err := server.Start(); err != nil {
635-
return nil, err
636-
}
637-
638-
// Create dockerLegacyService when the logging driver is not supported.
639-
supported, err := ds.IsCRISupportedLogDriver()
640-
if err != nil {
641-
return nil, err
642-
}
643-
if !supported {
644-
klet.dockerLegacyService = ds
645-
legacyLogProvider = ds
646-
}
647-
case kubetypes.RemoteContainerRuntime:
648-
// No-op.
649-
break
650-
default:
651-
return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime)
652-
}
653-
654-
runtimeService, imageService := kubeDeps.RemoteRuntimeService, kubeDeps.RemoteImageService
655-
if runtimeService == nil {
656-
runtimeService, err = remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration)
657-
if err != nil {
658-
return nil, err
659-
}
660-
}
661-
if imageService == nil {
662-
imageService, err = remote.NewRemoteImageService(remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout.Duration)
663-
if err != nil {
664-
return nil, err
665-
}
666-
}
667-
klet.runtimeService = runtimeService
672+
klet.dockerLegacyService = kubeDeps.dockerLegacyService
673+
klet.criHandler = kubeDeps.criHandler
674+
klet.runtimeService = kubeDeps.RemoteRuntimeService
668675

669676
if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.KubeClient != nil {
670677
klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.KubeClient)
@@ -687,10 +694,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
687694
int(kubeCfg.RegistryBurst),
688695
kubeCfg.CPUCFSQuota,
689696
kubeCfg.CPUCFSQuotaPeriod,
690-
runtimeService,
691-
imageService,
697+
kubeDeps.RemoteRuntimeService,
698+
kubeDeps.RemoteImageService,
692699
kubeDeps.ContainerManager.InternalContainerLifecycle(),
693-
legacyLogProvider,
700+
kubeDeps.dockerLegacyService,
694701
klet.runtimeClassManager,
695702
)
696703
if err != nil {
@@ -706,7 +713,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
706713
}
707714
klet.runtimeCache = runtimeCache
708715

709-
if cadvisor.UsingLegacyCadvisorStats(containerRuntime, remoteRuntimeEndpoint) {
716+
if kubeDeps.useLegacyCadvisorStats {
710717
klet.StatsProvider = stats.NewCadvisorStatsProvider(
711718
klet.cadvisor,
712719
klet.resourceAnalyzer,
@@ -720,8 +727,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
720727
klet.resourceAnalyzer,
721728
klet.podManager,
722729
klet.runtimeCache,
723-
runtimeService,
724-
imageService,
730+
kubeDeps.RemoteRuntimeService,
731+
kubeDeps.RemoteImageService,
725732
stats.NewLogMetricsService(),
726733
kubecontainer.RealOS{})
727734
}

0 commit comments

Comments
 (0)