diff --git a/examples/kubernetes_multicluster/canary/app.pipecd.yaml b/examples/kubernetes_multicluster/canary/app.pipecd.yaml new file mode 100644 index 0000000000..85eb0f5541 --- /dev/null +++ b/examples/kubernetes_multicluster/canary/app.pipecd.yaml @@ -0,0 +1,44 @@ +apiVersion: pipecd.dev/v1beta1 +kind: KubernetesApp +spec: + name: canary-multicluster + labels: + env: example + team: product + description: | + This app demonstrates how to deploy a Kubernetes application across multiple clusters + using a Canary strategy with the kubernetes_multicluster plugin. + The canary variant is first rolled out to cluster-us only, then after approval + the primary rollout is applied to all clusters, and finally the canary resources + are cleaned up with K8S_CANARY_CLEAN. + plugins: + kubernetes_multicluster: + input: + multiTargets: + - target: + name: cluster-us + manifests: + - cluster-us/deployment.yaml + - cluster-us/service.yaml + - target: + name: cluster-eu + manifests: + - cluster-eu/deployment.yaml + - cluster-eu/service.yaml + pipeline: + stages: + # Deploy the canary variant to cluster-us only (10% of replicas). + - name: K8S_CANARY_ROLLOUT + with: + replicas: 10% + multiTarget: + - target: + name: cluster-us + # Wait for approval before rolling out to all clusters. + - name: WAIT_APPROVAL + # Roll out the new version as primary to all clusters. + - name: K8S_PRIMARY_ROLLOUT + with: + prune: true + # Remove the canary variant resources from all clusters. + - name: K8S_CANARY_CLEAN diff --git a/examples/kubernetes_multicluster/canary/cluster-eu/deployment.yaml b/examples/kubernetes_multicluster/canary/cluster-eu/deployment.yaml new file mode 100644 index 0000000000..3a44702095 --- /dev/null +++ b/examples/kubernetes_multicluster/canary/cluster-eu/deployment.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: canary-multicluster + labels: + app: canary-multicluster +spec: + replicas: 2 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: canary-multicluster + pipecd.dev/variant: primary + template: + metadata: + labels: + app: canary-multicluster + pipecd.dev/variant: primary + spec: + containers: + - name: helloworld + image: ghcr.io/pipe-cd/helloworld:v0.32.0 + args: + - server + ports: + - containerPort: 9085 diff --git a/examples/kubernetes_multicluster/canary/cluster-eu/service.yaml b/examples/kubernetes_multicluster/canary/cluster-eu/service.yaml new file mode 100644 index 0000000000..8d77886f8b --- /dev/null +++ b/examples/kubernetes_multicluster/canary/cluster-eu/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: canary-multicluster +spec: + selector: + app: canary-multicluster + ports: + - protocol: TCP + port: 9085 + targetPort: 9085 diff --git a/examples/kubernetes_multicluster/canary/cluster-us/deployment.yaml b/examples/kubernetes_multicluster/canary/cluster-us/deployment.yaml new file mode 100644 index 0000000000..3a44702095 --- /dev/null +++ b/examples/kubernetes_multicluster/canary/cluster-us/deployment.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: canary-multicluster + labels: + app: canary-multicluster +spec: + replicas: 2 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: canary-multicluster + pipecd.dev/variant: primary + template: + metadata: + labels: + app: canary-multicluster + pipecd.dev/variant: primary + spec: + containers: + - name: helloworld + image: ghcr.io/pipe-cd/helloworld:v0.32.0 + args: + - server + ports: + - containerPort: 9085 diff --git a/examples/kubernetes_multicluster/canary/cluster-us/service.yaml b/examples/kubernetes_multicluster/canary/cluster-us/service.yaml new file mode 100644 index 0000000000..8d77886f8b --- /dev/null +++ b/examples/kubernetes_multicluster/canary/cluster-us/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: canary-multicluster +spec: + selector: + app: canary-multicluster + ports: + - protocol: TCP + port: 9085 + targetPort: 9085 diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/canary.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/canary.go index 168e340aec..50aa244b2b 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/canary.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/canary.go @@ -356,3 +356,87 @@ func patchManifest(m provider.Manifest, patch kubeconfig.K8sResourcePatch) (*pro return buildManifest(proc.Bytes()) } + +func (p *Plugin) executeK8sMultiCanaryCleanStage(ctx context.Context, input *sdk.ExecuteStageInput[kubeconfig.KubernetesApplicationSpec], dts []*sdk.DeployTarget[kubeconfig.KubernetesDeployTargetConfig]) sdk.StageStatus { + lp := input.Client.LogPersister() + + cfg, err := input.Request.TargetDeploymentSource.AppConfig() + if err != nil { + lp.Errorf("Failed while decoding application config (%v)", err) + return sdk.StageStatusFailure + } + + deployTargetMap := make(map[string]*sdk.DeployTarget[kubeconfig.KubernetesDeployTargetConfig], len(dts)) + for _, dt := range dts { + deployTargetMap[dt.Name] = dt + } + + type targetConfig struct { + deployTarget *sdk.DeployTarget[kubeconfig.KubernetesDeployTargetConfig] + } + + targetConfigs := make([]targetConfig, 0, len(dts)) + if len(cfg.Spec.Input.MultiTargets) == 0 { + for _, dt := range dts { + targetConfigs = append(targetConfigs, targetConfig{deployTarget: dt}) + } + } else { + for _, mt := range cfg.Spec.Input.MultiTargets { + dt, ok := deployTargetMap[mt.Target.Name] + if !ok { + lp.Infof("Ignore multi target '%s': not matched any deployTarget", mt.Target.Name) + continue + } + targetConfigs = append(targetConfigs, targetConfig{deployTarget: dt}) + } + } + + eg, ctx := errgroup.WithContext(ctx) + for _, tc := range targetConfigs { + eg.Go(func() error { + lp.Infof("Start cleaning CANARY variant on target %s", tc.deployTarget.Name) + if err := p.canaryClean(ctx, input, tc.deployTarget, cfg); err != nil { + return fmt.Errorf("failed to clean CANARY variant on target %s: %w", tc.deployTarget.Name, err) + } + return nil + }) + } + + if err := eg.Wait(); err != nil { + lp.Errorf("Failed while cleaning CANARY variant (%v)", err) + return sdk.StageStatusFailure + } + + return sdk.StageStatusSuccess +} + +func (p *Plugin) canaryClean( + ctx context.Context, + input *sdk.ExecuteStageInput[kubeconfig.KubernetesApplicationSpec], + dt *sdk.DeployTarget[kubeconfig.KubernetesDeployTargetConfig], + cfg *sdk.ApplicationConfig[kubeconfig.KubernetesApplicationSpec], +) error { + lp := input.Client.LogPersister() + + var ( + appCfg = cfg.Spec + variantLabel = appCfg.VariantLabel.Key + canaryVariant = appCfg.VariantLabel.CanaryValue + ) + + toolRegistry := toolregistry.NewRegistry(input.Client.ToolRegistry()) + + kubectlPath, err := toolRegistry.Kubectl(ctx, cmp.Or(appCfg.Input.KubectlVersion, dt.Config.KubectlVersion)) + if err != nil { + return fmt.Errorf("failed while getting kubectl tool: %w", err) + } + + kubectl := provider.NewKubectl(kubectlPath) + applier := provider.NewApplier(kubectl, appCfg.Input, dt.Config, input.Logger) + + if err := deleteVariantResources(ctx, lp, kubectl, dt.Config.KubeConfigPath, applier, input.Request.Deployment.ApplicationID, variantLabel, canaryVariant); err != nil { + return fmt.Errorf("unable to remove canary resources: %w", err) + } + + return nil +} diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/canary_test.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/canary_test.go index e596b99dd6..85e8a38c92 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/canary_test.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/canary_test.go @@ -320,3 +320,362 @@ func TestPlugin_executeK8sMultiCanaryRolloutStage_WithoutCreateService(t *testin require.Error(t, err) assert.True(t, k8serrors.IsNotFound(err)) } + + +func TestPlugin_executeK8sMultiCanaryCleanStage(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + // initialize tool registry + testRegistry := toolregistrytest.NewTestToolRegistry(t) + + // read the application config from the example file + appCfg := sdk.LoadApplicationConfigForTest[kubeconfig.KubernetesApplicationSpec](t, filepath.Join("testdata", "simple", "app.pipecd.yaml"), "kubernetes_multicluster") + + input := &sdk.ExecuteStageInput[kubeconfig.KubernetesApplicationSpec]{ + Request: sdk.ExecuteStageRequest[kubeconfig.KubernetesApplicationSpec]{ + StageName: "K8S_CANARY_CLEAN", + StageConfig: []byte(`{}`), + TargetDeploymentSource: sdk.DeploymentSource[kubeconfig.KubernetesApplicationSpec]{ + ApplicationDirectory: filepath.Join("testdata", "simple"), + CommitHash: "0123456789", + ApplicationConfig: appCfg, + ApplicationConfigFilename: "app.pipecd.yaml", + }, + Deployment: sdk.Deployment{ + PipedID: "piped-id", + ApplicationID: "app-id", + }, + }, + Client: sdk.NewClient(nil, "kubernetes_multicluster", "", "", logpersistertest.NewTestLogPersister(t), testRegistry), + Logger: zaptest.NewLogger(t), + } + + // initialize deploy target config and dynamic client for assertions with envtest + dtConfig, dynamicClient := setupTestDeployTargetConfigAndDynamicClient(t) + + deploymentRes := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} + + // Pre-create a canary deployment resource in the cluster (simulating what K8S_CANARY_ROLLOUT would do). + canaryDeployment := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]any{ + "name": "simple-canary", + "namespace": "default", + "labels": map[string]any{ + "app": "simple", + "pipecd.dev/managed-by": "piped", + "pipecd.dev/piped": "piped-id", + "pipecd.dev/application": "app-id", + "pipecd.dev/variant": "canary", + }, + "annotations": map[string]any{ + "pipecd.dev/managed-by": "piped", + "pipecd.dev/application": "app-id", + "pipecd.dev/variant": "canary", + }, + }, + "spec": map[string]any{ + "replicas": int64(1), + "selector": map[string]any{ + "matchLabels": map[string]any{ + "app": "simple", + "pipecd.dev/variant": "canary", + }, + }, + "template": map[string]any{ + "metadata": map[string]any{ + "labels": map[string]any{ + "app": "simple", + "pipecd.dev/variant": "canary", + }, + }, + "spec": map[string]any{ + "containers": []any{ + map[string]any{ + "name": "helloworld", + "image": "ghcr.io/pipe-cd/helloworld:v0.32.0", + }, + }, + }, + }, + }, + }, + } + + _, err := dynamicClient.Resource(deploymentRes).Namespace("default").Create(ctx, canaryDeployment, metav1.CreateOptions{}) + require.NoError(t, err) + + // Verify the canary deployment exists before running the stage. + _, err = dynamicClient.Resource(deploymentRes).Namespace("default").Get(ctx, "simple-canary", metav1.GetOptions{}) + require.NoError(t, err) + + plugin := &Plugin{} + + status := plugin.executeK8sMultiCanaryCleanStage(ctx, input, []*sdk.DeployTarget[kubeconfig.KubernetesDeployTargetConfig]{ + { + Name: "default", + Config: *dtConfig, + }, + }) + + assert.Equal(t, sdk.StageStatusSuccess, status) + + // Assert that the canary deployment has been deleted. + _, err = dynamicClient.Resource(deploymentRes).Namespace("default").Get(ctx, "simple-canary", metav1.GetOptions{}) + require.Error(t, err) + assert.True(t, k8serrors.IsNotFound(err)) +} + +func TestPlugin_executeK8sMultiCanaryCleanStage_multipleTargets(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + // initialize tool registry + testRegistry := toolregistrytest.NewTestToolRegistry(t) + + // read the application config from the example file + appCfg := sdk.LoadApplicationConfigForTest[kubeconfig.KubernetesApplicationSpec](t, filepath.Join("testdata", "simple", "app.pipecd.yaml"), "kubernetes_multicluster") + + // Set up two separate clusters. + clusterUS := setupCluster(t, "cluster-us") + clusterEU := setupCluster(t, "cluster-eu") + + input := &sdk.ExecuteStageInput[kubeconfig.KubernetesApplicationSpec]{ + Request: sdk.ExecuteStageRequest[kubeconfig.KubernetesApplicationSpec]{ + StageName: "K8S_CANARY_CLEAN", + StageConfig: []byte(`{}`), + TargetDeploymentSource: sdk.DeploymentSource[kubeconfig.KubernetesApplicationSpec]{ + ApplicationDirectory: filepath.Join("testdata", "simple"), + CommitHash: "0123456789", + ApplicationConfig: appCfg, + ApplicationConfigFilename: "app.pipecd.yaml", + }, + Deployment: sdk.Deployment{ + PipedID: "piped-id", + ApplicationID: "app-id", + }, + }, + Client: sdk.NewClient(nil, "kubernetes_multicluster", "", "", logpersistertest.NewTestLogPersister(t), testRegistry), + Logger: zaptest.NewLogger(t), + } + + deploymentRes := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} + + // Pre-create canary deployment resources on both clusters. + for _, c := range []*cluster{clusterUS, clusterEU} { + canaryDeployment := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]any{ + "name": "simple-canary", + "namespace": "default", + "labels": map[string]any{ + "app": "simple", + "pipecd.dev/managed-by": "piped", + "pipecd.dev/piped": "piped-id", + "pipecd.dev/application": "app-id", + "pipecd.dev/variant": "canary", + }, + "annotations": map[string]any{ + "pipecd.dev/managed-by": "piped", + "pipecd.dev/application": "app-id", + "pipecd.dev/variant": "canary", + }, + }, + "spec": map[string]any{ + "replicas": int64(1), + "selector": map[string]any{ + "matchLabels": map[string]any{ + "app": "simple", + "pipecd.dev/variant": "canary", + }, + }, + "template": map[string]any{ + "metadata": map[string]any{ + "labels": map[string]any{ + "app": "simple", + "pipecd.dev/variant": "canary", + }, + }, + "spec": map[string]any{ + "containers": []any{ + map[string]any{ + "name": "helloworld", + "image": "ghcr.io/pipe-cd/helloworld:v0.32.0", + }, + }, + }, + }, + }, + }, + } + _, err := c.cli.Resource(deploymentRes).Namespace("default").Create(ctx, canaryDeployment, metav1.CreateOptions{}) + require.NoError(t, err) + } + + plugin := &Plugin{} + + status := plugin.executeK8sMultiCanaryCleanStage(ctx, input, []*sdk.DeployTarget[kubeconfig.KubernetesDeployTargetConfig]{ + {Name: clusterUS.name, Config: *clusterUS.dtc}, + {Name: clusterEU.name, Config: *clusterEU.dtc}, + }) + + assert.Equal(t, sdk.StageStatusSuccess, status) + + // Assert that the canary deployments have been deleted on both clusters. + for _, c := range []*cluster{clusterUS, clusterEU} { + _, err := c.cli.Resource(deploymentRes).Namespace("default").Get(ctx, "simple-canary", metav1.GetOptions{}) + require.Error(t, err) + assert.True(t, k8serrors.IsNotFound(err), "canary deployment should be deleted on cluster %s", c.name) + } +} + +func TestPlugin_executeK8sMultiCanaryCleanStage_withCreateService(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + testRegistry := toolregistrytest.NewTestToolRegistry(t) + + configDir := filepath.Join("testdata", "canary_clean_with_create_service") + + appCfg := sdk.LoadApplicationConfigForTest[kubeconfig.KubernetesApplicationSpec](t, filepath.Join(configDir, "app.pipecd.yaml"), "kubernetes_multicluster") + + input := &sdk.ExecuteStageInput[kubeconfig.KubernetesApplicationSpec]{ + Request: sdk.ExecuteStageRequest[kubeconfig.KubernetesApplicationSpec]{ + StageName: "K8S_CANARY_ROLLOUT", + StageConfig: []byte(`{"createService": true}`), + TargetDeploymentSource: sdk.DeploymentSource[kubeconfig.KubernetesApplicationSpec]{ + ApplicationDirectory: configDir, + CommitHash: "0123456789", + ApplicationConfig: appCfg, + ApplicationConfigFilename: "app.pipecd.yaml", + }, + Deployment: sdk.Deployment{ + PipedID: "piped-id", + ApplicationID: "app-id", + }, + }, + Client: sdk.NewClient(nil, "kubernetes_multicluster", "", "", logpersistertest.NewTestLogPersister(t), testRegistry), + Logger: zaptest.NewLogger(t), + } + + dtConfig, dynamicClient := setupTestDeployTargetConfigAndDynamicClient(t) + + deployTarget := []*sdk.DeployTarget[kubeconfig.KubernetesDeployTargetConfig]{ + {Name: "default", Config: *dtConfig}, + } + + plugin := &Plugin{} + + deploymentRes := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} + serviceRes := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services"} + + ok := t.Run("execute canary rollout stage", func(t *testing.T) { + status := plugin.executeK8sMultiCanaryRolloutStage(ctx, input, deployTarget) + assert.Equal(t, sdk.StageStatusSuccess, status) + + // Assert canary deployment and service exist. + _, err := dynamicClient.Resource(deploymentRes).Namespace("default").Get(ctx, "simple-canary", metav1.GetOptions{}) + require.NoError(t, err) + + _, err = dynamicClient.Resource(serviceRes).Namespace("default").Get(ctx, "simple-canary", metav1.GetOptions{}) + require.NoError(t, err) + }) + require.True(t, ok) + + ok = t.Run("execute canary clean stage", func(t *testing.T) { + input.Request.StageName = "K8S_CANARY_CLEAN" + input.Request.StageConfig = []byte(`{}`) + + status := plugin.executeK8sMultiCanaryCleanStage(ctx, input, deployTarget) + assert.Equal(t, sdk.StageStatusSuccess, status) + + // Assert both canary deployment and service are deleted. + _, err := dynamicClient.Resource(deploymentRes).Namespace("default").Get(ctx, "simple-canary", metav1.GetOptions{}) + require.Error(t, err) + assert.True(t, k8serrors.IsNotFound(err)) + + _, err = dynamicClient.Resource(serviceRes).Namespace("default").Get(ctx, "simple-canary", metav1.GetOptions{}) + require.Error(t, err) + assert.True(t, k8serrors.IsNotFound(err)) + }) + require.True(t, ok) +} + +func TestPlugin_executeK8sMultiCanaryCleanStage_withoutCreateService(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + testRegistry := toolregistrytest.NewTestToolRegistry(t) + + configDir := filepath.Join("testdata", "canary_clean_without_create_service") + + appCfg := sdk.LoadApplicationConfigForTest[kubeconfig.KubernetesApplicationSpec](t, filepath.Join(configDir, "app.pipecd.yaml"), "kubernetes_multicluster") + + input := &sdk.ExecuteStageInput[kubeconfig.KubernetesApplicationSpec]{ + Request: sdk.ExecuteStageRequest[kubeconfig.KubernetesApplicationSpec]{ + StageName: "K8S_CANARY_ROLLOUT", + StageConfig: []byte(`{}`), + TargetDeploymentSource: sdk.DeploymentSource[kubeconfig.KubernetesApplicationSpec]{ + ApplicationDirectory: configDir, + CommitHash: "0123456789", + ApplicationConfig: appCfg, + ApplicationConfigFilename: "app.pipecd.yaml", + }, + Deployment: sdk.Deployment{ + PipedID: "piped-id", + ApplicationID: "app-id", + }, + }, + Client: sdk.NewClient(nil, "kubernetes_multicluster", "", "", logpersistertest.NewTestLogPersister(t), testRegistry), + Logger: zaptest.NewLogger(t), + } + + dtConfig, dynamicClient := setupTestDeployTargetConfigAndDynamicClient(t) + + deployTarget := []*sdk.DeployTarget[kubeconfig.KubernetesDeployTargetConfig]{ + {Name: "default", Config: *dtConfig}, + } + + plugin := &Plugin{} + + deploymentRes := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} + serviceRes := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services"} + + ok := t.Run("execute canary rollout stage", func(t *testing.T) { + status := plugin.executeK8sMultiCanaryRolloutStage(ctx, input, deployTarget) + assert.Equal(t, sdk.StageStatusSuccess, status) + + // Assert canary deployment exists but no canary service was created. + _, err := dynamicClient.Resource(deploymentRes).Namespace("default").Get(ctx, "simple-canary", metav1.GetOptions{}) + require.NoError(t, err) + + _, err = dynamicClient.Resource(serviceRes).Namespace("default").Get(ctx, "simple-canary", metav1.GetOptions{}) + require.Error(t, err) + assert.True(t, k8serrors.IsNotFound(err)) + }) + require.True(t, ok) + + ok = t.Run("execute canary clean stage", func(t *testing.T) { + input.Request.StageName = "K8S_CANARY_CLEAN" + input.Request.StageConfig = []byte(`{}`) + + status := plugin.executeK8sMultiCanaryCleanStage(ctx, input, deployTarget) + assert.Equal(t, sdk.StageStatusSuccess, status) + + // Assert canary deployment is deleted. + _, err := dynamicClient.Resource(deploymentRes).Namespace("default").Get(ctx, "simple-canary", metav1.GetOptions{}) + require.Error(t, err) + assert.True(t, k8serrors.IsNotFound(err)) + }) + require.True(t, ok) +} + diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go index 34c728190a..d73b587eb0 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go @@ -224,6 +224,8 @@ func generateVariantWorkloadManifests(workloads, configmaps, secrets []provider. } // deleteVariantResources deletes the resources of the specified variant. +// It finds the resources of the specified variant and deletes them. +// It deletes the resources in the order of Service -> Workload -> Others -> Cluster-scoped resources. func deleteVariantResources(ctx context.Context, lp sdk.StageLogPersister, kubectl *provider.Kubectl, kubeConfig string, applier *provider.Applier, applicationID, variantLabel, variant string) error { namespacedLiveResources, clusterScopedLiveResources, err := provider.GetLiveResources(ctx, kubectl, kubeConfig, applicationID, fmt.Sprintf("%s=%s", variantLabel, variant)) if err != nil { diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/pipeline.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/pipeline.go index 68123f5bef..63894ae1b7 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/pipeline.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/pipeline.go @@ -28,12 +28,15 @@ const ( StageK8sMultiRollback = "K8S_MULTI_ROLLBACK" // StageK8sMultiCanaryRollout represents the state where the new version is deployed as CANARY to all targets. StageK8sMultiCanaryRollout = "K8S_CANARY_ROLLOUT" + // StageK8sMultiCanaryClean represents the state where all canary resources should be removed. + StageK8sMultiCanaryClean = "K8S_CANARY_CLEAN" ) var allStages = []string{ StageK8sMultiSync, StageK8sMultiRollback, StageK8sMultiCanaryRollout, + StageK8sMultiCanaryClean, } const ( @@ -43,6 +46,8 @@ const ( StageDescriptionK8sMultiRollback = "Rollback the deployment" // StageDescriptionK8sMultiCanaryRollout represents the description of the K8sCanaryRollout stage. StageDescriptionK8sMultiCanaryRollout = "Rollout the new version as CANARY to all targets" + // StageDescriptionK8sMultiCanaryClean represents the description of the K8sCanaryClean stage. + StageDescriptionK8sMultiCanaryClean = "Remove all canary resources" ) func buildQuickSyncPipeline(autoRollback bool) []sdk.QuickSyncStage { diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/plugin.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/plugin.go index 7bdc75214a..f1e56b3f64 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/plugin.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/plugin.go @@ -69,6 +69,10 @@ func (p *Plugin) ExecuteStage(ctx context.Context, _ *sdk.ConfigNone, dts []*sdk }, nil case StageK8sMultiCanaryRollout: return &sdk.ExecuteStageResponse{Status: p.executeK8sMultiCanaryRolloutStage(ctx, input, dts)}, nil + case StageK8sMultiCanaryClean: + return &sdk.ExecuteStageResponse{ + Status: p.executeK8sMultiCanaryCleanStage(ctx, input, dts), + }, nil default: return nil, errors.New("unimplemented or unsupported stage") } diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_with_create_service/app.pipecd.yaml b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_with_create_service/app.pipecd.yaml new file mode 100644 index 0000000000..7d963fc32d --- /dev/null +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_with_create_service/app.pipecd.yaml @@ -0,0 +1,24 @@ +apiVersion: pipecd.dev/v1beta1 +kind: KubernetesApp +spec: + name: canary-clean + labels: + env: example + team: product + description: | + This app is test data for canary clean with create service. + pipeline: + stages: + - name: K8S_CANARY_ROLLOUT + with: + createService: true + - name: K8S_CANARY_CLEAN + plugins: + kubernetes_multicluster: + input: + manifests: + - deployment.yaml + - service.yaml + kubectlVersion: 1.32.2 + service: + name: simple diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_with_create_service/deployment.yaml b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_with_create_service/deployment.yaml new file mode 100644 index 0000000000..eb0a683f5c --- /dev/null +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_with_create_service/deployment.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: simple + labels: + app: simple +spec: + replicas: 2 + selector: + matchLabels: + app: simple + template: + metadata: + labels: + app: simple + spec: + containers: + - name: helloworld + image: ghcr.io/pipe-cd/helloworld:v0.32.0 + args: + - server + ports: + - containerPort: 9085 diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_with_create_service/service.yaml b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_with_create_service/service.yaml new file mode 100644 index 0000000000..29edcfeb79 --- /dev/null +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_with_create_service/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: simple +spec: + selector: + app: simple + ports: + - protocol: TCP + port: 9085 + targetPort: 9085 diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_without_create_service/app.pipecd.yaml b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_without_create_service/app.pipecd.yaml new file mode 100644 index 0000000000..e38817a082 --- /dev/null +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_without_create_service/app.pipecd.yaml @@ -0,0 +1,20 @@ +apiVersion: pipecd.dev/v1beta1 +kind: KubernetesApp +spec: + name: canary-clean + labels: + env: example + team: product + description: | + This app is test data for canary clean without create service. + pipeline: + stages: + - name: K8S_CANARY_ROLLOUT + - name: K8S_CANARY_CLEAN + plugins: + kubernetes_multicluster: + input: + manifests: + - deployment.yaml + - service.yaml + kubectlVersion: 1.32.2 diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_without_create_service/deployment.yaml b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_without_create_service/deployment.yaml new file mode 100644 index 0000000000..eb0a683f5c --- /dev/null +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_without_create_service/deployment.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: simple + labels: + app: simple +spec: + replicas: 2 + selector: + matchLabels: + app: simple + template: + metadata: + labels: + app: simple + spec: + containers: + - name: helloworld + image: ghcr.io/pipe-cd/helloworld:v0.32.0 + args: + - server + ports: + - containerPort: 9085 diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_without_create_service/service.yaml b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_without_create_service/service.yaml new file mode 100644 index 0000000000..29edcfeb79 --- /dev/null +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/testdata/canary_clean_without_create_service/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: simple +spec: + selector: + app: simple + ports: + - protocol: TCP + port: 9085 + targetPort: 9085 diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/liveresources.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/liveresources.go index 4064a9513e..d2bbaafb66 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/liveresources.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/liveresources.go @@ -21,19 +21,21 @@ import ( // GetLiveResources returns all live resources that belong to the given application. func GetLiveResources(ctx context.Context, kubectl *Kubectl, kubeconfig string, appID string, selector ...string) (namespaceScoped []Manifest, clusterScoped []Manifest, _ error) { - namespacedLiveResources, err := kubectl.GetAll(ctx, kubeconfig, - "", + selectors := make([]string, 0, len(selector)+2) + selectors = append(selectors, fmt.Sprintf("%s=%s", LabelManagedBy, ManagedByPiped), fmt.Sprintf("%s=%s", LabelApplication, appID), ) + if len(selector) > 0 { + selectors = append(selectors, selector...) + } + + namespacedLiveResources, err := kubectl.GetAll(ctx, kubeconfig, "", selectors...) if err != nil { return nil, nil, fmt.Errorf("failed while listing all namespace-scoped resources (%v)", err) } - clusterScopedLiveResources, err := kubectl.GetAllClusterScoped(ctx, kubeconfig, - fmt.Sprintf("%s=%s", LabelManagedBy, ManagedByPiped), - fmt.Sprintf("%s=%s", LabelApplication, appID), - ) + clusterScopedLiveResources, err := kubectl.GetAllClusterScoped(ctx, kubeconfig, selectors...) if err != nil { return nil, nil, fmt.Errorf("failed while listing all cluster-scoped resources (%v)", err) } diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/manifest.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/manifest.go index 560842d5f6..71657ab730 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/manifest.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/manifest.go @@ -108,6 +108,28 @@ func (m Manifest) Name() string { return m.body.GetName() } +// IsWorkload returns true if the manifest is a Deployment, StatefulSet, DaemonSet, ReplicaSet, or Pod. +// It checks the API group and the kind of the manifest. +func (m Manifest) IsWorkload() bool { + // TODO: check the API group more strictly. + if !isBuiltinAPIGroup(m.body.GroupVersionKind().Group) { + return false + } + switch m.body.GetKind() { + case KindDeployment, KindStatefulSet, KindDaemonSet, KindReplicaSet, KindPod: + return true + default: + return false + } +} + +// IsService returns true if the manifest is a Service. +// It checks the API group and the kind of the manifest. +func (m Manifest) IsService() bool { + // TODO: check the API group more strictly. + return isBuiltinAPIGroup(m.body.GroupVersionKind().Group) && m.body.GetKind() == KindService +} + // IsDeployment returns true if the manifest is a Deployment. // It checks the API group and the kind of the manifest. func (m Manifest) IsDeployment() bool { @@ -129,12 +151,6 @@ func (m Manifest) IsSecret() bool { return isBuiltinAPIGroup(m.body.GroupVersionKind().Group) && m.body.GetKind() == KindSecret } -// IsService returns true if the manifest is a Service. -// It checks the API group and the kind of the manifest. -func (m Manifest) IsService() bool { - return isBuiltinAPIGroup(m.body.GroupVersionKind().Group) && m.body.GetKind() == KindService -} - // IsConfigMap returns true if the manifest is a ConfigMap. // It checks the API group and the kind of the manifest. func (m Manifest) IsConfigMap() bool { diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/resource.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/resource.go index 882e185d49..ec97f9ba35 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/resource.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider/resource.go @@ -26,6 +26,9 @@ const ( // Workload KindDeployment = "Deployment" KindStatefulSet = "StatefulSet" + KindDaemonSet = "DaemonSet" + KindReplicaSet = "ReplicaSet" + KindPod = "Pod" // Service KindService = "Service"