diff --git a/.github/workflows/int-test-workflow.yml b/.github/workflows/int-test-workflow.yml index 5cd50d879..d463c5bca 100644 --- a/.github/workflows/int-test-workflow.yml +++ b/.github/workflows/int-test-workflow.yml @@ -4,7 +4,6 @@ on: branches: - develop - master - - cspl-1740 jobs: int-tests: strategy: diff --git a/test/c3/appframework/appframework_test.go b/test/c3/appframework/appframework_test.go index 47a51dd3c..afa04e0b0 100644 --- a/test/c3/appframework/appframework_test.go +++ b/test/c3/appframework/appframework_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v3" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" testenv "github.com/splunk/splunk-operator/test/testenv" corev1 "k8s.io/api/core/v1" ) @@ -2885,4 +2886,101 @@ var _ = Describe("c3appfw test", func() { testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") }) }) + + Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { + It("integration, c3, appframeworkc3, appframework: can deploy a C3 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { + + /* Test Steps + ################## SETUP #################### + * Upload big-size app to S3 for Indexer Cluster and Search Head Cluster + * Create app sources for Cluster Manager and Deployer + * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready + * When app download is complete, delete apps from app directory + ######### VERIFICATIONS ############# + * Verify Apps are Downloaded in App Deployment Info + * Verify Apps Copied in App Deployment Info + * Verify App Package is deleted from Operator Pod + * Verify Apps Installed in App Deployment Info + * Verify App Package is deleted from Splunk Pod + * Verify bundle push is successful + * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods + */ + + //################## SETUP #################### + // Download big size apps from S3 + appList := testenv.BigSingleApp + appFileList := testenv.GetAppFileList(appList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download big-size app") + + // Upload big size app to S3 for Indexer Cluster + appVersion := "V1" + testcaseEnvInst.Log.Info("Upload big size app to S3 for Indexer Cluster") + s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Indexer Cluster") + uploadedApps = append(uploadedApps, uploadedFiles...) + + // Upload big size app to S3 for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big size app to S3 for Search Head Cluster") + s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") + uploadedApps = append(uploadedApps, uploadedFiles...) + + // Create App framework Spec for C3 + appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) + appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) + appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) + appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + + // Deploy C3 CRD + testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") + indexerReplicas := 3 + shReplicas := 3 + cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") + + // Verify App Download is completed on Cluster Manager + testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + + //Delete apps from app directory when app download is complete + opPod := testenv.GetOperatorPodName(testcaseEnvInst) + podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) + err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) + Expect(err).To(Succeed(), "Unable to delete file on pod") + + // Ensure Cluster Manager goes to Ready phase + testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + + // Ensure Search Head Cluster go to Ready phase + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + + // Verify RF SF is met + testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + + // Get Pod age to check for pod resets later + splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) + + //######### VERIFICATIONS ############# + var idxcPodNames, shcPodNames []string + idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) + shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) + cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} + deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} + cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} + shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} + allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} + testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") + + // Verify no pods reset by checking the pod age + testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + + }) + }) }) diff --git a/test/m4/appframework/appframework_test.go b/test/m4/appframework/appframework_test.go index a5d931a6a..65a647a16 100644 --- a/test/m4/appframework/appframework_test.go +++ b/test/m4/appframework/appframework_test.go @@ -24,9 +24,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - testenv "github.com/splunk/splunk-operator/test/testenv" - enterpriseApi "github.com/splunk/splunk-operator/api/v3" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" + testenv "github.com/splunk/splunk-operator/test/testenv" corev1 "k8s.io/api/core/v1" ) @@ -2453,4 +2453,99 @@ var _ = Describe("m4appfw test", func() { testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") }) }) + + Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + It("integration, m4, appframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { + + /* Test Steps + ################## SETUP ################## + * Upload big-size app to S3 for Indexer Cluster and Search Head Cluster + * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready + * When app download is complete, delete apps from app directory + ########## VERIFICATIONS ########## + * Verify Apps Downloaded in App Deployment Info + * Verify Apps Copied in App Deployment Info + * Verify App Package is deleted from Operator Pod + * Verify Apps Installed in App Deployment Info + * Verify App Package is deleted from Splunk Pod + * Verify bundle push is successful + * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods + */ + + //################## SETUP ################## + // Download big size apps from S3 + appList := testenv.BigSingleApp + appFileList := testenv.GetAppFileList(appList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download big-size app") + + // Upload big size app to S3 for Indexer Cluster + appVersion := "V1" + testcaseEnvInst.Log.Info("Upload big size app to S3 for Indexer Cluster") + uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Indexer Cluster") + uploadedApps = append(uploadedApps, uploadedFiles...) + + // Upload big size app to S3 for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big size app to S3 for Search Head Cluster") + uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") + uploadedApps = append(uploadedApps, uploadedFiles...) + + // Create App framework Spec for M4 + appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) + appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + + // Deploy M4 CRD + testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") + siteCount := 3 + shReplicas := 3 + indexersPerSite := 1 + cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + + // Verify App Download is completed on Cluster Manager + testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + + //Delete apps from app directory when app download is complete + opPod := testenv.GetOperatorPodName(testcaseEnvInst) + podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) + err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) + Expect(err).To(Succeed(), "Unable to delete file on pod") + + // Ensure that the Cluster Manager goes to Ready phase + testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + + // Ensure the Indexers of all sites go to Ready phase + testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) + + // Ensure Indexer Cluster configured as Multisite + testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) + + // Ensure Search Head Cluster go to Ready phase + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + + // Verify RF SF is met + testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + + // Get Pod age to check for pod resets later + splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) + + //########## VERIFICATIONS ########## + var idxcPodNames, shcPodNames []string + idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) + shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) + cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} + deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} + cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} + shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} + allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} + testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") + + // Verify no pods reset by checking the pod age + testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + }) + }) }) diff --git a/test/s1/appframework/appframework_test.go b/test/s1/appframework/appframework_test.go index 7c0ef189a..2e36f9c12 100644 --- a/test/s1/appframework/appframework_test.go +++ b/test/s1/appframework/appframework_test.go @@ -23,9 +23,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - testenv "github.com/splunk/splunk-operator/test/testenv" - enterpriseApi "github.com/splunk/splunk-operator/api/v3" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" + testenv "github.com/splunk/splunk-operator/test/testenv" corev1 "k8s.io/api/core/v1" ) @@ -1779,4 +1779,75 @@ var _ = Describe("s1appfw test", func() { testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") }) }) + + Context("Standalone deployment (S1) with App Framework", func() { + It("integration, s1, appframeworks1, appframework: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", func() { + + /* Test Steps + ################## SETUP #################### + * Upload big-size app to S3 for Standalone + * Create app source for Standalone + * Prepare and deploy Standalone + * When app download is complete, delete apps from app directory + ############## VERIFICATIONS ################ + * Verify App installation is in progress on Standalone + * Upload more apps from S3 during bigger app install + * Wait for polling interval to pass + * Verify all apps are installed on Standalone + */ + + // ################## SETUP FOR STANDALONE #################### + // Download big size apps from S3 + appVersion := "V1" + appList := testenv.BigSingleApp + appFileList := testenv.GetAppFileList(appList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download big app") + + // Upload big-size app to S3 for Standalone + testcaseEnvInst.Log.Info("Upload big-size app to S3 for Standalone") + uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") + uploadedApps = append(uploadedApps, uploadedFiles...) + + // Create App framework spec for Standalone + appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + }, + Volumes: []corev1.Volume{}, + }, + AppFrameworkConfig: appFrameworkSpec, + } + + // Deploy Standalone + testcaseEnvInst.Log.Info("Deploy Standalone") + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + + // Verify App Download is completed on Standalone + testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + + //Delete apps from app-directory when app download is complete + opPod := testenv.GetOperatorPodName(testcaseEnvInst) + podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) + err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) + Expect(err).To(Succeed(), "Unable to delete file on pod") + + // Wait for Standalone to be in READY status + testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) + + // Get Pod age to check for pod resets later + splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) + + // ############ VERIFICATION ########### + standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} + standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} + allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} + testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") + }) + }) }) diff --git a/test/testenv/appframework_utils.go b/test/testenv/appframework_utils.go index 2f33872e9..6c7234307 100644 --- a/test/testenv/appframework_utils.go +++ b/test/testenv/appframework_utils.go @@ -385,12 +385,7 @@ func AppFrameWorkVerifications(ctx context.Context, deployment *Deployment, test } // Verify apps packages are deleted from the operator pod for all CRs - var opPod string - if testenvInstance.clusterWideOperator != "true" { - opPod = GetOperatorPodName(testenvInstance.GetName()) - } else { - opPod = GetOperatorPodName("splunk-operator") - } + opPod := GetOperatorPodName(testenvInstance) for _, appSource := range appSource { testenvInstance.Log.Info(fmt.Sprintf("Verify apps %s packages are deleted from the operator pod for CR %v with name %v", appSource.CrAppVersion, appSource.CrKind, appSource.CrName)) opPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), appSource.CrKind, deployment.GetName(), appSource.CrAppScope, appSource.CrAppSourceName) diff --git a/test/testenv/deployment.go b/test/testenv/deployment.go index 46026c163..326c4402d 100644 --- a/test/testenv/deployment.go +++ b/test/testenv/deployment.go @@ -111,12 +111,10 @@ func (d *Deployment) Teardown() error { var err error var output []byte - var podName string + podName := GetOperatorPodName(d.testenv) if d.testenv.clusterWideOperator != "true" { - podName = GetOperatorPodName(d.testenv.GetName()) output, err = exec.Command("kubectl", "logs", "-n", d.testenv.GetName(), podName).Output() } else { - podName = GetOperatorPodName("splunk-operator") output, err = exec.Command("kubectl", "logs", "-n", "splunk-operator", podName, "manager").Output() } if err != nil { diff --git a/test/testenv/util.go b/test/testenv/util.go index 8bbbf9df9..1246fe046 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -603,7 +603,13 @@ func DumpGetTopPods(ns string) []string { } // GetOperatorPodName returns name of operator pod in the namespace -func GetOperatorPodName(ns string) string { +func GetOperatorPodName(testcaseEnvInst *TestCaseEnv) string { + var ns string + if testcaseEnvInst.clusterWideOperator != "true" { + ns = testcaseEnvInst.GetName() + } else { + ns = "splunk-operator" + } output, err := exec.Command("kubectl", "get", "pods", "-n", ns).Output() var splunkPods string if err != nil { @@ -881,7 +887,7 @@ func DeleteOperatorPod(testcaseEnvInst *TestCaseEnv) error { } else { ns = "splunk-operator" } - podName = GetOperatorPodName(ns) + podName = GetOperatorPodName(testcaseEnvInst) _, err := exec.Command("kubectl", "delete", "pod", "-n", ns, podName).Output() if err != nil { @@ -890,3 +896,16 @@ func DeleteOperatorPod(testcaseEnvInst *TestCaseEnv) error { } return nil } + +// DeleteFilesOnOperatorPod Delete files on Operator Pod +func DeleteFilesOnOperatorPod(ctx context.Context, deployment *Deployment, podName string, filenames []string) error { + for _, filepath := range filenames { + cmd := fmt.Sprintf("rm -f %s", filepath) + _, err := ExecuteCommandOnOperatorPod(ctx, deployment, podName, cmd) + if err != nil { + logf.Log.Error(err, "Failed to delete file on pod ", "PodName", podName, "location", filepath, "command", cmd) + return err + } + } + return nil +}