CSPL-3558 Adding extra validation for integraion test

kasiakoziol · kasiakoziol · commit 03931398b533 · 2025-10-02T12:59:26.000+02:00
diff --git a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml
@@ -120,7 +120,7 @@ jobs:
                 appframeworksS1,
                 managersecret,
                 managermc,
-                indexingestseparation,
+                indingsep,
                ]
     runs-on: ubuntu-latest
     env:
diff --git a/.github/workflows/arm-AL2023-int-test-workflow.yml b/.github/workflows/arm-AL2023-int-test-workflow.yml
@@ -68,7 +68,7 @@ jobs:
             managercrcrud,
             licensemanager,
             managerdeletecr,
-            indexingestseparation,
+            indingsep,
           ]
     runs-on: ubuntu-latest
     needs: build-operator-image-arm-al2023
diff --git a/.github/workflows/arm-RHEL-build-test-push-workflow.yml b/.github/workflows/arm-RHEL-build-test-push-workflow.yml
@@ -68,7 +68,7 @@ jobs:
             managercrcrud,
             licensemanager,
             managerdeletecr,
-            indexingestseparation,
+            indingsep,
           ]
     runs-on: ubuntu-latest
     needs: build-operator-image-arm-rhel
diff --git a/.github/workflows/arm-RHEL-int-test-workflow.yml b/.github/workflows/arm-RHEL-int-test-workflow.yml
@@ -68,7 +68,7 @@ jobs:
             managercrcrud,
             licensemanager,
             managerdeletecr,
-            indexingestseparation,
+            indingsep,
           ]
     runs-on: ubuntu-latest
     needs: build-operator-image-arm-rhel
diff --git a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml
@@ -120,7 +120,7 @@ jobs:
                 appframeworksS1,
                 managersecret,
                 managermc,
-                indexingestseparation,
+                indingsep,
               ]
     runs-on: ubuntu-latest
     env:
diff --git a/.github/workflows/arm-Ubuntu-int-test-workflow.yml b/.github/workflows/arm-Ubuntu-int-test-workflow.yml
@@ -68,7 +68,7 @@ jobs:
             managercrcrud,
             licensemanager,
             managerdeletecr,
-            indexingestseparation,
+            indingsep,
           ]
     runs-on: ubuntu-latest
     needs: build-operator-image-arm-ubuntu
diff --git a/.github/workflows/build-test-push-workflow.yml b/.github/workflows/build-test-push-workflow.yml
@@ -166,7 +166,7 @@ jobs:
                 managerappframeworkm4,
                 managersecret,
                 managermc,
-                indexingestseparation,
+                indingsep,
                ]
     runs-on: ubuntu-latest
     env:
diff --git a/.github/workflows/distroless-build-test-push-workflow.yml b/.github/workflows/distroless-build-test-push-workflow.yml
@@ -165,7 +165,7 @@ jobs:
           managerappframeworkm4,
           managersecret,
           managermc,
-          indexingestseparation,
+          indingsep,
         ]
     runs-on: ubuntu-latest
     env:
diff --git a/.github/workflows/distroless-int-test-workflow.yml b/.github/workflows/distroless-int-test-workflow.yml
@@ -64,7 +64,7 @@ jobs:
             managercrcrud,
             licensemanager,
             managerdeletecr,
-            indexingestseparation,
+            indingsep,
           ]
     runs-on: ubuntu-latest
     needs: build-operator-image-distroless
diff --git a/.github/workflows/int-test-workflow.yml b/.github/workflows/int-test-workflow.yml
@@ -61,7 +61,7 @@ jobs:
             managercrcrud,
             licensemanager,
             managerdeletecr,
-            indexingestseparation,
+            indingsep,
           ]
     runs-on: ubuntu-latest
     needs: build-operator-image
diff --git a/.github/workflows/manual-int-test-workflow.yml b/.github/workflows/manual-int-test-workflow.yml
@@ -23,7 +23,7 @@ jobs:
             managerscaling,
             managercrcrud,
             licensemanager,
-            indexingestseparation,
+            indingsep,
           ]
     runs-on: ubuntu-latest
     env:
diff --git a/.github/workflows/namespace-scope-int-workflow.yml b/.github/workflows/namespace-scope-int-workflow.yml
@@ -19,7 +19,7 @@ jobs:
             managerscaling,
             managercrcrud,
             licensemanager,
-            indexingestseparation,
+            indingsep,
           ]
     runs-on: ubuntu-latest
     env:
diff --git a/.github/workflows/nightly-int-test-workflow.yml b/.github/workflows/nightly-int-test-workflow.yml
@@ -59,7 +59,7 @@ jobs:
             managerscaling,
             managercrcrud,
             licensemanager,
-            indexingestseparation,
+            indingsep,
           ]
     runs-on: ubuntu-latest
     needs: build-operator-image
diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go
@@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package indexingestsep
+package indingsep
 
 import (
 	"testing"
@@ -34,7 +34,7 @@ const (
 
 var (
 	testenvInstance *testenv.TestEnv
-	testSuiteName   = "indexingestsep-" + testenv.RandomDNSName(3)
+	testSuiteName   = "indingsep-" + testenv.RandomDNSName(3)
 )
 
 // TestBasic is the main entry point
diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go
@@ -11,11 +11,12 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package indexingestsep
+package indingsep
 
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/onsi/ginkgo/types"
 	. "github.com/onsi/ginkgo/v2"
@@ -26,7 +27,7 @@ import (
 	"github.com/splunk/splunk-operator/test/testenv"
 )
 
-var _ = Describe("indexingestsep test", func() {
+var _ = Describe("indingsep test", func() {
 
 	var testcaseEnvInst *testenv.TestCaseEnv
 	var deployment *testenv.Deployment
@@ -36,7 +37,7 @@ var _ = Describe("indexingestsep test", func() {
 	BeforeEach(func() {
 		var err error
 
-		name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3))
+		name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3))
 		testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name)
 		Expect(err).To(Succeed(), "Unable to create testcaseenv")
 
@@ -58,7 +59,40 @@ var _ = Describe("indexingestsep test", func() {
 	})
 
 	Context("Ingestor and Indexer deployment", func() {
-		It("indexingestsep, smoke, indexingestseparation: Splunk Operator can configure Ingestors and Indexers", func() {
+		It("indingsep, smoke, indingsep: Splunk Operator can configure Ingestors and Indexers", func() {
+			bus := enterpriseApi.PushBusSpec{
+				Type: "sqs_smartbus",
+				SQS: enterpriseApi.SQSSpec{
+					QueueName:                 "test-queue",
+					AuthRegion:                "us-west-2",
+					Endpoint:                  "https://sqs.us-west-2.amazonaws.com",
+					LargeMessageStoreEndpoint: "https://s3.us-west-2.amazonaws.com",
+					LargeMessageStorePath:     "s3://test-bucket/smartbus-test",
+					DeadLetterQueueName:       "test-dead-letter-queue",
+					MaxRetriesPerPart:         4,
+					RetryPolicy:               "max_count",
+					SendInterval:              "5s",
+					EncodingFormat:            "s2s",
+				},
+			}
+			pipelineConfig := enterpriseApi.PipelineConfigSpec{
+				RemoteQueueRuleset: false,
+				RuleSet:            true,
+				RemoteQueueTyping:  false,
+				RemoteQueueOutput:  false,
+				Typing:             true,
+				IndexerPipe:        true,
+			}
+			serviceAccountName := "index-ingest-sa"
+
+			// Create Service Account
+			testcaseEnvInst.CreateServiceAccount(serviceAccountName)
+
+			// Deploy Ingestor Cluster
+			testcaseEnvInst.Log.Info("Deploy Ingestor Cluster")
+			_, err := deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, bus, pipelineConfig, serviceAccountName)
+			Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster")
+
 			// Deploy Cluster Manager
 			testcaseEnvInst.Log.Info("Deploy Cluster Manager")
 			cmSpec := enterpriseApi.ClusterManagerSpec{
@@ -69,16 +103,28 @@ var _ = Describe("indexingestsep test", func() {
 					},
 				},
 			}
-			_, err := deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec)
+			_, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec)
 			Expect(err).To(Succeed(), "Unable to deploy Cluster Manager")
 
-			// Create Service Account
-			serviceAccountName := "index-ingest-sa"
-			testcaseEnvInst.CreateServiceAccount(serviceAccountName)
-
 			// Deploy Indexer Cluster
 			testcaseEnvInst.Log.Info("Deploy Indexer Cluster")
-			pullBus := enterpriseApi.PushBusSpec{
+			_, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", bus, pipelineConfig, serviceAccountName)
+			Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster")
+
+			// Ensure that Ingestor Cluster is in Ready phase
+			testenv.IngestorReady(ctx, deployment, testcaseEnvInst)
+
+			// Ensure that Cluster Manager is in Ready phase
+			testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst)
+
+			// Ensure that Indexer Cluster is in Ready phase
+			testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst)
+		})
+	})
+
+	Context("Ingestor and Indexer deployment", func() {
+		It("indingsep, integration, indingsep: Splunk Operator can configure Ingestors and Indexers", func() {
+			bus := enterpriseApi.PushBusSpec{
 				Type: "sqs_smartbus",
 				SQS: enterpriseApi.SQSSpec{
 					QueueName:                 "test-queue",
@@ -101,22 +147,121 @@ var _ = Describe("indexingestsep test", func() {
 				Typing:             true,
 				IndexerPipe:        true,
 			}
-			_, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", pullBus, pipelineConfig, serviceAccountName)
-			Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster")
+			serviceAccountName := "index-ingest-sa"
+
+			// Create Service Account
+			testcaseEnvInst.CreateServiceAccount(serviceAccountName)
 
 			// Deploy Ingestor Cluster
 			testcaseEnvInst.Log.Info("Deploy Ingestor Cluster")
-			_, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, pullBus, pipelineConfig, serviceAccountName)
+			_, err := deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, bus, pipelineConfig, serviceAccountName)
 			Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster")
 
+			// Deploy Cluster Manager
+			testcaseEnvInst.Log.Info("Deploy Cluster Manager")
+			cmSpec := enterpriseApi.ClusterManagerSpec{
+				CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{
+					Spec: enterpriseApi.Spec{
+						ImagePullPolicy: "Always",
+						Image:           testcaseEnvInst.GetSplunkImage(),
+					},
+				},
+			}
+			_, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec)
+			Expect(err).To(Succeed(), "Unable to deploy Cluster Manager")
+
+			// Deploy Indexer Cluster
+			testcaseEnvInst.Log.Info("Deploy Indexer Cluster")
+			_, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", bus, pipelineConfig, serviceAccountName)
+			Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster")
+
+			// Ensure that Ingestor Cluster is in Ready phase
+			testenv.IngestorReady(ctx, deployment, testcaseEnvInst)
+
 			// Ensure that Cluster Manager is in Ready phase
 			testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst)
 
 			// Ensure that Indexer Cluster is in Ready phase
 			testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst)
 
-			// Ensure that Ingestor Cluster is in Ready phase
-			testenv.IngestorReady(ctx, deployment, testcaseEnvInst)
+			// Get instance of current Ingestor Cluster CR with latest config
+			ingest := &enterpriseApi.IngestorCluster{}
+			err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest)
+			Expect(err).To(Succeed(), "Failed to get instance of Ingestor Cluster")
+
+			// Verify Ingestor Cluster Status
+			Expect(ingest.Status.PushBus).To(Equal(bus), "Ingestor PushBus status is not the same as provided as input")
+			Expect(ingest.Status.PipelineConfig).To(Equal(pipelineConfig), "Ingestor PipelineConfig status is not the same as provided as input")
+
+			// Get instance of current Indexer Cluster CR with latest config
+			index := &enterpriseApi.IndexerCluster{}
+			err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", index)
+			Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster")
+
+			// Verify Indexer Cluster Status
+			Expect(index.Status.PullBus).To(Equal(bus), "Indexer PullBus status is not the same as provided as input")
+			Expect(index.Status.PipelineConfig).To(Equal(pipelineConfig), "Indexer PipelineConfig status is not the same as provided as input")
+
+			// Verify conf files
+			pods := testenv.DumpGetPods(deployment.GetName())
+			for _, pod := range pods {
+				defaultsConf := ""
+
+				if strings.Contains(pod, "ingest") || strings.Contains(pod, "idxc") {
+					// Get outputs.conf
+					outputsPath := "opt/splunk/etc/system/local/outputs.conf"
+					outputsConf, err := testenv.GetConfFile(pod, outputsPath, deployment.GetName())
+					Expect(err).To(Succeed(), "Failed to get outputs.conf from Ingestor Cluster pod")
+					Expect(outputsConf).To(ContainSubstring("[remote_queue:test-queue]"), "outputs.conf does not contain smartbus queue name configuration")
+					Expect(outputsConf).To(ContainSubstring("remote_queue.type = sqs_smartbus"), "outputs.conf does not contain smartbus type configuration")
+					Expect(outputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.auth_region = us-west-2"), "outputs.conf does not contain smartbus region configuration")
+					Expect(outputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.dead_letter_queue.name = test-dead-letter-queue"), "outputs.conf does not contain smartbus dead letter queue configuration")
+					Expect(outputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.encoding_format = s2s"), "outputs.conf does not contain smartbus encoding format configuration")
+					Expect(outputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.endpoint = https://sqs.us-west-2.amazonaws.com"), "outputs.conf does not contain smartbus endpoint configuration")
+					Expect(outputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.large_message_store.endpoint = https://s3.us-west-2.amazonaws.com"), "outputs.conf does not contain smartbus large message store endpoint configuration")
+					Expect(outputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.large_message_store.path = s3://test-bucket/smartbus-test"), "outputs.conf does not contain smartbus large message store path configuration")
+					Expect(outputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.retry_policy = max_count"), "outputs.conf does not contain smartbus retry policy configuration")
+					Expect(outputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.send_interval = 5s"), "outputs.conf does not contain smartbus send interval configuration")
+					Expect(outputsConf).To(ContainSubstring("remote_queue.max_count.sqs_smartbus.max_retries_per_part = 4"), "outputs.conf does not contain smartbus max retries per part configuration")
+
+					// Get default-mode.conf
+					defaultsPath := "opt/splunk/etc/system/local/default-mode.conf"
+					defaultsConf, err := testenv.GetConfFile(pod, defaultsPath, deployment.GetName())
+					Expect(err).To(Succeed(), "Failed to get default-mode.conf from Ingestor Cluster pod")
+					Expect(defaultsConf).To(ContainSubstring("[pipeline:remotequeueruleset]\ndisabled = false"), "default-mode.conf does not contain remote queue ruleset stanza")
+					Expect(defaultsConf).To(ContainSubstring("[pipeline:ruleset]\ndisabled = true"), "default-mode.conf does not contain ruleset stanza")
+					Expect(defaultsConf).To(ContainSubstring("[pipeline:remotequeuetyping]\ndisabled = false"), "default-mode.conf does not contain remote queue typing stanza")
+					Expect(defaultsConf).To(ContainSubstring("[pipeline:remotequeueoutput]\ndisabled = false"), "default-mode.conf does not contain remote queue output stanza")
+					Expect(defaultsConf).To(ContainSubstring("[pipeline:typing]\ndisabled = true"), "default-mode.conf does not contain typing stanza")
+
+					// Get AWS env variables
+					envVars, err := testenv.GetAWSEnv(pod, deployment.GetName())
+					Expect(err).To(Succeed(), "Failed to get AWS env variables from Ingestor Cluster pod")
+					Expect(envVars).To(ContainSubstring("AWS_REGION=us-west-2"), "AWS env variables do not contain region")
+					Expect(envVars).To(ContainSubstring("AWS_DEFAULT_REGION=us-west-2"), "AWS env variables do not contain default region")
+					Expect(envVars).To(ContainSubstring("AWS_WEB_IDENTITY_TOKEN_FILE=/var/run/secrets/eks.amazonaws.com/serviceaccount/token"), "AWS env variables do not contain web identity token file")
+					Expect(envVars).To(ContainSubstring("AWS_ROLE_ARN=arn:aws:iam::"), "AWS env variables do not contain role arn")
+					Expect(envVars).To(ContainSubstring("AWS_STS_REGIONAL_ENDPOINTS=regional"), "AWS env variables do not contain STS regional endpoints")
+				}
+
+				if strings.Contains(pod, "ingest") {
+					Expect(defaultsConf).To(ContainSubstring("[pipeline:indexerPipe]\ndisabled = true"), "default-mode.conf does not contain indexer pipe stanza")
+				} else if strings.Contains(pod, "idxc") {
+					// Get inputs.conf
+					inputsPath := "opt/splunk/etc/system/local/inputs.conf"
+					inputsConf, err := testenv.GetConfFile(pod, inputsPath, deployment.GetName())
+					Expect(err).To(Succeed(), "Failed to get inputs.conf from Indexer Cluster pod")
+					Expect(inputsConf).To(ContainSubstring("[remote_queue:test-queue]"), "inputs.conf does not contain smartbus queue name configuration")
+					Expect(inputsConf).To(ContainSubstring("remote_queue.type = sqs_smartbus"), "inputs.conf does not contain smartbus type configuration")
+					Expect(inputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.auth_region = us-west-2"), "inputs.conf does not contain smartbus region configuration")
+					Expect(inputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.dead_letter_queue.name = test-dead-letter-queue"), "inputs.conf does not contain smartbus dead letter queue configuration")
+					Expect(inputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.endpoint = https://sqs.us-west-2.amazonaws.com"), "inputs.conf does not contain smartbus endpoint configuration")
+					Expect(inputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.large_message_store.endpoint = https://s3.us-west-2.amazonaws.com"), "inputs.conf does not contain smartbus large message store endpoint configuration")
+					Expect(inputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.large_message_store.path = s3://test-bucket/smartbus-test"), "inputs.conf does not contain smartbus large message store path configuration")
+					Expect(inputsConf).To(ContainSubstring("remote_queue.sqs_smartbus.retry_policy = max_count"), "inputs.conf does not contain smartbus retry policy configuration")
+					Expect(inputsConf).To(ContainSubstring("remote_queue.max_count.sqs_smartbus.max_retries_per_part = 4"), "inputs.conf does not contain smartbus max retries per part configuration")
+				}
+			}
 		})
 	})
 })
diff --git a/test/testenv/util.go b/test/testenv/util.go

Original file line number	Diff line number	Diff line change
`@@ -120,7 +120,7 @@ jobs:`
`120`	`120`	`appframeworksS1,`
`121`	`121`	`managersecret,`
`122`	`122`	`managermc,`
`123`		`- indexingestseparation,`
	`123`	`+ indingsep,`
`124`	`124`	`]`
`125`	`125`	`runs-on: ubuntu-latest`
`126`	`126`	`env:`
Original file line number	Diff line number	Diff line change
`@@ -68,7 +68,7 @@ jobs:`
`68`	`68`	`managercrcrud,`
`69`	`69`	`licensemanager,`
`70`	`70`	`managerdeletecr,`
`71`		`- indexingestseparation,`
	`71`	`+ indingsep,`
`72`	`72`	`]`
`73`	`73`	`runs-on: ubuntu-latest`
`74`	`74`	`needs: build-operator-image-arm-al2023`
Original file line number	Diff line number	Diff line change
`@@ -166,7 +166,7 @@ jobs:`
`166`	`166`	`managerappframeworkm4,`
`167`	`167`	`managersecret,`
`168`	`168`	`managermc,`
`169`		`- indexingestseparation,`
	`169`	`+ indingsep,`
`170`	`170`	`]`
`171`	`171`	`runs-on: ubuntu-latest`
`172`	`172`	`env:`
Original file line number	Diff line number	Diff line change
`@@ -165,7 +165,7 @@ jobs:`
`165`	`165`	`managerappframeworkm4,`
`166`	`166`	`managersecret,`
`167`	`167`	`managermc,`
`168`		`- indexingestseparation,`
	`168`	`+ indingsep,`
`169`	`169`	`]`
`170`	`170`	`runs-on: ubuntu-latest`
`171`	`171`	`env:`
Original file line number	Diff line number	Diff line change
`@@ -64,7 +64,7 @@ jobs:`
`64`	`64`	`managercrcrud,`
`65`	`65`	`licensemanager,`
`66`	`66`	`managerdeletecr,`
`67`		`- indexingestseparation,`
	`67`	`+ indingsep,`
`68`	`68`	`]`
`69`	`69`	`runs-on: ubuntu-latest`
`70`	`70`	`needs: build-operator-image-distroless`
Original file line number	Diff line number	Diff line change
`@@ -61,7 +61,7 @@ jobs:`
`61`	`61`	`managercrcrud,`
`62`	`62`	`licensemanager,`
`63`	`63`	`managerdeletecr,`
`64`		`- indexingestseparation,`
	`64`	`+ indingsep,`
`65`	`65`	`]`
`66`	`66`	`runs-on: ubuntu-latest`
`67`	`67`	`needs: build-operator-image`