CSPL-4022 Addressing PR comments

Author: kasiakoziol

cmd/main.go

@@ -48,7 +48,6 @@ import (
 
 	enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3"
 	enterpriseApi "github.com/splunk/splunk-operator/api/v4"
-	enterprisev4 "github.com/splunk/splunk-operator/api/v4"
 	"github.com/splunk/splunk-operator/internal/controller"
 	//+kubebuilder:scaffold:imports
 	//extapi "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -63,7 +62,6 @@ func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(enterpriseApi.AddToScheme(scheme))
 	utilruntime.Must(enterpriseApiV3.AddToScheme(scheme))
-	utilruntime.Must(enterprisev4.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 	//utilruntime.Must(extapi.AddToScheme(scheme))
 }

docs/AppFramework.md

@@ -21,6 +21,7 @@
 - [App Framework Fields](#description-of-app-framework-specification-fields)
 - [App Framework Examples](#examples-of-app-framework-usage)
   - [Standalone](#how-to-use-the-app-framework-on-a-standalone-cr)
+  - [Ingestor Cluster](#how-to-use-the-app-framework-on-ingestor-cluster)
   - [Cluster Manager](#how-to-use-the-app-framework-on-indexer-cluster)
   - [Search Head Cluster](#how-to-use-the-app-framework-on-search-head-cluster)
   - [Multiple Scopes](#how-to-install-apps-for-both-local-and-cluster-scopes)
@@ -812,11 +813,11 @@ Copy your Splunk App or Add-on archive files to the unique folders on the remote
 
 
 ## Description of App Framework Specification fields
-The App Framework configuration is supported on the following Custom Resources: Standalone, ClusterManager, SearchHeadCluster, MonitoringConsole and LicenseManager. Configuring the App framework requires:
+The App Framework configuration is supported on the following Custom Resources: Standalone, IngestorCluster, ClusterManager, SearchHeadCluster, MonitoringConsole and LicenseManager. Configuring the App framework requires:
 
 * Remote Source of Apps: Define the remote storage location, including unique folders, and the path to each folder.
 * Destination of Apps: Define which Custom Resources need to be configured.
-* Scope of Apps: Define if the apps need to be installed and run locally (such as Standalone, Monitoring Console and License Manager,) or cluster-wide (such as Indexer Cluster, and Search Head Cluster.)
+* Scope of Apps: Define if the apps need to be installed and run locally (such as Standalone, Monitoring Console, License Manager and Ingestor Cluster) or cluster-wide (such as Indexer Cluster, and Search Head Cluster.)
 
 Here is a typical App framework configuration in a Custom Resource definition:
 
@@ -931,6 +932,7 @@ NOTE: If an app source name needs to be changed, make sure the name change is pe
     | Standalone        | local                                  | Yes                   | $SPLUNK_HOME/etc/apps            | N/A |
     | LicenseManager    | local                                  | Yes                   | $SPLUNK_HOME/etc/apps            | N/A |
     | MonitoringConsole | local                                  | Yes                   | $SPLUNK_HOME/etc/apps            | N/A |
+    | IngestorCluster   | local                                  | Yes                   | $SPLUNK_HOME/etc/apps            | N/A |
     | IndexerCluster    | N/A                                    | No                    | N/A                              | $SPLUNK_HOME/etc/peer-apps |
 
 * `volume` refers to the remote storage volume name configured under the `volumes` stanza (see previous section.)
@@ -1008,6 +1010,69 @@ volumes:
 
 Apply the Custom Resource specification: `kubectl apply -f Standalone.yaml`
 
+### How to use the App Framework on Ingestor Cluster
+
+In this example, you'll deploy Ingestor Cluster with a remote storage volume, the location of the app archive, and set the installation location for the Splunk Enterprise Pod instance by using `scope`.
+
+Example using s3: IngestorCluster.yaml
+
+```yaml
+apiVersion: enterprise.splunk.com/v4
+kind: IngestorCluster
+metadata:
+  name: ic
+  finalizers:
+  - enterprise.splunk.com/delete-pvc
+spec:
+  replicas: 1
+  appRepo:
+    appsRepoPollIntervalSeconds: 600
+    defaults:
+      volumeName: volume_app_repo
+      scope: local
+    appSources:
+      - name: networkApps
+        location: networkAppsLoc/
+      - name: authApps
+        location: authAppsLoc/
+    volumes:
+      - name: volume_app_repo
+        storageType: s3
+        provider: aws
+        path: bucket-app-framework/IngestorCluster-us/
+        endpoint: https://s3-us-west-2.amazonaws.com
+        region: us-west-2
+        secretRef: s3-secret
+```
+
+Volume variants for other providers (replace only the volumes stanza):
+
+Azure Blob volumes snippet:
+
+```yaml
+volumes:
+  - name: volume_app_repo
+    storageType: blob
+    provider: azure
+    path: bucket-app-framework/IngestorCluster-us/
+    endpoint: https://mystorageaccount.blob.core.windows.net
+    secretRef: azureblob-secret
+```
+
+GCP GCS volumes snippet:
+
+```yaml
+volumes:
+  - name: volume_app_repo
+    storageType: gcs
+    provider: gcp
+    path: bucket-app-framework/IngestorCluster-us/
+    endpoint: https://storage.googleapis.com
+    secretRef: gcs-secret
+```
+
+Apply the Custom Resource specification: `kubectl apply -f IngestorCluster.yaml`
+
 ### How to use the App Framework on Indexer Cluster
 
 This example describes the installation of apps on an Indexer Cluster and Cluster Manager. This is achieved by deploying a ClusterManager CR with a remote storage volume, setting the location of the app archives, and the installation scope to support both local and cluster app path distribution.

docs/CustomResources.md

@@ -13,6 +13,7 @@ you can use to manage Splunk Enterprise deployments in your Kubernetes cluster.
   - [SearchHeadCluster Resource Spec Parameters](#searchheadcluster-resource-spec-parameters)
   - [ClusterManager Resource Spec Parameters](#clustermanager-resource-spec-parameters)
   - [IndexerCluster Resource Spec Parameters](#indexercluster-resource-spec-parameters)
+  - [IngestorCluster Resource Spec Parameters](#ingestorcluster-resource-spec-parameters)
   - [MonitoringConsole Resource Spec Parameters](#monitoringconsole-resource-spec-parameters)
   - [Examples of Guaranteed and Burstable QoS](#examples-of-guaranteed-and-burstable-qos)
     - [A Guaranteed QoS Class example:](#a-guaranteed-qos-class-example)
@@ -134,7 +135,7 @@ spec:
 
 The following additional configuration parameters may be used for all Splunk
 Enterprise resources, including: `Standalone`, `LicenseManager`,
-`SearchHeadCluster`, `ClusterManager` and `IndexerCluster`:
+`SearchHeadCluster`, `ClusterManager`, `IndexerCluster` and `IngestorCluster`:
 
 | Key                | Type    | Description                                                                   |
 | ------------------ | ------- | ----------------------------------------------------------------------------- |
@@ -321,6 +322,27 @@ the `IndexerCluster` resource provides the following `Spec` configuration parame
 | ---------- | ------- | ----------------------------------------------------- |
 | replicas   | integer | The number of indexer cluster members (minimum of 3, which is the default) |
 
+## IngestorCluster Resource Spec Parameters
+
+```yaml
+apiVersion: enterprise.splunk.com/v4
+kind: IngestorCluster
+metadata:
+  name: ic
+spec:
+  replicas: 3
+  busConfigurationRef: 
+    name: bus-config
+```
+Note:  `busConfigurationRef` is required field in case of IngestorCluster resource since it will be used to connect the IngestorCluster to BusConfiguration resource.
+
+In addition to [Common Spec Parameters for All Resources](#common-spec-parameters-for-all-resources)
+and [Common Spec Parameters for All Splunk Enterprise Resources](#common-spec-parameters-for-all-splunk-enterprise-resources),
+the `IngestorCluster` resource provides the following `Spec` configuration parameters:
+
+| Key        | Type    | Description                                           |
+| ---------- | ------- | ----------------------------------------------------- |
+| replicas   | integer | The number of ingestor peers (minimum of 3 which is the default) |
 
 ## MonitoringConsole Resource Spec Parameters
 
@@ -436,6 +458,7 @@ The Splunk Operator controller reconciles every Splunk Enterprise CR. However, t
 | clustermaster.enterprise.splunk.com | "clustermaster.enterprise.splunk.com/paused" |
 | clustermanager.enterprise.splunk.com | "clustermanager.enterprise.splunk.com/paused" |
 | indexercluster.enterprise.splunk.com | "indexercluster.enterprise.splunk.com/paused" |
+| ingestorcluster.enterprise.splunk.com | "ingestorcluster.enterprise.splunk.com/paused" |
 | licensemaster.enterprise.splunk.com | "licensemaster.enterprise.splunk.com/paused" |
 | monitoringconsole.enterprise.splunk.com | "monitoringconsole.enterprise.splunk.com/paused" |
 | searchheadcluster.enterprise.splunk.com | "searchheadcluster.enterprise.splunk.com/paused" |
@@ -505,6 +528,7 @@ Below is a table listing `app.kubernetes.io/name` values mapped to CRDs
 | clustermanager.enterprise.splunk.com | cluster-manager |
 | clustermaster.enterprise.splunk.com | cluster-master |
 | indexercluster.enterprise.splunk.com | indexer-cluster |
+| ingestorcluster.enterprise.splunk.com | ingestor-cluster |
 | licensemanager.enterprise.splunk.com | license-manager |
 | licensemaster.enterprise.splunk.com | license-master |
 | monitoringconsole.enterprise.splunk.com | monitoring-console |

pkg/splunk/client/enterprise.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-logr/logr"
 	splcommon "github.com/splunk/splunk-operator/pkg/splunk/common"
 )
 
@@ -969,19 +970,23 @@ func (c *SplunkClient) RestartSplunk() error {
 
 // Updates conf files and their properties
 // See https://help.splunk.com/en/splunk-enterprise/leverage-rest-apis/rest-api-reference/10.0/configuration-endpoints/configuration-endpoint-descriptions
-func (c *SplunkClient) UpdateConfFile(fileName, property string, propertyKVList [][]string) error {
+func (c *SplunkClient) UpdateConfFile(scopedLog logr.Logger, fileName, property string, propertyKVList [][]string) error {
 	// Creates an object in a conf file if it doesn't exist
 	endpoint := fmt.Sprintf("%s/servicesNS/nobody/system/configs/conf-%s", c.ManagementURI, fileName)
 	body := fmt.Sprintf("name=%s", property)
 
+	scopedLog.Info("Creating conf file object if it does not exist", "fileName", fileName, "property", property)
 	request, err := http.NewRequest("POST", endpoint, strings.NewReader(body))
 	if err != nil {
+		scopedLog.Error(err, "Failed to create conf file object if it does not exist", "fileName", fileName, "property", property)
 		return err
 	}
 
+	scopedLog.Info("Validating conf file object creation", "fileName", fileName, "property", property)
 	expectedStatus := []int{200, 201, 409}
 	err = c.Do(request, expectedStatus, nil)
 	if err != nil {
+		scopedLog.Error(err, fmt.Sprintf("Status not in %v for conf file object creation", expectedStatus), "fileName", fileName, "property", property)
 		return err
 	}
 
@@ -995,25 +1000,37 @@ func (c *SplunkClient) UpdateConfFile(fileName, property string, propertyKVList
 		body = body[:len(body)-1]
 	}
 
+	scopedLog.Info("Updating conf file object", "fileName", fileName, "property", property, "body", body)
 	request, err = http.NewRequest("POST", endpoint, strings.NewReader(body))
 	if err != nil {
+		scopedLog.Error(err, "Failed to update conf file object", "fileName", fileName, "property", property, "body", body)
 		return err
 	}
 
+	scopedLog.Info("Validating conf file object update", "fileName", fileName, "property", property)
 	expectedStatus = []int{200, 201}
 	err = c.Do(request, expectedStatus, nil)
+	if err != nil {
+		scopedLog.Error(err, fmt.Sprintf("Status not in %v for conf file object update", expectedStatus), "fileName", fileName, "property", property, "body", body)
+	}
 	return err
 }
 
 // Deletes conf files properties
-func (c *SplunkClient) DeleteConfFileProperty(fileName, property string) error {
+func (c *SplunkClient) DeleteConfFileProperty(scopedLog logr.Logger, fileName, property string) error {
 	endpoint := fmt.Sprintf("%s/servicesNS/nobody/system/configs/conf-%s/%s", c.ManagementURI, fileName, property)
 
+	scopedLog.Info("Deleting conf file object", "fileName", fileName, "property", property)
 	request, err := http.NewRequest("DELETE", endpoint, nil)
 	if err != nil {
+		scopedLog.Error(err, "Failed to delete conf file object", "fileName", fileName, "property", property)
 		return err
 	}
 
 	expectedStatus := []int{200, 201, 404}
-	return c.Do(request, expectedStatus, nil)
+	err = c.Do(request, expectedStatus, nil)
+	if err != nil {
+		scopedLog.Error(err, fmt.Sprintf("Status not in %v for conf file object deletion", expectedStatus), "fileName", fileName, "property", property)
+	}
+	return err
 }

pkg/splunk/client/enterprise_test.go

@@ -16,13 +16,15 @@
 package client
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"net/url"
 	"strings"
 	"testing"
 
 	splcommon "github.com/splunk/splunk-operator/pkg/splunk/common"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	spltest "github.com/splunk/splunk-operator/pkg/splunk/test"
 )
@@ -660,6 +662,9 @@ func TestUpdateConfFile(t *testing.T) {
 	value := "myvalue"
 	fileName := "outputs"
 
+	reqLogger := log.FromContext(context.TODO())
+	scopedLog := reqLogger.WithName("TestUpdateConfFile")
+
 	// First request: create the property (object) if it doesn't exist
 	createBody := strings.NewReader(fmt.Sprintf("name=%s", property))
 	wantCreateRequest, _ := http.NewRequest("POST", "https://localhost:8089/servicesNS/nobody/system/configs/conf-outputs", createBody)
@@ -675,7 +680,7 @@ func TestUpdateConfFile(t *testing.T) {
 	c := NewSplunkClient("https://localhost:8089", "admin", "p@ssw0rd")
 	c.Client = mockSplunkClient
 
-	err := c.UpdateConfFile(fileName, property, [][]string{{key, value}})
+	err := c.UpdateConfFile(scopedLog, fileName, property, [][]string{{key, value}})
 	if err != nil {
 		t.Errorf("UpdateConfFile err = %v", err)
 	}
@@ -685,7 +690,7 @@ func TestUpdateConfFile(t *testing.T) {
 	mockSplunkClient = &spltest.MockHTTPClient{}
 	mockSplunkClient.AddHandler(wantCreateRequest, 500, "", nil)
 	c.Client = mockSplunkClient
-	err = c.UpdateConfFile(fileName, property, [][]string{{key, value}})
+	err = c.UpdateConfFile(scopedLog, fileName, property, [][]string{{key, value}})
 	if err == nil {
 		t.Errorf("UpdateConfFile expected error on create, got nil")
 	}
@@ -695,7 +700,7 @@ func TestUpdateConfFile(t *testing.T) {
 	mockSplunkClient.AddHandler(wantCreateRequest, 201, "", nil)
 	mockSplunkClient.AddHandler(wantUpdateRequest, 500, "", nil)
 	c.Client = mockSplunkClient
-	err = c.UpdateConfFile(fileName, property, [][]string{{key, value}})
+	err = c.UpdateConfFile(scopedLog, fileName, property, [][]string{{key, value}})
 	if err == nil {
 		t.Errorf("UpdateConfFile expected error on update, got nil")
 	}
@@ -706,6 +711,9 @@ func TestDeleteConfFileProperty(t *testing.T) {
 	property := "myproperty"
 	fileName := "outputs"
 
+	reqLogger := log.FromContext(context.TODO())
+	scopedLog := reqLogger.WithName("TestDeleteConfFileProperty")
+
 	wantDeleteRequest, _ := http.NewRequest("DELETE", fmt.Sprintf("https://localhost:8089/servicesNS/nobody/system/configs/conf-outputs/%s", property), nil)
 
 	mockSplunkClient := &spltest.MockHTTPClient{}
@@ -714,7 +722,7 @@ func TestDeleteConfFileProperty(t *testing.T) {
 	c := NewSplunkClient("https://localhost:8089", "admin", "p@ssw0rd")
 	c.Client = mockSplunkClient
 
-	err := c.DeleteConfFileProperty(fileName, property)
+	err := c.DeleteConfFileProperty(scopedLog, fileName, property)
 	if err != nil {
 		t.Errorf("DeleteConfFileProperty err = %v", err)
 	}
@@ -724,7 +732,7 @@ func TestDeleteConfFileProperty(t *testing.T) {
 	mockSplunkClient = &spltest.MockHTTPClient{}
 	mockSplunkClient.AddHandler(wantDeleteRequest, 500, "", nil)
 	c.Client = mockSplunkClient
-	err = c.DeleteConfFileProperty(fileName, property)
+	err = c.DeleteConfFileProperty(scopedLog, fileName, property)
 	if err == nil {
 		t.Errorf("DeleteConfFileProperty expected error on delete, got nil")
 	}