[K8s testing] Support use of Kustomize (#1125)

* Add support for kustomize.

Author: constanca-m

docs/howto/system_testing.md

@@ -303,11 +303,11 @@ There are some specific environment variables that should be leveraged to overco
 ### Kubernetes service deployer
 
 The Kubernetes service deployer requires the `_dev/deploy/k8s` directory to be present. It can include additional `*.yaml` files to deploy
-custom applications in the Kubernetes cluster (e.g. Nginx deployment). If no resource definitions (`*.yaml` files ) are needed,
+custom applications in the Kubernetes cluster (e.g. Nginx deployment). It is possible to use a `kustomization.yaml` file. 
+If no resource definitions (`*.yaml` files ) are needed,
 the `_dev/deploy/k8s` directory must contain an `.empty` file (to preserve the `k8s` directory under version control).
 
 The Kubernetes service deployer needs [kind](https://kind.sigs.k8s.io/) to be installed and the cluster to be up and running:
-
 ```bash
 wget -qO-  https://raw.githubusercontent.com/elastic/elastic-package/main/scripts/kind-config.yaml | kind create cluster --config -
 ```

internal/kubectl/kubectl.go

@@ -7,12 +7,15 @@ package kubectl
 import (
 	"bytes"
 	"os/exec"
+	"path/filepath"
 
 	"github.com/pkg/errors"
 
 	"github.com/elastic/elastic-package/internal/logger"
 )
 
+const kustomizationFile = "kustomization.yaml"
+
 // CurrentContext function returns the selected Kubernetes context.
 func CurrentContext() (string, error) {
 	cmd := exec.Command("kubectl", "config", "current-context")
@@ -27,9 +30,13 @@ func CurrentContext() (string, error) {
 	return string(bytes.TrimSpace(output)), nil
 }
 
-func modifyKubernetesResources(action string, definitionPaths ...string) ([]byte, error) {
+func modifyKubernetesResources(action string, definitionPaths []string) ([]byte, error) {
 	args := []string{action}
 	for _, definitionPath := range definitionPaths {
+		if filepath.Base(definitionPath) == kustomizationFile {
+			args = []string{action, "-k", filepath.Dir(definitionPath)}
+			break
+		}
 		args = append(args, "-f")
 		args = append(args, definitionPath)
 	}

internal/kubectl/kubectl_apply.go

@@ -70,9 +70,9 @@ func (c condition) String() string {
 }
 
 // Apply function adds resources to the Kubernetes cluster based on provided definitions.
-func Apply(definitionPaths ...string) error {
-	logger.Debugf("Apply Kubernetes definitions")
-	out, err := modifyKubernetesResources("apply", definitionPaths...)
+func Apply(definitionsPath []string) error {
+	logger.Debugf("Apply Kubernetes custom definitions")
+	out, err := modifyKubernetesResources("apply", definitionsPath)
 	if err != nil {
 		return errors.Wrap(err, "can't modify Kubernetes resources (apply)")
 	}

internal/kubectl/kubectl_delete.go

@@ -5,7 +5,7 @@
 package kubectl
 
 // Delete function removes resources from the Kubernetes cluster based on provided definitions.
-func Delete(definitionPaths ...string) error {
-	_, err := modifyKubernetesResources("delete", definitionPaths...)
+func Delete(definitionsPath []string) error {
+	_, err := modifyKubernetesResources("delete", definitionsPath)
 	return err
 }

internal/testrunner/runners/system/servicedeployer/kubernetes.go

@@ -47,7 +47,7 @@ func (s kubernetesDeployedService) TearDown() error {
 		return nil
 	}
 
-	err = kubectl.Delete(definitionPaths...)
+	err = kubectl.Delete(definitionPaths)
 	if err != nil {
 		return errors.Wrapf(err, "can't uninstall Kubernetes resources (path: %s)", s.definitionsDir)
 	}
@@ -70,9 +70,9 @@ func (s *kubernetesDeployedService) SetContext(sc ServiceContext) error {
 var _ DeployedService = new(kubernetesDeployedService)
 
 // NewKubernetesServiceDeployer function creates a new instance of KubernetesServiceDeployer.
-func NewKubernetesServiceDeployer(definitionsDir string) (*KubernetesServiceDeployer, error) {
+func NewKubernetesServiceDeployer(definitionsPath string) (*KubernetesServiceDeployer, error) {
 	return &KubernetesServiceDeployer{
-		definitionsDir: definitionsDir,
+		definitionsDir: definitionsPath,
 	}, nil
 }
 
@@ -115,15 +115,15 @@ func (ksd KubernetesServiceDeployer) installCustomDefinitions() error {
 
 	definitionPaths, err := findKubernetesDefinitions(ksd.definitionsDir)
 	if err != nil {
-		return errors.Wrapf(err, "can't find Kubernetes definitions in given directory (path: %s)", ksd.definitionsDir)
+		return errors.Wrapf(err, "can't find Kubernetes definitions in given path: %s", ksd.definitionsDir)
 	}
 
 	if len(definitionPaths) == 0 {
-		logger.Debugf("no custom definitions found (directory: %s). Nothing else will be installed.", ksd.definitionsDir)
+		logger.Debugf("no custom definitions found (path: %s). Nothing else will be installed.", ksd.definitionsDir)
 		return nil
 	}
 
-	err = kubectl.Apply(definitionPaths...)
+	err = kubectl.Apply(definitionPaths)
 	if err != nil {
 		return errors.Wrap(err, "can't install custom definitions")
 	}

…dev/deploy/k8s/example-redis-config.yaml …ubernetes/_dev/deploy/k8s/configmap.yamltest/packages/with-kind/kubernetes/_dev/deploy/k8s/example-redis-config.yaml renamed to test/packages/with-kind/kubernetes/_dev/deploy/k8s/configmap.yaml test/packages/with-kind/kubernetes/_dev/deploy/k8s/example-redis-config.yaml renamed to test/packages/with-kind/kubernetes/_dev/deploy/k8s/configmap.yaml

@@ -1,7 +1,9 @@
-apiVersion: batch/v1beta1
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: hello
+  labels:
+    k8s-app: myjob
 spec:
   schedule: "*/1 * * * *"
   jobTemplate:

…/kubernetes/_dev/deploy/k8s/conrjob.yaml …/kubernetes/_dev/deploy/k8s/cronjob.yamltest/packages/with-kind/kubernetes/_dev/deploy/k8s/conrjob.yaml renamed to test/packages/with-kind/kubernetes/_dev/deploy/k8s/cronjob.yaml test/packages/with-kind/kubernetes/_dev/deploy/k8s/conrjob.yaml renamed to test/packages/with-kind/kubernetes/_dev/deploy/k8s/cronjob.yaml

@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluentd-elasticsearch
+  namespace: kube-system
+  labels:
+    k8s-app: fluentd-logging
+spec:
+  selector:
+    matchLabels:
+      name: fluentd-elasticsearch
+  template:
+    metadata:
+      labels:
+        name: fluentd-elasticsearch
+    spec:
+      tolerations:
+        # these tolerations are to have the daemonset runnable on control plane nodes
+        # remove them if your control plane nodes should not run pods
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: fluentd-elasticsearch
+          image: quay.io/fluentd_elasticsearch/fluentd:v2.5.2
+          resources:
+            limits:
+              memory: 200Mi
+            requests:
+              cpu: 100m
+              memory: 200Mi
+          volumeMounts:
+            - name: varlog
+              mountPath: /var/log
+            - name: varlibdockercontainers
+              mountPath: /var/lib/docker/containers
+              readOnly: true
+      terminationGracePeriodSeconds: 30
+      volumes:
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers

test/packages/with-kind/kubernetes/_dev/deploy/k8s/daemonset.yaml

@@ -0,0 +1,10 @@
+resources:
+  - pv.yaml
+  - pvc.yaml
+  - cronjob.yaml
+  - job.yaml
+  - daemonset.yaml
+  - configmap.yaml
+  - rq.yaml
+  - ss.yaml
+  - github.com/kubernetes/kube-state-metrics?ref=abe3fd3184e16893b5a47196f90a94ed13e1b04d

test/packages/with-kind/kubernetes/_dev/deploy/k8s/kustomization.yaml

@@ -1,5 +1,8 @@
 metricsets: ["pod"]
 add_metadata: {{add_metadata}}
+{{#if add_resource_metadata_config}}
+{{add_resource_metadata_config}}
+{{/if}}
 hosts:
 {{#each hosts}}
   - {{this}}
@@ -9,3 +12,14 @@ period: {{period}}
 bearer_token_file: {{bearer_token_file}}
 ssl.verification_mode: {{ssl.verification_mode}}
 {{/if}}
+{{#if ssl.certificate_authorities}}
+ssl.certificate_authorities:
+{{#each ssl.certificate_authorities}}
+  - {{this}}
+{{/each}}
+{{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}