Get elastic-agent-managed-daemonset.yaml from upstream 7.x instead of using a local static file (#452)

* Get elastic-agent-managed-daemonset.yaml from upstream 7.x instead of using a local static file

Author: MichaelKatsoulis

internal/configuration/locations/locations.go

@@ -23,8 +23,7 @@ const (
 
 	fieldsCachedDir = "cache/fields"
 
-	kubernetesDeployerElasticAgentYmlFile = "elastic-agent.yml"
-	terraformDeployerYmlFile              = "terraform-deployer.yml"
+	terraformDeployerYmlFile = "terraform-deployer.yml"
 )
 
 var (
@@ -84,11 +83,6 @@ func (loc LocationManager) KubernetesDeployerDir() string {
 	return filepath.Join(loc.stackPath, kubernetesDeployerDir)
 }
 
-// KubernetesDeployerAgentYml returns the Kubernetes Deployer Elastic Agent yml
-func (loc LocationManager) KubernetesDeployerAgentYml() string {
-	return filepath.Join(loc.stackPath, kubernetesDeployerDir, kubernetesDeployerElasticAgentYmlFile)
-}
-
 // TerraformDeployerDir returns the Terraform Directory
 func (loc LocationManager) TerraformDeployerDir() string {
 	return filepath.Join(loc.stackPath, terraformDeployerDir)

internal/install/install.go

@@ -9,7 +9,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -61,11 +60,6 @@ func EnsureInstalled() error {
 		return errors.Wrap(err, "writing stack resources failed")
 	}
 
-	err = writeKubernetesDeployerResources(elasticPackagePath)
-	if err != nil {
-		return errors.Wrap(err, "writing Kubernetes deployer resources failed")
-	}
-
 	err = writeTerraformDeployerResources(elasticPackagePath)
 	if err != nil {
 		return errors.Wrap(err, "writing Terraform deployer resources failed")
@@ -173,26 +167,6 @@ func writeStackResources(elasticPackagePath *locations.LocationManager) error {
 
 }
 
-func writeKubernetesDeployerResources(elasticPackagePath *locations.LocationManager) error {
-	err := os.MkdirAll(elasticPackagePath.KubernetesDeployerDir(), 0755)
-	if err != nil {
-		return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.KubernetesDeployerDir())
-	}
-
-	appConfig, err := Configuration()
-	if err != nil {
-		return errors.Wrap(err, "can't read application configuration")
-	}
-
-	err = writeStaticResource(err, elasticPackagePath.KubernetesDeployerAgentYml(),
-		strings.ReplaceAll(kubernetesDeployerElasticAgentYml, "{{ ELASTIC_AGENT_IMAGE_REF }}",
-			appConfig.DefaultStackImageRefs().ElasticAgent))
-	if err != nil {
-		return errors.Wrap(err, "writing static resource failed")
-	}
-	return nil
-}
-
 func writeTerraformDeployerResources(elasticPackagePath *locations.LocationManager) error {
 	terraformDeployer := elasticPackagePath.TerraformDeployerDir()
 	err := os.MkdirAll(terraformDeployer, 0755)

internal/kubectl/kubectl.go

@@ -49,3 +49,21 @@ func modifyKubernetesResources(action string, definitionPaths ...string) ([]byte
 	}
 	return output, nil
 }
+
+// applyKubernetesResourcesStdin applies a Kubernetes manifest provided as stdin.
+// It returns the resources created as output and an error
+func applyKubernetesResourcesStdin(input []byte) ([]byte, error) {
+	// create kubectl apply command
+	kubectlCmd := exec.Command("kubectl", "apply", "-f", "-", "-o", "yaml")
+	//Stdin of kubectl command is the manifest provided
+	kubectlCmd.Stdin = bytes.NewReader(input)
+	errOutput := new(bytes.Buffer)
+	kubectlCmd.Stderr = errOutput
+
+	logger.Debugf("run command: %s", kubectlCmd)
+	output, err := kubectlCmd.Output()
+	if err != nil {
+		return nil, errors.Wrapf(err, "kubectl apply failed (stderr=%q)", errOutput.String())
+	}
+	return output, nil
+}

internal/kubectl/kubectl_apply.go

@@ -84,6 +84,22 @@ func Apply(definitionPaths ...string) error {
 	return nil
 }
 
+// ApplyStdin function adds resources to the Kubernetes cluster based on provided stdin.
+func ApplyStdin(input []byte) error {
+	logger.Debugf("Apply Kubernetes stdin")
+	out, err := applyKubernetesResourcesStdin(input)
+	if err != nil {
+		return errors.Wrap(err, "can't modify Kubernetes resources (apply stdin)")
+	}
+
+	logger.Debugf("Handle \"apply\" command output")
+	err = handleApplyCommandOutput(out)
+	if err != nil {
+		return errors.Wrap(err, "can't handle command output")
+	}
+	return nil
+}
+
 func handleApplyCommandOutput(out []byte) error {
 	logger.Debugf("Extract resources from command output")
 	resources, err := extractResources(out)

internal/testrunner/runners/system/servicedeployer/kubernetes.go

@@ -5,18 +5,23 @@
 package servicedeployer
 
 import (
+	"io"
 	"io/ioutil"
+	"net/http"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/pkg/errors"
 
-	"github.com/elastic/elastic-package/internal/configuration/locations"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/kind"
 	"github.com/elastic/elastic-package/internal/kubectl"
 	"github.com/elastic/elastic-package/internal/logger"
 )
 
+const elasticAgentManagedYamlURL = "https://raw.githubusercontent.com/elastic/beats/7.x/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml"
+
 // KubernetesServiceDeployer is responsible for deploying resources in the Kubernetes cluster.
 type KubernetesServiceDeployer struct {
 	definitionsDir string
@@ -144,14 +149,58 @@ func findKubernetesDefinitions(definitionsDir string) ([]string, error) {
 func installElasticAgentInCluster() error {
 	logger.Debug("install Elastic Agent in the Kubernetes cluster")
 
-	locationManager, err := locations.NewLocationManager()
+	elasticAgentManagedYaml, err := getElasticAgentYAML()
+	logger.Debugf("downloaded %d bytes", len(elasticAgentManagedYaml))
 	if err != nil {
-		return errors.Wrap(err, "can't locate Kubernetes file for Elastic Agent in ")
+		return errors.Wrap(err, "can't retrieve Kubernetes file for Elastic Agent")
 	}
 
-	err = kubectl.Apply(locationManager.KubernetesDeployerAgentYml())
+	err = kubectl.ApplyStdin(elasticAgentManagedYaml)
 	if err != nil {
 		return errors.Wrap(err, "can't install Elastic-Agent in Kubernetes cluster")
 	}
 	return nil
 }
+
+// downloadElasticAgentManagedYAML will download a url from a path and return the response body.
+func downloadElasticAgentManagedYAML(url string) ([]byte, error) {
+	// Get the data
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get file from URL %s", url)
+	}
+	defer resp.Body.Close()
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read response body")
+	}
+	return b, nil
+}
+
+// getElasticAgentYAML retrieves elastic-agent-managed.yaml from upstream and modifies the file as needed
+// to run locally.
+func getElasticAgentYAML() ([]byte, error) {
+	appConfig, err := install.Configuration()
+	if err != nil {
+		return nil, errors.Wrap(err, "can't read application configuration")
+	}
+
+	logger.Debugf("downloading elastic-agent-managed-kubernetes.yaml from %s", elasticAgentManagedYamlURL)
+	elasticAgentManagedYaml, err := downloadElasticAgentManagedYAML(elasticAgentManagedYamlURL)
+	if err != nil {
+		return nil, errors.Wrapf(err, "downloading failed for file from source  %s", elasticAgentManagedYamlURL)
+	}
+
+	// Set regex to match fleet url from yaml file
+	fleetURLRegex := regexp.MustCompile("http(s){0,1}:\\/\\/fleet-server:(\\d+)")
+	// Replace fleet url
+	elasticAgentManagedYaml = fleetURLRegex.ReplaceAll(elasticAgentManagedYaml, []byte("http://fleet-server:8220"))
+
+	// Set regex to match image name from yaml file
+	imageRegex := regexp.MustCompile("docker.elastic.co/beats/elastic-agent:\\d.+")
+	// Replace image name
+	elasticAgentManagedYaml = imageRegex.ReplaceAll(elasticAgentManagedYaml, []byte(appConfig.DefaultStackImageRefs().ElasticAgent))
+
+	return elasticAgentManagedYaml, nil
+}

test/packages/kubernetes/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]

test/packages/kubernetes/data_stream/pod/fields/ecs.yml

@@ -7,3 +7,7 @@
 - name: service.type
   type: keyword
   description: Service type
+- name: orchestrator.cluster.name
+  external: ecs
+- name: orchestrator.cluster.url
+  external: ecs