Revert: Get elastic-agent-managed-daemonset.yaml from upstream 7.x instead of using a local static file (#459)

Author: mtojek

internal/configuration/locations/locations.go

@@ -23,7 +23,8 @@ const (
 
 	fieldsCachedDir = "cache/fields"
 
-	terraformDeployerYmlFile = "terraform-deployer.yml"
+	kubernetesDeployerElasticAgentYmlFile = "elastic-agent.yml"
+	terraformDeployerYmlFile              = "terraform-deployer.yml"
 )
 
 var (
@@ -83,6 +84,11 @@ func (loc LocationManager) KubernetesDeployerDir() string {
 	return filepath.Join(loc.stackPath, kubernetesDeployerDir)
 }
 
+// KubernetesDeployerAgentYml returns the Kubernetes Deployer Elastic Agent yml
+func (loc LocationManager) KubernetesDeployerAgentYml() string {
+	return filepath.Join(loc.stackPath, kubernetesDeployerDir, kubernetesDeployerElasticAgentYmlFile)
+}
+
 // TerraformDeployerDir returns the Terraform Directory
 func (loc LocationManager) TerraformDeployerDir() string {
 	return filepath.Join(loc.stackPath, terraformDeployerDir)

internal/install/install.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -59,6 +60,11 @@ func EnsureInstalled() error {
 		return errors.Wrap(err, "writing stack resources failed")
 	}
 
+	err = writeKubernetesDeployerResources(elasticPackagePath)
+	if err != nil {
+		return errors.Wrap(err, "writing Kubernetes deployer resources failed")
+	}
+
 	err = writeTerraformDeployerResources(elasticPackagePath)
 	if err != nil {
 		return errors.Wrap(err, "writing Terraform deployer resources failed")
@@ -166,6 +172,26 @@ func writeStackResources(elasticPackagePath *locations.LocationManager) error {
 
 }
 
+func writeKubernetesDeployerResources(elasticPackagePath *locations.LocationManager) error {
+	err := os.MkdirAll(elasticPackagePath.KubernetesDeployerDir(), 0755)
+	if err != nil {
+		return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.KubernetesDeployerDir())
+	}
+
+	appConfig, err := Configuration()
+	if err != nil {
+		return errors.Wrap(err, "can't read application configuration")
+	}
+
+	err = writeStaticResource(err, elasticPackagePath.KubernetesDeployerAgentYml(),
+		strings.ReplaceAll(kubernetesDeployerElasticAgentYml, "{{ ELASTIC_AGENT_IMAGE_REF }}",
+			appConfig.DefaultStackImageRefs().ElasticAgent))
+	if err != nil {
+		return errors.Wrap(err, "writing static resource failed")
+	}
+	return nil
+}
+
 func writeTerraformDeployerResources(elasticPackagePath *locations.LocationManager) error {
 	terraformDeployer := elasticPackagePath.TerraformDeployerDir()
 	err := os.MkdirAll(terraformDeployer, 0755)

internal/kubectl/kubectl.go

@@ -49,21 +49,3 @@ func modifyKubernetesResources(action string, definitionPaths ...string) ([]byte
 	}
 	return output, nil
 }
-
-// applyKubernetesResourcesStdin applies a Kubernetes manifest provided as stdin.
-// It returns the resources created as output and an error
-func applyKubernetesResourcesStdin(input []byte) ([]byte, error) {
-	// create kubectl apply command
-	kubectlCmd := exec.Command("kubectl", "apply", "-f", "-", "-o", "yaml")
-	//Stdin of kubectl command is the manifest provided
-	kubectlCmd.Stdin = bytes.NewReader(input)
-	errOutput := new(bytes.Buffer)
-	kubectlCmd.Stderr = errOutput
-
-	logger.Debugf("run command: %s", kubectlCmd)
-	output, err := kubectlCmd.Output()
-	if err != nil {
-		return nil, errors.Wrapf(err, "kubectl apply failed (stderr=%q)", errOutput.String())
-	}
-	return output, nil
-}

internal/kubectl/kubectl_apply.go

@@ -84,22 +84,6 @@ func Apply(definitionPaths ...string) error {
 	return nil
 }
 
-// ApplyStdin function adds resources to the Kubernetes cluster based on provided stdin.
-func ApplyStdin(input []byte) error {
-	logger.Debugf("Apply Kubernetes stdin")
-	out, err := applyKubernetesResourcesStdin(input)
-	if err != nil {
-		return errors.Wrap(err, "can't modify Kubernetes resources (apply stdin)")
-	}
-
-	logger.Debugf("Handle \"apply\" command output")
-	err = handleApplyCommandOutput(out)
-	if err != nil {
-		return errors.Wrap(err, "can't handle command output")
-	}
-	return nil
-}
-
 func handleApplyCommandOutput(out []byte) error {
 	logger.Debugf("Extract resources from command output")
 	resources, err := extractResources(out)

internal/testrunner/runners/system/servicedeployer/kubernetes.go

@@ -5,23 +5,18 @@
 package servicedeployer
 
 import (
-	"io"
-	"net/http"
 	"os"
 	"path/filepath"
-	"regexp"
 	"strings"
 
 	"github.com/pkg/errors"
 
-	"github.com/elastic/elastic-package/internal/install"
+	"github.com/elastic/elastic-package/internal/configuration/locations"
 	"github.com/elastic/elastic-package/internal/kind"
 	"github.com/elastic/elastic-package/internal/kubectl"
 	"github.com/elastic/elastic-package/internal/logger"
 )
 
-const elasticAgentManagedYamlURL = "https://raw.githubusercontent.com/elastic/beats/7.x/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml"
-
 // KubernetesServiceDeployer is responsible for deploying resources in the Kubernetes cluster.
 type KubernetesServiceDeployer struct {
 	definitionsDir string
@@ -149,58 +144,14 @@ func findKubernetesDefinitions(definitionsDir string) ([]string, error) {
 func installElasticAgentInCluster() error {
 	logger.Debug("install Elastic Agent in the Kubernetes cluster")
 
-	elasticAgentManagedYaml, err := getElasticAgentYAML()
-	logger.Debugf("downloaded %d bytes", len(elasticAgentManagedYaml))
+	locationManager, err := locations.NewLocationManager()
 	if err != nil {
-		return errors.Wrap(err, "can't retrieve Kubernetes file for Elastic Agent")
+		return errors.Wrap(err, "can't locate Kubernetes file for Elastic Agent in ")
 	}
 
-	err = kubectl.ApplyStdin(elasticAgentManagedYaml)
+	err = kubectl.Apply(locationManager.KubernetesDeployerAgentYml())
 	if err != nil {
 		return errors.Wrap(err, "can't install Elastic-Agent in Kubernetes cluster")
 	}
 	return nil
 }
-
-// downloadElasticAgentManagedYAML will download a url from a path and return the response body.
-func downloadElasticAgentManagedYAML(url string) ([]byte, error) {
-	// Get the data
-	resp, err := http.Get(url)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to get file from URL %s", url)
-	}
-	defer resp.Body.Close()
-
-	b, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to read response body")
-	}
-	return b, nil
-}
-
-// getElasticAgentYAML retrieves elastic-agent-managed.yaml from upstream and modifies the file as needed
-// to run locally.
-func getElasticAgentYAML() ([]byte, error) {
-	appConfig, err := install.Configuration()
-	if err != nil {
-		return nil, errors.Wrap(err, "can't read application configuration")
-	}
-
-	logger.Debugf("downloading elastic-agent-managed-kubernetes.yaml from %s", elasticAgentManagedYamlURL)
-	elasticAgentManagedYaml, err := downloadElasticAgentManagedYAML(elasticAgentManagedYamlURL)
-	if err != nil {
-		return nil, errors.Wrapf(err, "downloading failed for file from source  %s", elasticAgentManagedYamlURL)
-	}
-
-	// Set regex to match fleet url from yaml file
-	fleetURLRegex := regexp.MustCompile("http(s){0,1}:\\/\\/fleet-server:(\\d+)")
-	// Replace fleet url
-	elasticAgentManagedYaml = fleetURLRegex.ReplaceAll(elasticAgentManagedYaml, []byte("http://fleet-server:8220"))
-
-	// Set regex to match image name from yaml file
-	imageRegex := regexp.MustCompile("docker.elastic.co/beats/elastic-agent:\\d.+")
-	// Replace image name
-	elasticAgentManagedYaml = imageRegex.ReplaceAll(elasticAgentManagedYaml, []byte(appConfig.DefaultStackImageRefs().ElasticAgent))
-
-	return elasticAgentManagedYaml, nil
-}

test/packages/kubernetes/_dev/build/build.yml

@@ -7,7 +7,3 @@
 - name: service.type
   type: keyword
   description: Service type
-- name: orchestrator.cluster.name
-  external: ecs
-- name: orchestrator.cluster.url
-  external: ecs