EDOT and Slim docker images added (#7173) (#7466)

* EDOT and Slim docker images added

* lint

* fix arm

* Update pkg/testing/kubernetes/supported.go

Co-authored-by: Craig MacKenzie <[email protected]>

* edot separate dockerfile template

* verbose test

* verbose test

* fixed package-test conflict

* condition

* dockers len check

* name in packages.yml

* merged into same dockerfile again

* merged into same dockerfile again

* agent package version

* agent package version

* rename and a bit of refactor

* change images

* Revert "change images"

This reverts commit 4d641bb.

* revert helm changes

* helm

* feat: utilise edot-related images in otel k8s integration tests

* fix: download all docker artifacts in k8s integration steps

* Revert "helm"

This reverts commit eee5fe6.

* feat: introduce new k8s integration test for edot and edot-wolfi images

* fix: overwrite env var of values.yaml

* feat: add slim and slim-wolfi images in k8s integration tests

* fix: static image repository for upgrading managed agent helm test

* fix: static string value for OVERWRITE_ENV_VAR

* fix: apply docker image size checks only for built variants

---------

Co-authored-by: Craig MacKenzie <[email protected]>
Co-authored-by: Panos Koutsovasilis <[email protected]>
(cherry picked from commit 93a7d4c)

Co-authored-by: Michal Pristas <[email protected]>

Author: mergify[bot]

.buildkite/bk.integration.pipeline.yml

@@ -260,7 +260,7 @@ steps:
           DOCKER_VARIANTS: "{{matrix.variants}}"
           TARGET_ARCH: "amd64"
         command: |
-          buildkite-agent artifact download build/distributions/elastic-agent-*-linux-amd64.docker.tar.gz . --step 'packaging-containers-x86-64'
+          buildkite-agent artifact download build/distributions/*-linux-amd64.docker.tar.gz . --step 'packaging-containers-x86-64'
           .buildkite/scripts/steps/integration_tests_tf.sh kubernetes false
         artifact_paths:
           - build/**
@@ -276,7 +276,8 @@ steps:
         matrix:
           setup:
             variants:
-            - "basic,wolfi,complete,complete-wolfi,service,cloud"
+            - "basic,slim,complete,service,elastic-otel-collector"
+            - "wolfi,slim-wolfi,complete-wolfi,elastic-otel-collector-wolfi"
             version:
             - v1.27.16
             - v1.28.9

.buildkite/scripts/buildkite-k8s-integration-tests.sh

@@ -49,6 +49,10 @@ for variant in "${docker_variants[@]}"; do
   image_archive="elastic-agent-${variant}-${AGENT_VERSION}-linux-${TARGET_ARCH}.docker.tar.gz"
   if [[ "${variant}" == "basic" ]]; then
     image_archive="elastic-agent-${AGENT_VERSION}-linux-${TARGET_ARCH}.docker.tar.gz"
+  elif [[ "${variant}" == "elastic-otel-collector" ]]; then
+    image_archive="elastic-otel-collector-${AGENT_VERSION}-linux-${TARGET_ARCH}.docker.tar.gz"
+  elif [[ "${variant}" == "elastic-otel-collector-wolfi" ]]; then
+    image_archive="elastic-otel-collector-wolfi-${AGENT_VERSION}-linux-${TARGET_ARCH}.docker.tar.gz"
   fi
   image_archive_path="${DOCKER_IMAGE_ARCHIVES_DIR}/$image_archive"
 

dev-tools/mage/copy.go

@@ -13,9 +13,15 @@ import (
 	"regexp"
 )
 
+type SkipFn func(string) bool
+
 // Copy copies a file or a directory (recursively) and preserves the permissions.
 func Copy(src, dest string) error {
-	copy := &CopyTask{Source: src, Dest: dest}
+	return CopyWithCheck(src, dest, nil)
+}
+
+func CopyWithCheck(src, dest string, skipFn SkipFn) error {
+	copy := &CopyTask{Source: src, Dest: dest, skipFn: skipFn}
 	return copy.Execute()
 }
 
@@ -27,6 +33,7 @@ type CopyTask struct {
 	DirMode  os.FileMode      // Mode to use for copied dirs. Defaults to preserve permissions.
 	Exclude  []string         // Exclude paths that match these regular expressions.
 	excludes []*regexp.Regexp // Compiled exclude regexes.
+	skipFn   SkipFn           // external check, returns true if we skip
 }
 
 // Execute executes the copy and returns an error of there is a failure.
@@ -63,6 +70,15 @@ func (t *CopyTask) isExcluded(src string) bool {
 			return true
 		}
 	}
+
+	return false
+}
+
+func (t *CopyTask) isDestinationExcluded(dst string) bool {
+	if t.skipFn != nil {
+		return t.skipFn(dst)
+	}
+
 	return false
 }
 
@@ -78,6 +94,10 @@ func (t *CopyTask) fileCopy(src, dest string, entry fs.DirEntry) error {
 		return nil
 	}
 
+	if t.isDestinationExcluded(dest) {
+		return nil
+	}
+
 	srcFile, err := os.Open(src)
 	if err != nil {
 		return err

dev-tools/mage/dockerbuilder.go

@@ -18,6 +18,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/elastic/elastic-agent/internal/pkg/agent/install"
+	"github.com/elastic/elastic-agent/pkg/component"
+
 	"github.com/magefile/mage/sh"
 )
 
@@ -103,8 +106,35 @@ func (b *dockerBuilder) exposePorts() []string {
 
 func (b *dockerBuilder) copyFiles() error {
 	for _, f := range b.Files {
+		source := f.Source
+		var checkFn func(string) bool
 		target := filepath.Join(b.beatDir, f.Target)
-		if err := Copy(f.Source, target); err != nil {
+
+		if f.ExpandSpec {
+			specFilename := filepath.Base(source)
+			specContent, err := os.ReadFile(source)
+			if err != nil {
+				if os.IsNotExist(err) {
+					return nil
+				}
+				return fmt.Errorf("failed reading spec file for component %q: %w", specFilename, err)
+			}
+
+			// create filter
+			allowedPaths, err := component.ParseComponentFiles(specContent, specFilename, true)
+			if err != nil {
+				return fmt.Errorf("failed computing component files %q: %w", specFilename, err)
+			}
+			checkFn, err = install.SkipComponentsPathWithSubpathsFn(allowedPaths)
+			if err != nil {
+				return fmt.Errorf("failed compiling skip fn %q: %w", specFilename, err)
+			}
+
+			source = filepath.Dir(source) // change source to components directory
+			target = filepath.Dir(target) // target pointing to spec file
+		}
+
+		if err := CopyWithCheck(source, target, checkFn); err != nil {
 			if f.SkipOnMissing && errors.Is(err, os.ErrNotExist) {
 				continue
 			}

dev-tools/mage/dockervariants.go

@@ -10,14 +10,18 @@ import (
 )
 
 const (
-	undefined     = "undefined"
-	basic         = "basic"
-	ubi           = "ubi"
-	wolfi         = "wolfi"
-	complete      = "complete"
-	completeWolfi = "complete-wolfi"
-	cloud         = "cloud"
-	service       = "service"
+	undefined          = "undefined"
+	basic              = "basic"
+	ubi                = "ubi"
+	wolfi              = "wolfi"
+	complete           = "complete"
+	completeWolfi      = "complete-wolfi"
+	cloud              = "cloud"
+	service            = "service"
+	edotCollector      = "elastic-otel-collector"
+	slim               = "slim"
+	edotCollectorWolfi = "elastic-otel-collector-wolfi"
+	slimWolfi          = "slim-wolfi"
 )
 
 // DockerVariant defines the docker variant to build.
@@ -33,6 +37,10 @@ const (
 	Complete
 	Cloud
 	Service
+	EdotCollector
+	Slim
+	EdotCollectorWolfi
+	SlimWolfi
 )
 
 // String returns the name of the docker variant type.
@@ -54,6 +62,14 @@ func (typ DockerVariant) String() string {
 		return cloud
 	case Service:
 		return service
+	case EdotCollector:
+		return edotCollector
+	case Slim:
+		return slim
+	case EdotCollectorWolfi:
+		return edotCollectorWolfi
+	case SlimWolfi:
+		return slimWolfi
 	default:
 		return invalid
 	}
@@ -83,6 +99,14 @@ func (typ *DockerVariant) UnmarshalText(text []byte) error {
 		*typ = Cloud
 	case service:
 		*typ = Service
+	case edotCollector:
+		*typ = EdotCollector
+	case slim:
+		*typ = Slim
+	case edotCollectorWolfi:
+		*typ = EdotCollectorWolfi
+	case slimWolfi:
+		*typ = SlimWolfi
 	default:
 		return fmt.Errorf("unknown docker variant: %v", string(text))
 	}

dev-tools/mage/pkgtypes.go

@@ -115,6 +115,10 @@ type PackageSpec struct {
 	localPostRmScript      string
 }
 
+// add new prop into package file called expand spc
+// expand spec is checked during packaging and expands to multiple files
+// if expand is not present file is copied normally
+
 // PackageFile represents a file or directory within a package.
 type PackageFile struct {
 	Source        string                  `yaml:"source,omitempty"`   // Regular source file or directory.
@@ -129,6 +133,7 @@ type PackageFile struct {
 	Owner         string                  `yaml:"owner,omitempty"`           // File Owner, for user and group name (rpm only).
 	SkipOnMissing bool                    `yaml:"skip_on_missing,omitempty"` // Prevents build failure if the file is missing.
 	Symlink       bool                    `yaml:"symlink"`                   // Symlink marks file as a symlink pointing from target to source.
+	ExpandSpec    bool                    `yaml:"expand_spec,omitempty"`     // Optional
 }
 
 // OSArchNames defines the names of architectures for use in packages.
@@ -481,9 +486,13 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec {
 // ImageName computes the image name from the spec.
 func (s PackageSpec) ImageName() string {
 	if s.DockerVariant == Basic {
+		return s.Name
+	}
+	if s.DockerVariant == EdotCollector || s.DockerVariant == EdotCollectorWolfi {
 		// no suffix for basic docker variant
 		return s.Name
 	}
+
 	return fmt.Sprintf("%s-%s", s.Name, s.DockerVariant)
 }