template_path validation for streams and template policies

code/go/internal/validator/semantic/validate_data_stream_streams_template_path.go

@@ -0,0 +1,104 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"errors"
+	"io/fs"
+	"path"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
+)
+
+const (
+	defaultTemplatePath = "stream.yml.hbs"
+)
+
+var (
+	errFailedToReadManifest  = errors.New("failed to read manifest")
+	errFailedToParseManifest = errors.New("failed to parse manifest")
+	errTemplateNotFound      = errors.New("template file not found")
+)
+
+// ValidateStreamTemplates validates that all referenced template_path files exist for data streams
+func ValidateStreamTemplates(fsys fspath.FS) specerrors.ValidationErrors {
+	var errs specerrors.ValidationErrors
+
+	dataStreams, err := listDataStreams(fsys)
+	if err != nil {
+		return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)}
+	}
+
+	for _, dataStream := range dataStreams {
+		streamErrs := validateDataStreamManifestTemplates(fsys, dataStream)
+		errs = append(errs, streamErrs...)
+	}
+
+	return errs
+}
+
+func validateDataStreamManifestTemplates(fsys fspath.FS, dataStreamName string) specerrors.ValidationErrors {
+	var errs specerrors.ValidationErrors
+
+	manifestPath := path.Join("data_stream", dataStreamName, "manifest.yml")
+	data, err := fs.ReadFile(fsys, manifestPath)
+	if err != nil {
+		return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToReadManifest)}
+	}
+
+	var manifest struct {
+		Streams []struct {
+			Input        string `yaml:"input"`
+			TemplatePath string `yaml:"template_path"`
+		} `yaml:"streams"`
+	}
+
+	err = yaml.Unmarshal(data, &manifest)
+	if err != nil {
+		return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToParseManifest)}
+	}
+
+	for _, stream := range manifest.Streams {
+		streamPath := stream.TemplatePath
+		if stream.TemplatePath == "" {
+			// When no template_path is specified, it defaults to "stream.yml.hbs"
+			streamPath = defaultTemplatePath
+		}
+
+		// Walk through the "data_stream/<dataStreamName>/agent/stream" directory
+		// This mirrors the logic in fleet where the assets are filtered based on the template_path
+		// https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts#L3317
+		streamDir := path.Join("data_stream", dataStreamName, "agent", "stream")
+		found := false
+		fs.WalkDir(fsys, streamDir, func(filePath string, d fs.DirEntry, walkErr error) error {
+			if walkErr != nil {
+				return walkErr
+			}
+			if !d.IsDir() && path.Base(filePath) != "" && strings.HasSuffix(filePath, streamPath) {
+				found = true
+				return fs.SkipDir // Stop walking once found
+			}
+			return nil
+		})
+		if !found {
+			if stream.TemplatePath == "" {
+				errs = append(errs, specerrors.NewStructuredErrorf(
+					"file \"%s\" is invalid: stream \"%s\" is missing template file \"%s\": %w",
+					fsys.Path(manifestPath), stream.Input, streamPath, errTemplateNotFound))
+				continue
+			}
+			errs = append(errs, specerrors.NewStructuredErrorf(
+				"file \"%s\" is invalid: stream \"%s\" references template_path \"%s\": %w",
+				fsys.Path(manifestPath), stream.Input, streamPath, errTemplateNotFound))
+			continue
+		}
+	}
+
+	return errs
+}

code/go/internal/validator/semantic/validate_data_stream_streams_template_path_test.go

@@ -0,0 +1,153 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+)
+
+func TestValidateStreamTemplates(t *testing.T) {
+
+	t.Run("valid_data_stream_with_templates", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "test", "agent", "stream"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "test", "manifest.yml"), []byte(`
+streams:
+  - input: udp
+    template_path: udp.yml.hbs
+    title: Test UDP
+    description: Test UDP stream
+`), 0o644)
+		require.NoError(t, err)
+
+		err = os.WriteFile(filepath.Join(d, "data_stream", "test", "agent", "stream", "udp.yml.hbs"), []byte("# UDP template"), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateStreamTemplates(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+	})
+
+	t.Run("valid_data_stream_with_default_templates", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "test", "agent", "stream"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "test", "manifest.yml"), []byte(`
+streams:
+  - input: udp
+    title: Test UDP
+    description: Test UDP stream
+`), 0o644)
+		require.NoError(t, err)
+
+		err = os.WriteFile(filepath.Join(d, "data_stream", "test", "agent", "stream", "stream.yml.hbs"), []byte("# UDP template"), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateStreamTemplates(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+
+	})
+
+	t.Run("valid_data_stream_with_default_templates_endsWith_stream", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "test", "agent", "stream"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "test", "manifest.yml"), []byte(`
+streams:
+  - input: udp
+    title: Test UDP
+    description: Test UDP stream
+`), 0o644)
+		require.NoError(t, err)
+
+		err = os.WriteFile(filepath.Join(d, "data_stream", "test", "agent", "stream", "filestream.yml.hbs"), []byte("# UDP template"), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateStreamTemplates(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+
+	})
+
+	t.Run("err_read_manifest", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "test", "agent", "stream"), 0o755)
+		require.NoError(t, err)
+
+		errs := ValidateStreamTemplates(fspath.DirFS(d))
+		require.NotEmpty(t, errs, "expected validation errors")
+		assert.Len(t, errs, 1)
+		assert.ErrorIs(t, errs[0], errFailedToReadManifest)
+	})
+
+	t.Run("err_parse_manifest", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "test", "agent", "stream"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "test", "manifest.yml"), []byte(`
+streams:
+  - input: udp
+    template_path: udp.yml.hbs
+	title: Test UDP
+    description: Test UDP stream
+`), 0o644)
+		require.NoError(t, err)
+
+		err = os.WriteFile(filepath.Join(d, "data_stream", "test", "agent", "stream", "udp.yml.hbs"), []byte("# UDP template"), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateStreamTemplates(fspath.DirFS(d))
+		require.NotEmpty(t, errs, "expected validation errors")
+		assert.Len(t, errs, 1)
+		assert.ErrorIs(t, errs[0], errFailedToParseManifest)
+
+	})
+
+	t.Run("err_missing_template_file", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "test", "agent", "stream"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "test", "manifest.yml"), []byte(`
+streams:
+  - input: udp
+    template_path: missing.yml.hbs
+    title: Test UDP
+    description: Test UDP stream
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateStreamTemplates(fspath.DirFS(d))
+		require.NotEmpty(t, errs, "expected validation errors")
+		assert.Len(t, errs, 1)
+		assert.ErrorIs(t, errs[0], errTemplateNotFound)
+		assert.ErrorContains(t, errs[0], "stream \"udp\" references template_path \"missing.yml.hbs\": template file not found")
+	})
+
+	t.Run("err_missing_default_template_file", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "test", "agent", "stream"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "test", "manifest.yml"), []byte(`
+streams:
+  - input: udp
+    title: Test UDP
+    description: Test UDP stream
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateStreamTemplates(fspath.DirFS(d))
+		require.NotEmpty(t, errs, "expected validation errors")
+		assert.Len(t, errs, 1)
+		assert.ErrorIs(t, errs[0], errTemplateNotFound)
+		assert.ErrorContains(t, errs[0], "stream \"udp\" is missing template file \"stream.yml.hbs\": template file not found")
+	})
+
+}

code/go/internal/validator/semantic/validate_input_policy_template_path.go

@@ -0,0 +1,94 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
+)
+
+// ValidateInputPolicyTemplates validates that all referenced template_path files exist for integration and input policy templates
+func ValidateInputPolicyTemplates(fsys fspath.FS) specerrors.ValidationErrors {
+	var errs specerrors.ValidationErrors
+
+	manifestPath := "manifest.yml"
+
+	data, err := fs.ReadFile(fsys, manifestPath)
+	if err != nil {
+		return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: %ww", fsys.Path(manifestPath), errFailedToReadManifest)}
+	}
+
+	var manifest struct { // package manifest
+		Type string `yaml:"type"` // integration or input
+
+		PolicyTemplates []struct {
+			Name         string `yaml:"name"`
+			TemplatePath string `yaml:"template_path"` // input type packages require this field
+			Inputs       []struct {
+				Title        string `yaml:"title"`
+				TemplatePath string `yaml:"template_path"` // optional for integration packages
+			} `yaml:"inputs"`
+		} `yaml:"policy_templates"`
+	}
+
+	err = yaml.Unmarshal(data, &manifest)
+	if err != nil {
+		return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToParseManifest)}
+	}
+
+	for _, policyTemplate := range manifest.PolicyTemplates {
+		switch manifest.Type {
+		case "integration":
+			for _, input := range policyTemplate.Inputs {
+				if input.TemplatePath == "" {
+					continue // template_path is optional
+				}
+				err := validateAgentInputTemplatePath(fsys, input.TemplatePath)
+				if err != nil {
+					errs = append(errs, specerrors.NewStructuredErrorf(
+						"file \"%s\" is invalid: policy template \"%s\" references template_path \"%s\": %w",
+						fsys.Path(manifestPath), policyTemplate.Name, input.TemplatePath, err))
+				}
+			}
+
+		case "input":
+			if policyTemplate.TemplatePath == "" {
+				errs = append(errs, specerrors.NewStructuredErrorf(
+					"file \"%s\" is invalid: policy template \"%s\" is missing required field \"template_path\"",
+					fsys.Path(manifestPath), policyTemplate.Name))
+				continue
+			}
+			err := validateAgentInputTemplatePath(fsys, policyTemplate.TemplatePath)
+			if err != nil {
+				errs = append(errs, specerrors.NewStructuredErrorf(
+					"file \"%s\" is invalid: policy template \"%s\" references template_path \"%s\": %w",
+					fsys.Path(manifestPath), policyTemplate.Name, policyTemplate.TemplatePath, err))
+			}
+		}
+	}
+
+	return errs
+}
+
+func validateAgentInputTemplatePath(fsys fspath.FS, tmplPath string) error {
+	templatePath := path.Join("agent", "input", tmplPath)
+	_, err := fs.Stat(fsys, templatePath)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return errTemplateNotFound
+		}
+		return fmt.Errorf("failed to stat template file %s: %w", fsys.Path(templatePath), err)
+	}
+
+	return nil
+}