Enable developer extensions (#840)

* Add test package

* Define another type

* Use: build/packages

* Fix

* Changelog 0

* Fix

* Fix: duplicated fields

* Fix: package

* Fix: format

* Remove replace tag

* Add build.yml

* Visit external dependencies

* Fix: format

* WIP

* Refactor functions

* Fix: go.mod

* Fix: go.mod

* Comment

* Fix: debug

Author: mtojek

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/elastic/go-elasticsearch/v7 v7.17.1
 	github.com/elastic/go-licenser v0.4.0
 	github.com/elastic/go-ucfg v0.8.5
-	github.com/elastic/package-spec v1.10.0
+	github.com/elastic/package-spec v1.11.0
 	github.com/fatih/color v1.13.0
 	github.com/go-git/go-billy/v5 v5.3.1
 	github.com/go-git/go-git/v5 v5.4.2

go.sum

@@ -411,8 +411,8 @@ github.com/elastic/go-licenser v0.4.0 h1:jLq6A5SilDS/Iz1ABRkO6BHy91B9jBora8FwGRs
 github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU=
 github.com/elastic/go-ucfg v0.8.5 h1:4GB/rMpuh7qTcSFaxJUk97a/JyvFzhi6t+kaskTTLdM=
 github.com/elastic/go-ucfg v0.8.5/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA=
-github.com/elastic/package-spec v1.10.0 h1:fkCZRmxN4jesLuylGOEX5g31iITCXZFMbkgX6qvzkZI=
-github.com/elastic/package-spec v1.10.0/go.mod h1:KzGTSDqCkdhmL1IFpOH2ZQNSSE9JEhNtndxU3ZrQilA=
+github.com/elastic/package-spec v1.11.0 h1:atrhfGqCDVOnVO83Qh+doAjWnk/3Cs+d4C0U1YhvhgI=
+github.com/elastic/package-spec v1.11.0/go.mod h1:KzGTSDqCkdhmL1IFpOH2ZQNSSE9JEhNtndxU3ZrQilA=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=

internal/builder/external_fields.go

@@ -37,11 +37,18 @@ func resolveExternalFields(packageRoot, destinationDir string) error {
 		return errors.Wrap(err, "can't create field dependency manager")
 	}
 
-	fieldsFile, err := filepath.Glob(filepath.Join(destinationDir, "data_stream", "*", "fields", "*.yml"))
+	dataStreamFieldsFiles, err := filepath.Glob(filepath.Join(destinationDir, "data_stream", "*", "fields", "*.yml"))
 	if err != nil {
 		return err
 	}
-	for _, file := range fieldsFile {
+
+	packageFieldsFiles, err := filepath.Glob(filepath.Join(destinationDir, "fields", "*.yml"))
+	if err != nil {
+		return err
+	}
+
+	var fieldsFiles = append(packageFieldsFiles, dataStreamFieldsFiles...)
+	for _, file := range fieldsFiles {
 		data, err := os.ReadFile(file)
 		if err != nil {
 			return err

internal/docs/exported_fields.go

@@ -6,7 +6,6 @@ package docs
 
 import (
 	"fmt"
-	"path/filepath"
 	"sort"
 	"strings"
 
@@ -25,11 +24,10 @@ type fieldsTableRecord struct {
 
 var escaper = strings.NewReplacer("*", "\\*", "{", "\\{", "}", "\\}", "<", "\\<", ">", "\\>")
 
-func renderExportedFields(packageRoot, dataStreamName string) (string, error) {
-	dataStreamPath := filepath.Join(packageRoot, "data_stream", dataStreamName)
-	validator, err := fields.CreateValidatorForDataStream(dataStreamPath)
+func renderExportedFields(fieldsParentDir string) (string, error) {
+	validator, err := fields.CreateValidatorForDirectory(fieldsParentDir)
 	if err != nil {
-		return "", errors.Wrapf(err, "can't create fields validator instance (path: %s)", dataStreamPath)
+		return "", errors.Wrapf(err, "can't create fields validator instance (path: %s)", fieldsParentDir)
 	}
 
 	collected, err := collectFieldsFromDefinitions(validator)

internal/docs/readme.go

@@ -168,8 +168,12 @@ func renderReadme(fileName, packageRoot, templatePath string) ([]byte, error) {
 		"event": func(dataStreamName string) (string, error) {
 			return renderSampleEvent(packageRoot, dataStreamName)
 		},
-		"fields": func(dataStreamName string) (string, error) {
-			return renderExportedFields(packageRoot, dataStreamName)
+		"fields": func(args ...string) (string, error) {
+			if len(args) > 0 {
+				dataStreamPath := filepath.Join(packageRoot, "data_stream", args[0])
+				return renderExportedFields(dataStreamPath)
+			}
+			return renderExportedFields(packageRoot)
 		},
 	}).ParseFiles(templatePath)
 	if err != nil {

internal/fields/testdata/manifest.yml

@@ -0,0 +1,2 @@
+type: integration
+version: 0.0.1

internal/fields/validate.go

@@ -19,7 +19,9 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/elastic/elastic-package/internal/common"
+	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/multierror"
+	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/packages/buildmanifest"
 )
 
@@ -39,7 +41,7 @@ type Validator struct {
 	allowedCIDRs          []*net.IPNet
 }
 
-// ValidatorOption represents an optional flag that can be passed to  CreateValidatorForDataStream.
+// ValidatorOption represents an optional flag that can be passed to  CreateValidatorForDirectory.
 type ValidatorOption func(*Validator) error
 
 // WithDefaultNumericConversion configures the validator to accept defined keyword (or constant_keyword) fields as numeric-type.
@@ -78,8 +80,8 @@ func WithEnabledAllowedIPCheck() ValidatorOption {
 	}
 }
 
-// CreateValidatorForDataStream function creates a validator for the data stream.
-func CreateValidatorForDataStream(dataStreamRootPath string, opts ...ValidatorOption) (v *Validator, err error) {
+// CreateValidatorForDirectory function creates a validator for the directory.
+func CreateValidatorForDirectory(fieldsParentDir string, opts ...ValidatorOption) (v *Validator, err error) {
 	v = new(Validator)
 	for _, opt := range opts {
 		if err := opt(v); err != nil {
@@ -89,16 +91,28 @@ func CreateValidatorForDataStream(dataStreamRootPath string, opts ...ValidatorOp
 
 	v.allowedCIDRs = initializeAllowedCIDRsList()
 
-	v.Schema, err = loadFieldsForDataStream(dataStreamRootPath)
+	fieldsDir := filepath.Join(fieldsParentDir, "fields")
+	v.Schema, err = loadFieldsFromDir(fieldsDir)
 	if err != nil {
-		return nil, errors.Wrapf(err, "can't load fields for data stream (path: %s)", dataStreamRootPath)
+		return nil, errors.Wrapf(err, "can't load fields from directory (path: %s)", fieldsDir)
 	}
 
 	if v.disabledDependencyManagement {
 		return v, nil
 	}
 
-	packageRoot := filepath.Dir(filepath.Dir(dataStreamRootPath))
+	packageRoot, found, err := packages.FindPackageRoot()
+	if err != nil {
+		return nil, errors.Wrap(err, "can't find package root")
+	}
+	// As every command starts with approximating where is the package root, it isn't required to return an error in case the root is missing.
+	// This is also useful for testing purposes, where we don't have a real package, but just "fields" directory. The package root is always absent.
+	if !found {
+		logger.Debug("Package root not found, dependency management will be disabled.")
+		v.disabledDependencyManagement = true
+		return v, nil
+	}
+
 	bm, ok, err := buildmanifest.ReadBuildManifest(packageRoot)
 	if err != nil {
 		return nil, errors.Wrap(err, "can't read build manifest")
@@ -132,8 +146,7 @@ func initializeAllowedCIDRsList() (cidrs []*net.IPNet) {
 	return cidrs
 }
 
-func loadFieldsForDataStream(dataStreamRootPath string) ([]FieldDefinition, error) {
-	fieldsDir := filepath.Join(dataStreamRootPath, "fields")
+func loadFieldsFromDir(fieldsDir string) ([]FieldDefinition, error) {
 	files, err := filepath.Glob(filepath.Join(fieldsDir, "*.yml"))
 	if err != nil {
 		return nil, errors.Wrapf(err, "reading directory with fields failed (path: %s)", fieldsDir)

internal/fields/validate_test.go

@@ -18,7 +18,7 @@ type results struct {
 }
 
 func TestValidate_NoWildcardFields(t *testing.T) {
-	validator, err := CreateValidatorForDataStream("../../test/packages/parallel/aws/data_stream/elb_logs")
+	validator, err := CreateValidatorForDirectory("../../test/packages/parallel/aws/data_stream/elb_logs")
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
@@ -30,7 +30,7 @@ func TestValidate_NoWildcardFields(t *testing.T) {
 }
 
 func TestValidate_WithWildcardFields(t *testing.T) {
-	validator, err := CreateValidatorForDataStream("../../test/packages/parallel/aws/data_stream/sns")
+	validator, err := CreateValidatorForDirectory("../../test/packages/parallel/aws/data_stream/sns")
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
@@ -40,7 +40,7 @@ func TestValidate_WithWildcardFields(t *testing.T) {
 }
 
 func TestValidate_WithFlattenedFields(t *testing.T) {
-	validator, err := CreateValidatorForDataStream("testdata",
+	validator, err := CreateValidatorForDirectory("testdata",
 		WithDisabledDependencyManagement())
 	require.NoError(t, err)
 	require.NotNil(t, validator)
@@ -51,7 +51,7 @@ func TestValidate_WithFlattenedFields(t *testing.T) {
 }
 
 func TestValidate_WithNumericKeywordFields(t *testing.T) {
-	validator, err := CreateValidatorForDataStream("testdata",
+	validator, err := CreateValidatorForDirectory("testdata",
 		WithNumericKeywordFields([]string{"foo.code"}),
 		WithDisabledDependencyManagement())
 	require.NoError(t, err)
@@ -63,7 +63,7 @@ func TestValidate_WithNumericKeywordFields(t *testing.T) {
 }
 
 func TestValidate_constantKeyword(t *testing.T) {
-	validator, err := CreateValidatorForDataStream("testdata")
+	validator, err := CreateValidatorForDirectory("testdata")
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
@@ -77,7 +77,7 @@ func TestValidate_constantKeyword(t *testing.T) {
 }
 
 func TestValidate_ipAddress(t *testing.T) {
-	validator, err := CreateValidatorForDataStream("testdata", WithEnabledAllowedIPCheck())
+	validator, err := CreateValidatorForDirectory("testdata", WithEnabledAllowedIPCheck())
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
@@ -491,7 +491,7 @@ func readSampleEvent(t *testing.T, path string) json.RawMessage {
 }
 
 func TestValidate_geo_point(t *testing.T) {
-	validator, err := CreateValidatorForDataStream("../../test/packages/other/fields_tests/data_stream/first")
+	validator, err := CreateValidatorForDirectory("../../test/packages/other/fields_tests/data_stream/first")
 
 	require.NoError(t, err)
 	require.NotNil(t, validator)

internal/testrunner/runners/pipeline/runner.go

@@ -134,7 +134,7 @@ func (r *runner) run() ([]testrunner.TestResult, error) {
 		}
 
 		tr.TimeElapsed = time.Since(startTime)
-		fieldsValidator, err := fields.CreateValidatorForDataStream(dataStreamPath,
+		fieldsValidator, err := fields.CreateValidatorForDirectory(dataStreamPath,
 			fields.WithNumericKeywordFields(tc.config.NumericKeywordFields),
 			// explicitly enabled for pipeline tests only
 			// since system tests can have dynamic public IPs

internal/testrunner/runners/static/runner.go

@@ -89,7 +89,7 @@ func (r runner) verifySampleEvent() []testrunner.TestResult {
 		return results
 	}
 
-	fieldsValidator, err := fields.CreateValidatorForDataStream(
+	fieldsValidator, err := fields.CreateValidatorForDirectory(
 		dataStreamPath,
 		fields.WithDefaultNumericConversion())
 	if err != nil {