Validate required fields (#291)

Author: jsoriano

code/go/internal/validator/semantic/types.go

@@ -5,9 +5,11 @@
 package semantic
 
 import (
+	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
@@ -16,6 +18,8 @@ import (
 	"github.com/elastic/package-spec/code/go/internal/fspath"
 )
 
+const dataStreamDir = "data_stream"
+
 type fields []field
 
 type field struct {
@@ -74,18 +78,13 @@ func validateNestedFields(parent string, fieldsFile string, fields fields, valid
 
 func listFieldsFiles(fsys fspath.FS) ([]string, error) {
 	var fieldsFiles []string
-
-	dataStreamDir := "data_stream"
-	dataStreams, err := fs.ReadDir(fsys, dataStreamDir)
-	if errors.Is(err, os.ErrNotExist) {
-		return fieldsFiles, nil
-	}
+	dataStreams, err := listDataStreams(fsys)
 	if err != nil {
-		return nil, errors.Wrap(err, "can't list data streams directory")
+		return nil, err
 	}
 
 	for _, dataStream := range dataStreams {
-		fieldsDir := filepath.Join(dataStreamDir, dataStream.Name(), "fields")
+		fieldsDir := filepath.Join(dataStreamDir, dataStream, "fields")
 		fs, err := fs.ReadDir(fsys, fieldsDir)
 		if errors.Is(err, os.ErrNotExist) {
 			continue
@@ -115,3 +114,34 @@ func unmarshalFields(fsys fspath.FS, fieldsPath string) (fields, error) {
 	}
 	return f, nil
 }
+
+func dataStreamFromFieldsPath(pkgRoot, fieldsFile string) (string, error) {
+	dataStreamPath := filepath.Clean(filepath.Join(pkgRoot, "data_stream"))
+	relPath, err := filepath.Rel(dataStreamPath, filepath.Clean(fieldsFile))
+	if err != nil {
+		return "", fmt.Errorf("looking for fields file (%s) in data streams path (%s): %w", fieldsFile, dataStreamPath, err)
+	}
+
+	parts := strings.SplitN(relPath, string(filepath.Separator), 2)
+	if len(parts) != 2 {
+		return "", errors.Errorf("could not find data stream for fields file %s", fieldsFile)
+	}
+	dataStream := parts[0]
+	return dataStream, nil
+}
+
+func listDataStreams(fsys fspath.FS) ([]string, error) {
+	dataStreams, err := fs.ReadDir(fsys, dataStreamDir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "can't list data streams directory")
+	}
+
+	list := make([]string, len(dataStreams))
+	for i, dataStream := range dataStreams {
+		list[i] = dataStream.Name()
+	}
+	return list, nil
+}

code/go/internal/validator/semantic/validate_fields_limits.go

@@ -5,10 +5,6 @@
 package semantic
 
 import (
-	"fmt"
-	"path/filepath"
-	"strings"
-
 	"github.com/pkg/errors"
 
 	ve "github.com/elastic/package-spec/code/go/internal/errors"
@@ -52,18 +48,3 @@ func validateFieldsLimits(fsys fspath.FS, limit int) ve.ValidationErrors {
 	}
 	return errs
 }
-
-func dataStreamFromFieldsPath(pkgRoot, fieldsFile string) (string, error) {
-	dataStreamPath := filepath.Clean(filepath.Join(pkgRoot, "data_stream"))
-	relPath, err := filepath.Rel(dataStreamPath, filepath.Clean(fieldsFile))
-	if err != nil {
-		return "", fmt.Errorf("looking for fields file (%s) in data streams path (%s): %w", fieldsFile, dataStreamPath, err)
-	}
-
-	parts := strings.SplitN(relPath, string(filepath.Separator), 2)
-	if len(parts) != 2 {
-		return "", errors.Errorf("could not find data stream for fields file %s", fieldsFile)
-	}
-	dataStream := parts[0]
-	return dataStream, nil
-}

code/go/internal/validator/semantic/validate_required_fields.go

@@ -0,0 +1,69 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	ve "github.com/elastic/package-spec/code/go/internal/errors"
+	"github.com/elastic/package-spec/code/go/internal/fspath"
+	"github.com/pkg/errors"
+)
+
+// ValidateRequiredFields validates that required fields are present and have the expected
+// types.
+func ValidateRequiredFields(fsys fspath.FS) ve.ValidationErrors {
+	requiredFields := map[string]string{
+		"data_stream.type":      "constant_keyword",
+		"data_stream.dataset":   "constant_keyword",
+		"data_stream.namespace": "constant_keyword",
+		"@timestamp":            "date",
+	}
+
+	return validateRequiredFields(fsys, requiredFields)
+}
+
+func validateRequiredFields(fsys fspath.FS, requiredFields map[string]string) ve.ValidationErrors {
+	// map datastream -> field name -> found
+	foundFields := make(map[string]map[string]struct{})
+
+	dataStreams, err := listDataStreams(fsys)
+	if err != nil {
+		return ve.ValidationErrors{err}
+	}
+
+	checkField := func(fieldsFile string, f field) ve.ValidationErrors {
+		expectedType, found := requiredFields[f.Name]
+		if !found {
+			return nil
+		}
+
+		datastream, err := dataStreamFromFieldsPath(fsys.Path(), fieldsFile)
+		if err != nil {
+			return ve.ValidationErrors{err}
+		}
+
+		if _, ok := foundFields[datastream]; !ok {
+			foundFields[datastream] = make(map[string]struct{})
+		}
+
+		foundFields[datastream][f.Name] = struct{}{}
+		if f.Type != expectedType {
+			return ve.ValidationErrors{errors.Errorf("expected type %q for required field %q, found %q in %q", expectedType, f.Name, f.Type, fieldsFile)}
+		}
+
+		return nil
+	}
+	errs := validateFields(fsys, checkField)
+
+	for _, dataStream := range dataStreams {
+		dataStreamFields := foundFields[dataStream]
+		for requiredName, requiredType := range requiredFields {
+			if _, found := dataStreamFields[requiredName]; !found {
+				errs = append(errs, errors.Errorf("expected field %q with type %q not found in datastream %q", requiredName, requiredType, dataStream))
+			}
+		}
+	}
+
+	return errs
+}

code/go/internal/validator/spec.go

@@ -72,6 +72,7 @@ func (s Spec) ValidatePackage(pkg Package) ve.ValidationErrors {
 		semantic.ValidateFieldGroups,
 		semantic.ValidateFieldsLimits(rootSpec.Limits.FieldsPerDataStreamLimit),
 		semantic.ValidateDimensionFields,
+		semantic.ValidateRequiredFields,
 	}
 
 	return rules.validate(&pkg)

code/go/pkg/validator/testdata/limits/data_stream/foo/fields/base-fields.yml

@@ -1,24 +1,12 @@
-- name: source
-  title: Source
-  group: 2
-  type: group
-  fields:
-    - name: geo.city_name
-      level: core
-      type: keyword
-      description: City name.
-      ignore_above: 1024
-    - name: geo.location
-      level: core
-      type: geo_point
-      description: Longitude and latitude.
-    - name: geo.region_iso_code
-      level: core
-      type: keyword
-      description: Region ISO code.
-      ignore_above: 1024
-    - name: geo.region_name
-      level: core
-      type: keyword
-      description: Region name.
-      ignore_above: 1024
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

code/go/pkg/validator/validator_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/elastic/package-spec/code/go/internal/errors"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -201,6 +202,46 @@ func TestValidateBadKibanaIDs(t *testing.T) {
 	}
 }
 
+func TestValidateMissingReqiredFields(t *testing.T) {
+	tests := map[string][]string{
+		"good": {},
+		"missing_required_fields": {
+			`expected type "constant_keyword" for required field "data_stream.dataset", found "keyword" in "../../../../test/packages/missing_required_fields/data_stream/foo/fields/base-fields.yml"`,
+			`expected field "data_stream.type" with type "constant_keyword" not found in datastream "foo"`,
+		},
+	}
+
+	for pkgName, expectedErrors := range tests {
+		t.Run(pkgName, func(t *testing.T) {
+			pkgRootPath := filepath.Join("..", "..", "..", "..", "test", "packages", pkgName)
+
+			err := ValidateFromPath(pkgRootPath)
+			if len(expectedErrors) == 0 {
+				assert.NoError(t, err)
+				return
+			}
+			assert.Error(t, err)
+
+			errs, ok := err.(errors.ValidationErrors)
+			require.True(t, ok)
+			assert.Len(t, errs, len(expectedErrors))
+
+			for _, expectedError := range expectedErrors {
+				found := false
+				for _, foundError := range errs {
+					if foundError.Error() == expectedError {
+						found = true
+						break
+					}
+				}
+				if !found {
+					t.Errorf("expected error: %q", expectedError)
+				}
+			}
+		})
+	}
+}
+
 func TestValidateVersionIntegrity(t *testing.T) {
 	tests := map[string]string{
 		"inconsistent_version": "current manifest version doesn't have changelog entry",

test/packages/deploy_docker/data_stream/foo/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/deploy_terraform/data_stream/foo/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/good/data_stream/foo/fields/base-fields.yml

@@ -1,24 +1,12 @@
-- name: source
-  title: Source
-  group: 2
-  type: group
-  fields:
-    - name: geo.city_name
-      level: core
-      type: keyword
-      description: City name.
-      ignore_above: 1024
-    - name: geo.location
-      level: core
-      type: geo_point
-      description: Longitude and latitude.
-    - name: geo.region_iso_code
-      level: core
-      type: keyword
-      description: Region ISO code.
-      ignore_above: 1024
-    - name: geo.region_name
-      level: core
-      type: keyword
-      description: Region name.
-      ignore_above: 1024
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/good/data_stream/hidden_data_stream/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.