diff --git a/Makefile b/Makefile index 0d3816f62..122c1ea47 100644 --- a/Makefile +++ b/Makefile @@ -184,7 +184,7 @@ ci: test lint-fix acceptance ## Run the usual required CI tasks LICENSE_IGNORE=\ -ignore 'dist/cli-reference/*.yaml' \ --ignore 'acceptance/examples/*.yaml' \ +-ignore 'acceptance/examples/**/*.yaml' \ -ignore 'configs/*/*.yaml' \ -ignore 'node_modules/**' \ -ignore 'hack/**/charts/**' \ diff --git a/acceptance/cli/cli.go b/acceptance/cli/cli.go index 143411a23..45780ba7a 100644 --- a/acceptance/cli/cli.go +++ b/acceptance/cli/cli.go @@ -764,7 +764,7 @@ func matchFileSnapshot(ctx context.Context, file string) error { return snaps.MatchSnapshot(ctx, file, string(content), status.vars) } -func createTrackBundleFile(ctx context.Context, name string, content *godog.DocString) (context.Context, error) { +func createGenericFile(ctx context.Context, name string, content *godog.DocString) (context.Context, error) { ctx, _, vars, err := variables(ctx) if err != nil { return ctx, err @@ -783,6 +783,10 @@ func createTrackBundleFile(ctx context.Context, name string, content *godog.DocS return ctx, os.WriteFile(file, []byte(data), 0o600) } +func createTrackBundleFile(ctx context.Context, name string, content *godog.DocString) (context.Context, error) { + return createGenericFile(ctx, name, content) +} + // AddStepsTo adds Gherkin steps to the godog ScenarioContext func AddStepsTo(sc *godog.ScenarioContext) { sc.Step(`^ec command is run with "(.+)"$`, ecCommandIsRunWith) @@ -793,6 +797,7 @@ func AddStepsTo(sc *godog.ScenarioContext) { sc.Step(`^the environment variable is set "([^"]*)"$`, theEnvironmentVarilableIsSet) sc.Step(`^the output should match the snapshot$`, matchSnapshot) sc.Step(`^the "([^"]*)" file should match the snapshot$`, matchFileSnapshot) + sc.Step(`^a file named "([^"]*)" containing$`, createGenericFile) sc.Step(`^a track bundle file named "([^"]*)" containing$`, createTrackBundleFile) sc.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) { logExecution(ctx) diff --git a/acceptance/examples/data-merges/data-1/data.yaml b/acceptance/examples/data-merges/data-1/data.yaml new file mode 100644 index 000000000..cd3486ca2 --- /dev/null +++ b/acceptance/examples/data-merges/data-1/data.yaml @@ -0,0 +1,6 @@ +--- +# Used in secenario "multiple data source top level key +# map merging" in features/validate_input.features +some_top_level_key: + john: "rhythm" + paul: "bass" diff --git a/acceptance/examples/data-merges/data-2/data.yaml b/acceptance/examples/data-merges/data-2/data.yaml new file mode 100644 index 000000000..03d1043bd --- /dev/null +++ b/acceptance/examples/data-merges/data-2/data.yaml @@ -0,0 +1,6 @@ +--- +# Used in secenario "multiple data source top level key +# map merging" in features/validate_input.features +some_top_level_key: + george: "lead" + ringo: "drums" diff --git a/acceptance/examples/data-merges/data-3/data.yaml b/acceptance/examples/data-merges/data-3/data.yaml new file mode 100644 index 000000000..871cd6d1b --- /dev/null +++ b/acceptance/examples/data-merges/data-3/data.yaml @@ -0,0 +1,8 @@ +--- +# Used in secenario "multiple data source top level key +# clash" in features/validate_input.features +# (We don't test this explicitly, but it would behave +# the same if it was a scalar value instead of a list.) +some_top_level_key: + - john + - paul diff --git a/acceptance/examples/data-merges/data-4/data.yaml b/acceptance/examples/data-merges/data-4/data.yaml new file mode 100644 index 000000000..af9c0afdc --- /dev/null +++ b/acceptance/examples/data-merges/data-4/data.yaml @@ -0,0 +1,8 @@ +--- +# Used in secenario "multiple data source top level key +# clash" in features/validate_input.features +# (We don't test this explicitly, but it would behave +# the same if it was a scalar value instead of a list.) +some_top_level_key: + - george + - ringo diff --git a/acceptance/examples/data-merges/policy/main.rego b/acceptance/examples/data-merges/policy/main.rego new file mode 100644 index 000000000..5672584ac --- /dev/null +++ b/acceptance/examples/data-merges/policy/main.rego @@ -0,0 +1,13 @@ +package main + +import rego.v1 + +# The acceptance test that uses this is about verifying the behavior +# when multiple data sources define the same top level data key. +# For this test we don't particularly care about the warning, but +# we're using the result msg to expose what the data looks like. +warn contains result if { + result := { + "msg": json.marshal(data.some_top_level_key), + } +} diff --git a/features/__snapshots__/validate_input.snap b/features/__snapshots__/validate_input.snap index 7f301a3e0..375cd4987 100755 --- a/features/__snapshots__/validate_input.snap +++ b/features/__snapshots__/validate_input.snap @@ -87,3 +87,38 @@ Error: error validating file pipeline_definition.yaml: evaluating policy: no reg Error: success criteria not met --- + +[multiple data source top level key map merging:stdout - 1] +ec-version: ${EC_VERSION} +effective-time: "${TIMESTAMP}" +filepaths: +- filepath: input.json + success: true + success-count: 0 + successes: null + violations: [] + warnings: + - msg: '{"george":"lead","john":"rhythm","paul":"bass","ringo":"drums"}' +policy: + sources: + - data: + - file::acceptance/examples/data-merges/data-1 + - file::acceptance/examples/data-merges/data-2 + policy: + - file::acceptance/examples/data-merges/policy +success: true + +--- + +[multiple data source top level key map merging:stderr - 1] + +--- + +[multiple data source top level key clash:stdout - 1] + +--- + +[multiple data source top level key clash:stderr - 1] +Error: error validating file input.json: evaluating policy: load: load documents: 1 error occurred during loading: ${TEMP}/ec-work-${RANDOM}/dat${RANDOM}/${RANDOM}/data.yaml: merge error + +--- diff --git a/features/validate_input.feature b/features/validate_input.feature index 71cf9c546..c4c5e93c1 100644 --- a/features/validate_input.feature +++ b/features/validate_input.feature @@ -66,3 +66,46 @@ Feature: validate input When ec command is run with "validate input --file input.yaml --policy git::https://${GITHOST}/git/multiple-sources-config.git" Then the exit status should be 1 Then the output should match the snapshot + + # In this example the same top level key is defined in + # two different data sources, but its value a map. + # In this situation a merge happens and we get second + # level keys from both sources. + Scenario: multiple data source top level key map merging + Given a file named "policy.yaml" containing + """ + sources: + - data: + - "file::acceptance/examples/data-merges/data-1" + - "file::acceptance/examples/data-merges/data-2" + policy: + - "file::acceptance/examples/data-merges/policy" + """ + Given a file named "input.json" containing + """ + {} + """ + When ec command is run with "validate input --file input.json --policy policy.yaml -o yaml" + Then the exit status should be 0 + Then the output should match the snapshot + + # In this example the same top level key is defined in + # two different data sources, but its value is not a map. + # In this situation ec throws a "merge error" error. + Scenario: multiple data source top level key clash + Given a file named "policy.yaml" containing + """ + sources: + - data: + - "file::acceptance/examples/data-merges/data-3" + - "file::acceptance/examples/data-merges/data-4" + policy: + - "file::acceptance/examples/data-merges/policy" + """ + Given a file named "input.json" containing + """ + {} + """ + When ec command is run with "validate input --file input.json --policy policy.yaml -o yaml" + Then the exit status should be 1 + Then the output should match the snapshot diff --git a/go.mod b/go.mod index ccefe2d05..3d4147706 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/gkampitakis/go-snaps v0.5.7 github.com/go-git/go-git/v5 v5.13.2 github.com/go-logr/logr v1.4.3 + github.com/go-openapi/strfmt v0.23.0 github.com/google/go-cmp v0.7.0 github.com/google/go-containerregistry v0.20.7 github.com/google/safearchive v0.0.0-20241025131057-f7ce9d7b6f9c @@ -31,6 +32,7 @@ require ( github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/secure-systems-lab/go-securesystemslib v0.9.0 github.com/sigstore/cosign/v2 v2.4.1 + github.com/sigstore/rekor v1.3.6 github.com/sigstore/sigstore v1.8.9 github.com/sirupsen/logrus v1.9.3 github.com/smarty/cproxy/v2 v2.1.1 @@ -190,7 +192,6 @@ require ( github.com/go-openapi/loads v0.22.0 // indirect github.com/go-openapi/runtime v0.28.0 // indirect github.com/go-openapi/spec v0.21.0 // indirect - github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-openapi/validate v0.24.0 // indirect github.com/go-viper/mapstructure/v2 v2.3.0 // indirect @@ -289,7 +290,6 @@ require ( github.com/shteou/go-ignore v0.3.1 // indirect github.com/sigstore/fulcio v1.6.3 // indirect github.com/sigstore/protobuf-specs v0.3.2 // indirect - github.com/sigstore/rekor v1.3.6 // indirect github.com/sigstore/timestamp-authority v1.2.2 // indirect github.com/skeema/knownhosts v1.3.0 // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect