feat(policy-devel): extend eval command to show raw evaluation of messages (#2357)

Signed-off-by: Sylwester Piskozub <[email protected]>

Author: Piskoo

app/cli/cmd/output.go

@@ -55,7 +55,6 @@ type tabulatedData interface {
 		[]*action.APITokenItem |
 		*action.AttestationStatusMaterial |
 		*action.ListMembershipResult |
-		*action.PolicyEvalResult |
 		*action.PolicyLintResult
 }
 

app/cli/cmd/policy_develop_eval.go

@@ -32,6 +32,7 @@ func newPolicyDevelopEvalCmd() *cobra.Command {
 		policyPath       string
 		inputs           []string
 		allowedHostnames []string
+		debug            bool
 	)
 
 	cmd := &cobra.Command{
@@ -51,6 +52,7 @@ evaluates the policy against the provided material or attestation.`,
 				PolicyPath:       policyPath,
 				Inputs:           parseKeyValue(inputs),
 				AllowedHostnames: allowedHostnames,
+				Debug:            debug,
 			}
 
 			policyEval, err := action.NewPolicyEval(opts, actionOpts)
@@ -74,6 +76,7 @@ evaluates the policy against the provided material or attestation.`,
 	cmd.Flags().StringVarP(&policyPath, "policy", "p", "policy.yaml", "Path to custom policy file")
 	cmd.Flags().StringArrayVar(&inputs, "input", []string{}, "Key-value pairs of policy inputs (key=value)")
 	cmd.Flags().StringSliceVar(&allowedHostnames, "allowed-hostnames", []string{}, "Additional hostnames allowed for http.send requests in policies")
+	cmd.Flags().BoolVarP(&debug, "debug", "", false, "Include detailed evaluation inputs/outputs in JSON output and enable verbose logging")
 
 	return cmd
 }

app/cli/documentation/cli-reference.mdx

@@ -2847,6 +2847,7 @@ Options
 ```
 --allowed-hostnames strings   Additional hostnames allowed for http.send requests in policies
 --annotation strings          Key-value pairs of material annotations (key=value)
+--debug                       Include detailed evaluation inputs/outputs in JSON output and enable verbose logging
 -h, --help                        help for eval
 --input stringArray           Key-value pairs of policy inputs (key=value)
 --kind string                 Kind of the material: ["ARTIFACT" "ATTESTATION" "BLACKDUCK_SCA_JSON" "CHAINLOOP_RUNNER_CONTEXT" "CONTAINER_IMAGE" "CSAF_INFORMATIONAL_ADVISORY" "CSAF_SECURITY_ADVISORY" "CSAF_SECURITY_INCIDENT_RESPONSE" "CSAF_VEX" "EVIDENCE" "GHAS_CODE_SCAN" "GHAS_DEPENDENCY_SCAN" "GHAS_SECRET_SCAN" "GITLAB_SECURITY_REPORT" "HELM_CHART" "JACOCO_XML" "JUNIT_XML" "OPENVEX" "SARIF" "SBOM_CYCLONEDX_JSON" "SBOM_SPDX_JSON" "SLSA_PROVENANCE" "STRING" "TWISTCLI_SCAN_JSON" "ZAP_DAST_ZIP"]
@@ -2862,7 +2863,6 @@ Options inherited from parent commands
 -c, --config string             Path to an existing config file (default is $HOME/.config/chainloop/config.toml)
 --control-plane string      URL for the Control Plane API ($CHAINLOOP_CONTROL_PLANE_API) (default "api.cp.chainloop.dev:443")
 --control-plane-ca string   CUSTOM CA file for the Control Plane API (optional) ($CHAINLOOP_CONTROL_PLANE_API_CA)
---debug                     Enable debug/verbose logging mode
 -i, --insecure                  Skip TLS transport during connection to the control plane ($CHAINLOOP_API_INSECURE)
 -n, --org string                organization name
 -o, --output string             Output format, valid options are json and table (default "table")

app/cli/internal/action/policy_develop_eval.go

@@ -26,13 +26,7 @@ type PolicyEvalOpts struct {
 	PolicyPath       string
 	Inputs           map[string]string
 	AllowedHostnames []string
-}
-
-type PolicyEvalResult struct {
-	Violations  []string `json:"violations"`
-	SkipReasons []string `json:"skip_reasons"`
-	Skipped     bool     `json:"skipped"`
-	Ignored     bool     `json:"ignored,omitempty"`
+	Debug            bool
 }
 
 type PolicyEval struct {
@@ -47,14 +41,15 @@ func NewPolicyEval(opts *PolicyEvalOpts, actionOpts *ActionsOpts) (*PolicyEval,
 	}, nil
 }
 
-func (action *PolicyEval) Run() ([]*PolicyEvalResult, error) {
+func (action *PolicyEval) Run() (*policydevel.EvalSummary, error) {
 	evalOpts := &policydevel.EvalOptions{
 		PolicyPath:       action.opts.PolicyPath,
 		MaterialKind:     action.opts.Kind,
 		Annotations:      action.opts.Annotations,
 		MaterialPath:     action.opts.MaterialPath,
 		Inputs:           action.opts.Inputs,
 		AllowedHostnames: action.opts.AllowedHostnames,
+		Debug:            action.opts.Debug,
 	}
 
 	// Evaluate policy
@@ -63,15 +58,5 @@ func (action *PolicyEval) Run() ([]*PolicyEvalResult, error) {
 		return nil, err
 	}
 
-	results := make([]*PolicyEvalResult, 0, len(resp))
-	for _, r := range resp {
-		results = append(results, &PolicyEvalResult{
-			Violations:  r.Violations,
-			SkipReasons: r.SkipReasons,
-			Skipped:     r.Skipped,
-			Ignored:     r.Ignored,
-		})
-	}
-
-	return results, nil
+	return resp, nil
 }

app/cli/internal/policydevel/eval.go

@@ -16,6 +16,7 @@ package policydevel
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 
@@ -35,16 +36,26 @@ type EvalOptions struct {
 	MaterialPath     string
 	Inputs           map[string]string
 	AllowedHostnames []string
+	Debug            bool
 }
 
 type EvalResult struct {
-	Skipped     bool
-	SkipReasons []string
-	Violations  []string
-	Ignored     bool
+	Violations  []string `json:"violations"`
+	SkipReasons []string `json:"skip_reasons"`
+	Skipped     bool     `json:"skipped"`
 }
 
-func Evaluate(opts *EvalOptions, logger zerolog.Logger) ([]*EvalResult, error) {
+type EvalSummary struct {
+	Result    *EvalResult           `json:"result"`
+	DebugInfo *EvalSummaryDebugInfo `json:"debug_info,omitempty"`
+}
+
+type EvalSummaryDebugInfo struct {
+	Inputs     []json.RawMessage `json:"inputs"`
+	RawResults []json.RawMessage `json:"raw_results"`
+}
+
+func Evaluate(opts *EvalOptions, logger zerolog.Logger) (*EvalSummary, error) {
 	// 1. Create crafting schema
 	schema, err := createCraftingSchema(opts.PolicyPath, opts.Inputs)
 	if err != nil {
@@ -59,12 +70,12 @@ func Evaluate(opts *EvalOptions, logger zerolog.Logger) ([]*EvalResult, error) {
 	material.Annotations = opts.Annotations
 
 	// 3. Verify material against policy
-	result, err := verifyMaterial(schema, material, opts.MaterialPath, opts.AllowedHostnames, &logger)
+	summary, err := verifyMaterial(schema, material, opts.MaterialPath, opts.Debug, opts.AllowedHostnames, &logger)
 	if err != nil {
 		return nil, err
 	}
 
-	return result, nil
+	return summary, nil
 }
 
 func createCraftingSchema(policyPath string, inputs map[string]string) (*v1.CraftingSchema, error) {
@@ -82,41 +93,63 @@ func createCraftingSchema(policyPath string, inputs map[string]string) (*v1.Craf
 	}, nil
 }
 
-func verifyMaterial(schema *v1.CraftingSchema, material *v12.Attestation_Material, materialPath string, allowedHostnames []string, logger *zerolog.Logger) ([]*EvalResult, error) {
+func verifyMaterial(schema *v1.CraftingSchema, material *v12.Attestation_Material, materialPath string, debug bool, allowedHostnames []string, logger *zerolog.Logger) (*EvalSummary, error) {
 	var opts []policies.PolicyVerifierOption
 	if len(allowedHostnames) > 0 {
 		opts = append(opts, policies.WithAllowedHostnames(allowedHostnames...))
 	}
+
+	opts = append(opts, policies.WithIncludeRawData(debug))
+
 	v := policies.NewPolicyVerifier(schema, nil, logger, opts...)
 	policyEvs, err := v.VerifyMaterial(context.Background(), material, materialPath)
 	if err != nil {
 		return nil, err
 	}
 
-	// no evaluations were returned
-	if len(policyEvs) == 0 {
+	if len(policyEvs) == 0 || policyEvs[0] == nil {
 		return nil, fmt.Errorf("no execution branch matched for kind %s", material.MaterialType.String())
 	}
 
-	results := make([]*EvalResult, 0, len(policyEvs))
-	for _, policyEv := range policyEvs {
-		result := &EvalResult{
+	// Only one evaluation expected for a single policy attachment
+	policyEv := policyEvs[0]
+
+	summary := &EvalSummary{
+		Result: &EvalResult{
 			Skipped:     policyEv.GetSkipped(),
 			SkipReasons: policyEv.SkipReasons,
-			Ignored:     false,
-		}
+			Violations:  make([]string, 0, len(policyEv.Violations)),
+		},
+	}
 
-		// Collect all violation messages
-		violations := make([]string, 0, len(policyEv.Violations))
-		for _, v := range policyEv.Violations {
-			violations = append(violations, v.Message)
+	// Collect violation messages
+	for _, v := range policyEv.Violations {
+		summary.Result.Violations = append(summary.Result.Violations, v.Message)
+	}
+
+	// Include raw debug info if requested
+	if debug {
+		summary.DebugInfo = &EvalSummaryDebugInfo{
+			Inputs:     []json.RawMessage{},
+			RawResults: []json.RawMessage{},
 		}
-		result.Violations = violations
 
-		results = append(results, result)
+		for _, rr := range policyEv.RawResults {
+			if rr == nil {
+				continue
+			}
+			// Take the first input found, as we only allow one material input
+			if len(summary.DebugInfo.Inputs) == 0 && rr.Input != nil {
+				summary.DebugInfo.Inputs = append(summary.DebugInfo.Inputs, json.RawMessage(rr.Input))
+			}
+			// Collect all output raw results
+			if rr.Output != nil {
+				summary.DebugInfo.RawResults = append(summary.DebugInfo.RawResults, json.RawMessage(rr.Output))
+			}
+		}
 	}
 
-	return results, nil
+	return summary, nil
 }
 
 func craftMaterial(materialPath, materialKind string, logger *zerolog.Logger) (*v12.Attestation_Material, error) {

app/cli/internal/policydevel/eval_test.go

@@ -55,14 +55,14 @@ func TestEvaluate(t *testing.T) {
 			Annotations:  map[string]string{"key": "value"},
 		}
 
-		results, err := Evaluate(opts, logger)
+		result, err := Evaluate(opts, logger)
 		require.NoError(t, err)
-		require.NotEmpty(t, results)
+		require.NotNil(t, result)
 
-		if len(results[0].Violations) == 0 {
+		if len(result.Result.Violations) == 0 {
 			t.Log("Policy evaluation passed (no violations)")
 		} else {
-			for _, violation := range results[0].Violations {
+			for _, violation := range result.Result.Violations {
 				t.Logf("Violation: %s", violation)
 			}
 		}
@@ -78,14 +78,14 @@ func TestEvaluate(t *testing.T) {
 			Annotations:  map[string]string{"key": "value"},
 		}
 
-		results, err := Evaluate(opts, logger)
+		result, err := Evaluate(opts, logger)
 		require.NoError(t, err)
-		require.NotEmpty(t, results)
+		require.NotNil(t, result)
 
-		if len(results[0].Violations) == 0 {
+		if len(result.Result.Violations) == 0 {
 			t.Log("Policy evaluation passed (no violations)")
 		} else {
-			for _, violation := range results[0].Violations {
+			for _, violation := range result.Result.Violations {
 				t.Logf("Violation: %s", violation)
 			}
 		}