Skip to content

Commit 862c2c3

Browse files
danlishkadanlishka
authored
example(policy): New json-field-validator policy example (#2328)
Signed-off-by: Daniel Liszka <[email protected]>
1 parent b00fbfc commit 862c2c3

File tree

13 files changed

+917
-0
lines changed

13 files changed

+917
-0
lines changed

‎docs/examples/policies/_testutils.sh‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
#!/bin/bash
2+
#
3+
# Chainloop Policy Test Utilities
4+
#
5+
# This file contains shared test framework functions for policy testing.
6+
# It's designed to be shared across all policy directories.
7+
#
8+
# Usage: source ../_testutils.sh
9+
#
10+
11+
set -e
12+
13+
# Unalias grep to use standard grep instead of rg
14+
unalias grep 2>/dev/null || true
15+
16+
# Colors for output
17+
RED='\033[0;31m'
18+
GREEN='\033[0;32m'
19+
YELLOW='\033[1;33m'
20+
BLUE='\033[0;34m'
21+
NC='\033[0m' # No Color
22+
23+
# Test counters
24+
TESTS_PASSED=0
25+
TESTS_FAILED=0
26+
27+
# Find chainloop binary - prefer PATH
28+
find_chainloop_binary() {
29+
if command -v chainloop &> /dev/null; then
30+
echo "chainloop"
31+
elif [ -f "../../../../app/cli/bin/chainloop" ]; then
32+
echo "../../../../app/cli/bin/chainloop"
33+
elif [ -f "../../../../bin/chainloop" ]; then
34+
echo "../../../../bin/chainloop"
35+
else
36+
echo -e "${RED}Error: chainloop binary not found${NC}" >&2
37+
echo "Please build the CLI first or ensure it's in your PATH" >&2
38+
exit 1
39+
fi
40+
}
41+
42+
# Initialize test framework
43+
init_tests() {
44+
CHAINLOOP_BIN=$(find_chainloop_binary)
45+
echo -e "${BLUE}Using Chainloop binary: ${CHAINLOOP_BIN}${NC}"
46+
echo ""
47+
48+
TESTS_PASSED=0
49+
TESTS_FAILED=0
50+
}
51+
52+
# Check if policy was actually executed (not skipped or ignored)
53+
is_policy_executed() {
54+
local output="$1"
55+
56+
# Check if policy was skipped
57+
if echo "$output" | command grep -q '"skipped": *true'; then
58+
return 1 # false - policy was skipped
59+
fi
60+
61+
# Check if policy was ignored
62+
if echo "$output" | command grep -q '"ignored": *true'; then
63+
return 1 # false - policy was ignored
64+
fi
65+
66+
return 0 # true - policy was executed
67+
}
68+
69+
# Check if violations exist in policy output
70+
has_violations() {
71+
local output="$1"
72+
73+
# Check if violations array is non-empty (not "violations": [])
74+
if echo "$output" | command grep -q '"violations": *\[\]'; then
75+
return 1 # false - empty violations array
76+
elif echo "$output" | command grep -q '"violations": *\['; then
77+
return 0 # true - has violations
78+
else
79+
return 1 # false - no violations field found
80+
fi
81+
}
82+
83+
# Test policy linting
84+
test_policy_lint() {
85+
local policy_file="$1"
86+
local test_name="${2:-Policy Lint Check}"
87+
88+
echo -e "${YELLOW}Testing: ${test_name}${NC}"
89+
echo "Command: $CHAINLOOP_BIN policy develop lint --policy $policy_file"
90+
91+
if output=$($CHAINLOOP_BIN policy develop lint --policy "$policy_file" 2>&1); then
92+
echo -e "${GREEN}✓ PASSED${NC}"
93+
((TESTS_PASSED++))
94+
else
95+
echo -e "${RED}✗ FAILED${NC}"
96+
echo "Output: $output"
97+
((TESTS_FAILED++))
98+
fi
99+
echo ""
100+
}
101+
102+
# Test policy evaluation
103+
test_policy_eval() {
104+
local test_name="$1"
105+
local expected_result="$2" # "pass" or "fail"
106+
shift 2
107+
local args="$@"
108+
109+
# Extract --kind parameter from args
110+
local material_kind="EVIDENCE" # Default fallback
111+
local remaining_args=""
112+
113+
while [[ $# -gt 0 ]]; do
114+
case $1 in
115+
--kind)
116+
material_kind="$2"
117+
shift 2
118+
;;
119+
*)
120+
remaining_args="$remaining_args $1"
121+
shift
122+
;;
123+
esac
124+
done
125+
126+
echo -e "${YELLOW}Testing: ${test_name}${NC}"
127+
echo "Command: $CHAINLOOP_BIN policy develop eval --policy policy.yaml --kind $material_kind $remaining_args"
128+
129+
# Execute the command
130+
if output=$($CHAINLOOP_BIN policy develop eval --policy policy.yaml --kind "$material_kind" $remaining_args 2>&1); then
131+
exit_code=0
132+
else
133+
exit_code=$?
134+
fi
135+
136+
# Determine actual result
137+
if [ $exit_code -eq 0 ]; then
138+
# First check if policy was actually executed
139+
if ! is_policy_executed "$output"; then
140+
actual_result="fail" # Policy was skipped or ignored
141+
if echo "$output" | command grep -q '"skipped": *true'; then
142+
skip_reason="Policy was skipped"
143+
elif echo "$output" | command grep -q '"ignored": *true'; then
144+
skip_reason="Policy was ignored (material type mismatch)"
145+
else
146+
skip_reason="Policy was not executed"
147+
fi
148+
elif has_violations "$output"; then
149+
actual_result="fail" # Has violations = test should fail
150+
else
151+
actual_result="pass" # No violations = test should pass
152+
fi
153+
else
154+
actual_result="fail" # Command failed
155+
fi
156+
157+
# Compare with expected result
158+
if [ "$actual_result" = "$expected_result" ]; then
159+
if [ "$expected_result" = "pass" ]; then
160+
echo -e "${GREEN}✓ PASSED${NC}"
161+
else
162+
echo -e "${GREEN}✓ FAILED (as expected)${NC}"
163+
fi
164+
((TESTS_PASSED++))
165+
else
166+
if [ "$expected_result" = "pass" ]; then
167+
echo -e "${RED}✗ FAILED (expected to pass but failed)${NC}"
168+
else
169+
echo -e "${RED}✗ PASSED (expected to fail but passed)${NC}"
170+
fi
171+
172+
# Show skip reason if policy wasn't executed
173+
if [ -n "${skip_reason:-}" ]; then
174+
echo "Reason: $skip_reason"
175+
fi
176+
echo "Output: $output"
177+
((TESTS_FAILED++))
178+
fi
179+
echo ""
180+
}
181+
182+
# Print test summary and exit with appropriate code
183+
test_summary() {
184+
echo -e "${BLUE}=== Test Results Summary ===${NC}"
185+
echo ""
186+
TOTAL_TESTS=$((TESTS_PASSED + TESTS_FAILED))
187+
echo -e "Total Tests: ${TOTAL_TESTS}"
188+
echo -e "${GREEN}Passed: ${TESTS_PASSED}${NC}"
189+
echo -e "${RED}Failed: ${TESTS_FAILED}${NC}"
190+
echo ""
191+
192+
if [ $TESTS_FAILED -eq 0 ]; then
193+
echo -e "${GREEN}🎉 All tests passed!${NC}"
194+
exit 0
195+
else
196+
echo -e "${RED}❌ Some tests failed. Please check the output above.${NC}"
197+
exit 1
198+
fi
199+
}
200+
201+
# Verify required files exist
202+
verify_files() {
203+
local files=("$@")
204+
for file in "${files[@]}"; do
205+
if [ ! -f "$file" ]; then
206+
echo -e "${RED}Error: $file not found${NC}" >&2
207+
exit 1
208+
fi
209+
done
210+
}
211+
212+
# Print a test section header
213+
test_section() {
214+
local section_name="$1"
215+
echo -e "${BLUE}=== ${section_name} ===${NC}"
216+
echo ""
217+
}

‎docs/examples/policies/json-field-validator/README.md‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
# JSON Field Validator
2+
3+
Validates specific fields in JSON files used as evidence in Chainloop workflows.
4+
5+
## What This Policy Does
6+
7+
This policy allows you to validate individual fields in JSON evidence files. It can:
8+
9+
- **Check field values** - Verify a field matches an expected value
10+
- **Pattern matching** - Validate field values against regex patterns
11+
- **Structure validation** - Ensure required fields exist
12+
- **Flexible comparison** - Supports various boolean formats (`true`, `True`, `1`)
13+
14+
## Policy Parameters
15+
16+
| Parameter | Description | Required | Example Values |
17+
|-----------|-------------|----------|----------------|
18+
| `required_field` | Dot-notation path to the field | ✅ Yes | `application.name`, `security.enabled` |
19+
| `expected_value` | Expected exact value | ❌ No | `web-service`, `production`, `true` |
20+
| `field_pattern` | Regex pattern to match | ❌ No | `^[0-9]+\.[0-9]+\.[0-9]+$` |
21+
22+
**Note**: Use either `expected_value` OR `field_pattern`, not both.
23+
24+
## Using in Workflow Contracts
25+
26+
Add this policy to your workflow contract:
27+
28+
```yaml
29+
apiVersion: workflowcontract.chainloop.dev/v1
30+
kind: WorkflowContract
31+
metadata:
32+
name: my-workflow
33+
spec:
34+
materials:
35+
- type: EVIDENCE
36+
name: app-config
37+
38+
policies:
39+
- ref: ./json-field-validator/policy.yaml
40+
with:
41+
required_field: application.environment
42+
expected_value: production
43+
```
44+
45+
### Multiple Field Validations
46+
47+
```yaml
48+
policies:
49+
# Validate environment
50+
- ref: ./json-field-validator/policy.yaml
51+
with:
52+
required_field: application.environment
53+
expected_value: production
54+
55+
# Validate version format
56+
- ref: ./json-field-validator/policy.yaml
57+
with:
58+
required_field: application.version
59+
field_pattern: "^[0-9]+\\.[0-9]+\\.[0-9]+$"
60+
61+
# Validate security is enabled
62+
- ref: ./json-field-validator/policy.yaml
63+
with:
64+
required_field: security.enabled
65+
expected_value: "true"
66+
```
67+
68+
## Development & Testing
69+
70+
### Lint the Policy
71+
```bash
72+
chainloop policy develop lint --policy policy.yaml --format
73+
```
74+
75+
### Manual Testing
76+
```bash
77+
# Test field value validation
78+
chainloop policy develop eval \
79+
--policy policy.yaml \
80+
--material testdata/config.json \
81+
--kind EVIDENCE \
82+
--input required_field=application.name \
83+
--input expected_value=web-service
84+
85+
# Test pattern validation
86+
chainloop policy develop eval \
87+
--policy policy.yaml \
88+
--material testdata/config.json \
89+
--kind EVIDENCE \
90+
--input required_field=application.version \
91+
--input field_pattern="^[0-9]+\\.[0-9]+\\.[0-9]+$"
92+
```
93+
94+
### Run All Tests
95+
```bash
96+
./test.sh
97+
```
98+
99+
## Supported Field Paths
100+
101+
The policy supports dot notation for nested fields:
102+
103+
```json
104+
{
105+
"application": {
106+
"name": "web-service", // → application.name
107+
"version": "2.1.0", // → application.version
108+
"environment": "production" // → application.environment
109+
},
110+
"security": {
111+
"enabled": true, // → security.enabled
112+
"tls_version": "1.3" // → security.tls_version
113+
}
114+
}
115+
```
116+
117+
## Boolean Value Support
118+
119+
The policy accepts multiple boolean formats for `expected_value`:
120+
- `true` (JSON boolean as string)
121+
- `True` (capitalized)
122+
- `1` (numeric string)
123+
124+
All match against JSON `true` values.
125+
126+
## Test Framework
127+
128+
- `_testutils.sh` - Contains shared test logic and utilities
129+
- `test.sh` - Policy-specific test cases
130+
131+
## Sample Test Data
132+
133+
The `testdata/` directory contains:
134+
- `config.json` - Application configuration with nested structure
135+
- `compliance-checklist.json` - Different JSON structure for negative testing
136+
- `invalid.json` - Invalid JSON for error testing

0 commit comments

Comments
 (0)