Skip to content

Commit bd5e6c9

Browse files
Merge pull request #37 from segmentio/yolken-improve-validation
Clean up validation, add OPA support
2 parents 7d226c3 + f34eda9 commit bd5e6c9

File tree

23 files changed

+1199
-174
lines changed

23 files changed

+1199
-174
lines changed

Dockerfile

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ RUN pip3 install awscli
3434

3535
COPY --from=builder \
3636
/usr/local/bin/helm \
37-
/usr/local/bin/kubeval \
3837
/usr/local/bin/kubectl \
3938
/usr/local/bin/kubeapply \
4039
/usr/local/bin/

Dockerfile.lambda

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ RUN pip3 install awscli
3434
COPY --from=builder \
3535
/usr/local/bin/aws-iam-authenticator \
3636
/usr/local/bin/helm \
37-
/usr/local/bin/kubeval \
3837
/usr/local/bin/kubectl \
3938
/usr/local/bin/kubeapply \
4039
/usr/local/bin/

README.md

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ environments. We welcome feedback and collaboration to make `kubeapply` useful t
5858

5959
- [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/): v1.16 or newer
6060
- [`helm`](https://helm.sh/docs/intro/install/): v3.5.0 or newer (only needed if using helm charts)
61-
- [`kubeval`](https://kubeval.instrumenta.dev/installation/): v0.15.0 or newer
6261

6362
Make sure that they're installed locally and available in your path.
6463

@@ -218,9 +217,14 @@ other source types use custom code in the `kubeapply` binary.
218217

219218
#### Validate
220219

221-
`kubeapply validate [path to cluster config]`
220+
`kubeapply validate [path to cluster config] --policy=[path to OPA policy in rego format]`
222221

223-
This validates all of the expanded configs for the cluster by wrapping `kubeval`.
222+
This validates all of the expanded configs for the cluster using the
223+
[`kubeconform`](https://github.com/yannh/kubeconform) library. It also, optionally, supports
224+
validating configs using one or more [OPA](https://www.openpolicyagent.org/) policies in
225+
rego format. The latter allows checking that configs satisfy organization-specific standards,
226+
e.g. that resource labels are in the correct format, that images are only pulled from the
227+
expected registries, etc.
224228

225229
#### Diff
226230

cmd/kubeapply/subcmd/check.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,6 @@ func checkRun(cmd *cobra.Command, args []string) error {
3030
if err := checkDep("kubectl", "version"); err != nil {
3131
return err
3232
}
33-
if err := checkDep("kubeval", "--version"); err != nil {
34-
return err
35-
}
3633

3734
return nil
3835
}

cmd/kubeapply/subcmd/validate.go

Lines changed: 104 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,21 @@ import (
1515

1616
var validateCmd = &cobra.Command{
1717
Use: "validate [cluster configs]",
18-
Short: "validate checks the cluster configs using kubeval",
18+
Short: "validate checks the cluster configs using kubeconform and (optionally) opa policies",
1919
Args: cobra.MinimumNArgs(1),
2020
RunE: validateRun,
2121
}
2222

2323
type validateFlags struct {
2424
// Expand before validating.
2525
expand bool
26+
27+
// Number of worker goroutines to use for validation.
28+
numWorkers int
29+
30+
// Paths to OPA policy rego files that will be run against kube resources.
31+
// See https://www.openpolicyagent.org/ for more details.
32+
policies []string
2633
}
2734

2835
var validateFlagValues validateFlags
@@ -34,6 +41,18 @@ func init() {
3441
false,
3542
"Expand before validating",
3643
)
44+
validateCmd.Flags().IntVar(
45+
&validateFlagValues.numWorkers,
46+
"num-workers",
47+
4,
48+
"Number of workers to use for validation",
49+
)
50+
validateCmd.Flags().StringArrayVar(
51+
&validateFlagValues.policies,
52+
"policy",
53+
[]string{},
54+
"Paths to OPA policies",
55+
)
3756

3857
RootCmd.AddCommand(validateCmd)
3958
}
@@ -88,43 +107,106 @@ func validateClusterPath(ctx context.Context, path string) error {
88107
func execValidation(ctx context.Context, clusterConfig *config.ClusterConfig) error {
89108
log.Infof("Validating cluster %s", clusterConfig.DescriptiveName())
90109

91-
kubeValidator := validation.NewKubeValidator()
110+
kubeconformChecker, err := validation.NewKubeconformChecker()
111+
if err != nil {
112+
return err
113+
}
92114

93-
log.Infof(
94-
"Checking that expanded configs for %s are valid YAML",
95-
clusterConfig.DescriptiveName(),
115+
policies, err := validation.DefaultPoliciesFromGlobs(
116+
ctx,
117+
validateFlagValues.policies,
118+
map[string]interface{}{
119+
// TODO: Add more parameters here (or entire config)?
120+
"cluster": clusterConfig.Cluster,
121+
"region": clusterConfig.Region,
122+
"env": clusterConfig.Env,
123+
},
96124
)
97-
err := kubeValidator.CheckYAML(clusterConfig.AbsSubpaths())
98125
if err != nil {
99126
return err
100127
}
101128

102-
log.Infof("Running kubeval on configs in %+v", clusterConfig.AbsSubpaths())
103-
results, err := kubeValidator.RunKubeval(ctx, clusterConfig.AbsSubpaths()[0])
129+
checkers := []validation.Checker{kubeconformChecker}
130+
for _, policy := range policies {
131+
checkers = append(checkers, policy)
132+
}
133+
134+
validator := validation.NewKubeValidator(
135+
validation.KubeValidatorConfig{
136+
NumWorkers: validateFlagValues.numWorkers,
137+
Checkers: checkers,
138+
},
139+
)
140+
141+
log.Infof("Running validator on configs in %+v", clusterConfig.AbsSubpaths())
142+
results, err := validator.RunChecks(ctx, clusterConfig.AbsSubpaths()[0])
104143
if err != nil {
105144
return err
106145
}
107146

108-
numInvalidFiles := 0
147+
numInvalidResourceChecks := 0
148+
numValidResourceChecks := 0
149+
numSkippedResourceChecks := 0
109150

110151
for _, result := range results {
111-
switch result.Status {
112-
case "valid":
113-
log.Infof("File %s OK", result.Filename)
114-
case "skipped":
115-
log.Debugf("File %s skipped", result.Filename)
116-
case "invalid":
117-
numInvalidFiles++
118-
log.Errorf("File %s is invalid; errors: %+v", result.Filename, result.Errors)
119-
default:
120-
log.Infof("Unrecognized result type: %+v", result)
152+
for _, checkResult := range result.CheckResults {
153+
switch checkResult.Status {
154+
case validation.StatusValid:
155+
numValidResourceChecks++
156+
log.Debugf(
157+
"Resource %s in file %s OK according to check %s",
158+
result.Resource.PrettyName(),
159+
result.Resource.Path,
160+
checkResult.CheckName,
161+
)
162+
case validation.StatusSkipped:
163+
numSkippedResourceChecks++
164+
log.Debugf(
165+
"Resource %s in file %s was skipped by check %s",
166+
result.Resource.PrettyName(),
167+
result.Resource.Path,
168+
checkResult.CheckName,
169+
)
170+
case validation.StatusError:
171+
numInvalidResourceChecks++
172+
log.Errorf(
173+
"Resource %s in file %s could not be processed by check %s: %s",
174+
result.Resource.PrettyName(),
175+
result.Resource.Path,
176+
checkResult.CheckName,
177+
checkResult.Message,
178+
)
179+
case validation.StatusInvalid:
180+
numInvalidResourceChecks++
181+
log.Errorf(
182+
"Resource %s in file %s is invalid according to check %s: %s",
183+
result.Resource.PrettyName(),
184+
result.Resource.Path,
185+
checkResult.CheckName,
186+
checkResult.Message,
187+
)
188+
case validation.StatusEmpty:
189+
default:
190+
log.Infof("Unrecognized result type: %+v", result)
191+
}
121192
}
122193
}
123194

124-
if numInvalidFiles > 0 {
125-
return fmt.Errorf("Validation failed for %d files", numInvalidFiles)
195+
if numInvalidResourceChecks > 0 {
196+
return fmt.Errorf(
197+
"Validation failed for %d resources in cluster %s (%d checks valid, %d skipped)",
198+
numInvalidResourceChecks,
199+
clusterConfig.DescriptiveName(),
200+
numValidResourceChecks,
201+
numSkippedResourceChecks,
202+
)
126203
}
127204

128-
log.Infof("Validation of cluster %s passed", clusterConfig.DescriptiveName())
205+
log.Infof(
206+
"Validation of cluster %s passed (%d checks valid, %d skipped)",
207+
clusterConfig.DescriptiveName(),
208+
numValidResourceChecks,
209+
numSkippedResourceChecks,
210+
)
129211
return nil
130212
}

examples/kubeapply-test-cluster/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ config or profile files.
5252

5353
##### (3) `make validate`
5454

55-
Runs `kubeval` over the expanded configs to validate that they are legitimate Kubernetes
55+
Runs `kubeconform` over the expanded configs to validate that they are legitimate Kubernetes
5656
configs before continuing.
5757

5858
##### (4) `make diff`

go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ require (
88
github.com/aws/aws-lambda-go v1.15.0
99
github.com/aws/aws-sdk-go v1.29.16
1010
github.com/briandowns/spinner v1.11.1
11-
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
1211
github.com/dgrijalva/jwt-go v3.2.0+incompatible
1312
github.com/fatih/color v1.7.0
1413
github.com/ghodss/yaml v1.0.0
@@ -22,6 +21,7 @@ require (
2221
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
2322
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
2423
github.com/olekukonko/tablewriter v0.0.4
24+
github.com/open-policy-agent/opa v0.27.1
2525
github.com/pmezard/go-difflib v1.0.0
2626
github.com/segmentio/conf v1.2.0
2727
github.com/segmentio/encoding v0.2.7
@@ -32,6 +32,7 @@ require (
3232
github.com/stretchr/testify v1.6.1
3333
github.com/stripe/skycfg v0.0.0-20200303020846-4f599970a3e6
3434
github.com/x-cray/logrus-prefixed-formatter v0.5.2
35+
github.com/yannh/kubeconform v0.4.6
3536
github.com/zorkian/go-datadog-api v2.28.0+incompatible // indirect
3637
go.starlark.net v0.0.0-20201204201740-42d4f566359b
3738
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d

0 commit comments

Comments
 (0)