Skip to content

Commit 158fee7

Browse files
Do validation in parallel
1 parent 6e2b24b commit 158fee7

File tree

3 files changed

+95
-47
lines changed

3 files changed

+95
-47
lines changed

cmd/kubeapply/subcmd/validate.go

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ var validateCmd = &cobra.Command{
2323
type validateFlags struct {
2424
// Expand before validating.
2525
expand bool
26+
27+
// Expand before validating.
28+
quiet bool
2629
}
2730

2831
var validateFlagValues validateFlags
@@ -100,12 +103,16 @@ func execValidation(ctx context.Context, clusterConfig *config.ClusterConfig) er
100103
}
101104

102105
numInvalidResources := 0
106+
numValidResources := 0
107+
numSkippedResources := 0
103108

104109
for _, result := range results {
105110
switch result.Status {
106111
case validation.StatusValid:
107-
log.Infof("Resource %s in file %s OK", result.PrettyName(), result.Filename)
112+
numValidResources++
113+
log.Debugf("Resource %s in file %s OK", result.PrettyName(), result.Filename)
108114
case validation.StatusSkipped:
115+
numSkippedResources++
109116
log.Debugf("Resource %s in file %s was skipped", result.PrettyName(), result.Filename)
110117
case validation.StatusError:
111118
numInvalidResources++
@@ -126,11 +133,19 @@ func execValidation(ctx context.Context, clusterConfig *config.ClusterConfig) er
126133

127134
if numInvalidResources > 0 {
128135
return fmt.Errorf(
129-
"Validation failed for %d resources",
136+
"Validation failed for cluster %s; %d resources/files invalid, %d valid, %d skipped",
137+
clusterConfig.DescriptiveName(),
130138
numInvalidResources,
139+
numValidResources,
140+
numSkippedResources,
131141
)
132142
}
133143

134-
log.Infof("Validation of cluster %s passed", clusterConfig.DescriptiveName())
144+
log.Infof(
145+
"Validation of cluster %s passed; %d resources valid, %d skipped",
146+
clusterConfig.DescriptiveName(),
147+
numValidResources,
148+
numSkippedResources,
149+
)
135150
return nil
136151
}

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ require (
3939
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
4040
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
4141
gopkg.in/src-d/go-git.v4 v4.13.1
42-
gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19
42+
gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19 // indirect
4343
gopkg.in/yaml.v2 v2.3.0
4444
gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2 // indirect
4545
gopkg.in/zorkian/go-datadog-api.v2 v2.28.0

pkg/validation/validation.go

Lines changed: 76 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@ package validation
33
import (
44
"context"
55
"fmt"
6-
"os"
7-
"path/filepath"
8-
"strings"
6+
"sync"
97

8+
"github.com/yannh/kubeconform/pkg/resource"
109
"github.com/yannh/kubeconform/pkg/validator"
1110
)
1211

12+
const (
13+
numWorkers = 6
14+
)
15+
1316
// KubeValidator is a struct that validates the kube configs associated with a cluster.
1417
type KubeValidator struct {
1518
validatorObj validator.Validator
@@ -74,58 +77,88 @@ func (k *KubeValidator) RunSchemaValidation(
7477
ctx context.Context,
7578
path string,
7679
) ([]ValidationResult, error) {
77-
results := []ValidationResult{}
78-
79-
err := filepath.Walk(
80-
path,
81-
func(subPath string, info os.FileInfo, err error) error {
82-
if err != nil {
83-
return err
84-
}
85-
86-
if info.IsDir() || !strings.HasSuffix(subPath, ".yaml") {
87-
return nil
80+
kResults := []validator.Result{}
81+
resourcesChan, errChan := resource.FromFiles(ctx, []string{path}, nil)
82+
mut := sync.Mutex{}
83+
wg := sync.WaitGroup{}
84+
85+
// Based on implementation in
86+
// https://github.com/yannh/kubeconform/blob/master/cmd/kubeconform/main.go.
87+
for i := 0; i < numWorkers; i++ {
88+
wg.Add(1)
89+
go func() {
90+
for res := range resourcesChan {
91+
mut.Lock()
92+
kResults = append(kResults, k.validatorObj.ValidateResource(res))
93+
mut.Unlock()
8894
}
95+
wg.Done()
96+
}()
97+
}
8998

90-
file, err := os.Open(subPath)
91-
if err != nil {
92-
return err
99+
wg.Add(1)
100+
go func() {
101+
// Process errors while discovering resources
102+
for err := range errChan {
103+
if err == nil {
104+
continue
93105
}
94106

95-
for _, kResult := range k.validatorObj.ValidateWithContext(ctx, subPath, file) {
96-
if kResult.Status == validator.Empty {
97-
// Skip over empty results
98-
continue
99-
}
100-
101-
result := ValidationResult{
102-
Filename: kResult.Resource.Path,
103-
Status: kubeconformStatusToStatus(kResult.Status),
104-
}
107+
var kResult validator.Result
105108

106-
if kResult.Err != nil {
107-
result.Message = kResult.Err.Error()
109+
if err, ok := err.(resource.DiscoveryError); ok {
110+
kResult = validator.Result{
111+
Resource: resource.Resource{Path: err.Path},
112+
Err: err.Err,
113+
Status: validator.Error,
108114
}
109-
110-
sig, err := kResult.Resource.Signature()
111-
if err == nil && sig != nil {
112-
result.Kind = sig.Kind
113-
result.Name = sig.Name
114-
result.Namespace = sig.Namespace
115-
result.Version = sig.Version
115+
} else {
116+
kResult = validator.Result{
117+
Resource: resource.Resource{},
118+
Err: err,
119+
Status: validator.Error,
116120
}
117-
118-
results = append(results, result)
119121
}
122+
mut.Lock()
123+
kResults = append(kResults, kResult)
124+
mut.Unlock()
125+
}
126+
wg.Done()
127+
}()
128+
wg.Wait()
120129

121-
return nil
122-
},
123-
)
130+
results := []ValidationResult{}
131+
132+
for _, kResult := range kResults {
133+
if kResult.Status == validator.Empty {
134+
// Skip over empty results
135+
continue
136+
}
137+
138+
result := ValidationResult{
139+
Filename: kResult.Resource.Path,
140+
Status: kStatusToStatus(kResult.Status),
141+
}
142+
143+
if kResult.Err != nil {
144+
result.Message = kResult.Err.Error()
145+
}
146+
147+
sig, err := kResult.Resource.Signature()
148+
if err == nil && sig != nil {
149+
result.Kind = sig.Kind
150+
result.Name = sig.Name
151+
result.Namespace = sig.Namespace
152+
result.Version = sig.Version
153+
}
154+
155+
results = append(results, result)
156+
}
124157

125-
return results, err
158+
return results, nil
126159
}
127160

128-
func kubeconformStatusToStatus(kStatus validator.Status) Status {
161+
func kStatusToStatus(kStatus validator.Status) Status {
129162
switch kStatus {
130163
case validator.Valid:
131164
return StatusValid

0 commit comments

Comments
 (0)