Skip to content

Commit ec94b98

Browse files
thamudithamudi
authored
Add yaml checks for flux validation (#2)
* feat: add yaml checks for flux validation * feat: refactor validate flux script add a validation function add summary printout add flags to controll output * chore: general fixs: removed todo fixed issue with duplicate cases added exit error code 1 when error occurs * patch: remove output redirection * refactor: improve file validation logic and output formatting * fix: update help message for validation options
1 parent 4940af4 commit ec94b98

File tree

2 files changed

+335
-27
lines changed

2 files changed

+335
-27
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ Add this to your .pre-commit-config.yaml:
3131

3232
```yaml
3333
- repo: https://github.com/jordanopensource/pre-commit-hooks
34-
rev: v0.1.0 # Use the ref you want to point at
34+
rev: v0.1.1 # Use the ref you want to point at
3535
hooks:
3636
- id: validate-flux
3737
# - id: ...

‎scripts/validate-flux.sh‎

Lines changed: 334 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,300 @@
2424
# - kustomize v5.0
2525
# - kubeconform v0.6
2626

27+
# General variables
28+
numberOfFilesChanged=0
29+
numberOfClusterFilesChanged=0
30+
numberOfKsFilesChanged=0
31+
numberOfKsChecksFailed=0
32+
numberOfClustersChecksFailed=0
33+
DEBUG=0
34+
# Define the color codes
35+
GREEN='\033[0;32m'
36+
MAGENTA='\033[0;35m'
37+
CYAN='\033[0;36m'
38+
YELLOW='\033[0;33m'
39+
RED='\033[0;31m'
40+
NC='\033[0m'
41+
# Define text decorations
42+
UNDERLINE='\033[4m'
43+
NOUNDERLINE='\033[0m'
44+
45+
# Define the help message
46+
show_help() {
47+
echo "Usage: $(basename "$0") [options]"
48+
echo
49+
echo "Options:"
50+
echo " --help, -h Show this help message and exit"
51+
echo " --validate, -v Validate specified items (clstr=clusters, ks=kustomize) -- NOTE: YAML file check will always run"
52+
echo " --debug, -d Enable debug mode showing more information"
53+
}
54+
55+
# Check for --help or -h flag
56+
if [[ "$1" == "--help" || "$1" == "-h" ]]; then
57+
show_help
58+
exit 0
59+
fi
60+
61+
# Initialize an array to store validation options
62+
validate_options=()
63+
64+
# Process command line arguments
65+
while [[ "$1" != "" ]]; do
66+
case $1 in
67+
-v | --validate)
68+
shift
69+
while [[ "$1" != "" && "$1" != -* ]]; do
70+
case $1 in
71+
clusters | clstr)
72+
validate_options+=("clusters")
73+
;;
74+
kustomize | ks)
75+
validate_options+=("kustomize")
76+
;;
77+
*)
78+
echo "Unknown validation option: $1"
79+
show_help
80+
exit 1
81+
;;
82+
esac
83+
shift
84+
done
85+
validate_options+=("files") # always validate YAML files
86+
;;
87+
-d | --debug)
88+
DEBUG=1
89+
shift
90+
;;
91+
*)
92+
echo "Unknown option: $1"
93+
show_help
94+
exit 1
95+
;;
96+
esac
97+
done
98+
99+
## Function: check_package
100+
#
101+
### Description
102+
# The `check_package` function is a Bash script that checks if a specified command-line package is installed on the system. If the package is not installed, the function provides instructions on how to install it, depending on the package name.
103+
#
104+
### Parameters
105+
# - `$1` (String): The name of the package to check.
106+
#
107+
### Usage
108+
# ```bash
109+
# check_package <package_name>
110+
111+
check_package() {
112+
if ! command -v $1 &>/dev/null; then
113+
echo -e "${YELLOW}WARN${NC} - Package $1 is not installed."
114+
if [ "$1" == "yq" ]; then
115+
echo -e "You can download it from the GitHub page: ${CYAN}${UNDERLINE}https://github.com/mikefarah/yq?tab=readme-ov-file#install${NOUNDERLINE}${NC}"
116+
elif [ "$1" == "kustomize" ]; then
117+
echo -e "You can download it from the GitHub page: ${CYAN}${UNDERLINE}https://github.com/kubernetes-sigs/kustomize${NOUNDERLINE}${NC}"
118+
elif [ "$1" == "kubeconform" ]; then
119+
echo -e "You can download it from the GitHub page: ${CYAN}${UNDERLINE}https://github.com/yannh/kubeconform${NOUNDERLINE}${NC}"
120+
else
121+
echo -e "Please install ${YELLOW}${UNDERLINE}$1${NOUNDERLINE}${NC} and try again."
122+
fi
123+
exit 1
124+
fi
125+
}
126+
127+
## Function: get_git_diff_files
128+
#
129+
### Description
130+
# The `get_git_diff_files` function is a Bash script that retrieves the list of modified files in a Git repository, filtered by a specified directory and pattern. This function is particularly useful for identifying specific types of files that have been staged for commit.
131+
#
132+
### Parameters
133+
# - `dir` (String): The directory to filter the git diff output. This parameter limits the scope of the files to be checked within the specified directory.
134+
# - `pattern` (String): The pattern to filter the file names using grep. This parameter allows you to specify a regular expression to match certain file types or naming conventions.
135+
#
136+
### Usage
137+
# ```bash
138+
# get_git_diff_files <dir> <pattern>
139+
140+
get_git_diff_files() {
141+
local dir="$1"
142+
local pattern="$2"
143+
git diff --diff-filter=d --cached --name-only -- "$dir" | grep "$pattern"
144+
}
145+
146+
# Define the error handling function
147+
handle_error() {
148+
local message="$1"
149+
echo -e "\n${RED}ERROR${NC} - ${message}"
150+
}
151+
152+
## Function: download_schemas
153+
154+
### Description
155+
# The `download_schemas` function downloads the latest Flux OpenAPI schemas and extracts them to a specified directory. It also retrieves and stores the latest version tag of the schemas for future reference.
156+
#
157+
### Usage
158+
# ```bash
159+
# download_schemas
160+
161+
download_schemas() {
162+
[[ $DEBUG -eq 1 ]] && echo -e "${CYAN}INFO${NC} - Downloading Flux OpenAPI schemas"
163+
mkdir -p "$SCHEMA_DIR"
164+
curl -sL https://github.com/fluxcd/flux2/releases/latest/download/crd-schemas.tar.gz -o "$SCHEMA_TAR"
165+
tar zxf "$SCHEMA_TAR" -C "$SCHEMA_DIR"
166+
curl -sL $LATEST_VERSION_URL | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' >"$LATEST_VERSION_FILE"
167+
}
168+
169+
## Function: check_latest_schemas
170+
#
171+
### Description
172+
# The `check_latest_schemas` function checks if the downloaded Flux OpenAPI schemas are the latest available version. It compares the version tag of the currently downloaded schemas with the latest version available on GitHub.
173+
#
174+
### Usage
175+
# ```bash
176+
# check_latest_schemas
177+
178+
check_latest_schemas() {
179+
local latest_version
180+
latest_version=$(curl -sL $LATEST_VERSION_URL | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
181+
if [[ -f "$LATEST_VERSION_FILE" ]]; then
182+
local current_version
183+
current_version=$(cat "$LATEST_VERSION_FILE")
184+
if [[ "$latest_version" == "$current_version" ]]; then
185+
[[ $DEBUG -eq 1 ]] && echo -e "${CYAN}INFO${NC} - Flux OpenAPI schemas are already up-to-date."
186+
return 0
187+
fi
188+
fi
189+
return 1
190+
}
191+
192+
print_initialization() {
193+
echo "----------------------------------"
194+
if [ $DEBUG -eq 1 ]; then
195+
echo -e "# ${MAGENTA}Running validation checks in Debug mode${NC} "
196+
else
197+
echo -e "# ${MAGENTA}Running validation checks in Regular mode${NC} "
198+
fi
199+
echo -e "----------------------------------"
200+
}
201+
202+
print_summary() {
203+
204+
echo -e "----------------------------------"
205+
echo -e "# ${YELLOW}Validation Summary${NC} "
206+
echo -e "Total YAML files validated: ${CYAN}$numberOfFilesChanged${NC}"
207+
echo -e "Validated Clusters: ${CYAN}$numberOfClusterFilesChanged${NC}"
208+
echo -e "Validated Kustomizations: ${CYAN}$numberOfKsFilesChanged${NC}"
209+
if [[ ! $numberOfClustersChecksFailed -eq 0 || ! $numberOfKsChecksFailed -eq 0 ]]; then
210+
echo -e "-------------------------"
211+
echo -e "Failed Clusters validation checks: ${RED}$numberOfClustersChecksFailed${NC}"
212+
echo -e "Failed Kustomizations checks: ${RED}$numberOfKsChecksFailed${NC}"
213+
echo "----------------------------------"
214+
exit 1
215+
fi
216+
echo "----------------------------------"
217+
}
218+
219+
## Function: validate_files
220+
#
221+
### Description
222+
# The `validate_files` function is a Bash script that identifies and validates YAML files that have been modified and staged for commit in a Git repository.
223+
# It uses the `yq` tool to check the syntax of each file.
224+
#
225+
### Usage
226+
# ```bash
227+
# validate_files
228+
229+
validate_files() {
230+
mapfile -t fileList < <(get_git_diff_files "./" ".*\.ya\?ml$")
231+
if [ ${#fileList[@]} -eq 0 ]; then
232+
echo "No files to validate"
233+
exit 0
234+
fi
235+
echo -n "VALIDATING YAML FILES ..." # validates the yaml convention,
236+
[[ $DEBUG -eq 1 ]] && echo -e "\n----------------------------------\n${YELLOW}Files changed:${NC}"
237+
for file in "${fileList[@]}"; do
238+
[[ $DEBUG -eq 1 ]] && echo "$file"
239+
[[ $DEBUG -eq 1 ]] && echo -e "${CYAN}INFO${NC} - Validating $file"
240+
yq e 'true' "$file" >/dev/null
241+
((++numberOfFilesChanged))
242+
done
243+
echo -e "[ ${GREEN}Done${NC} ]"
244+
}
245+
246+
## Function: validate_clusters
247+
#
248+
### Description
249+
# The `validate_clusters` function is a Bash script that validates YAML files related to clusters that have been modified and staged for commit in a Git repository.
250+
# It uses the `kubeconform` tool to perform validation on these files.
251+
#
252+
### Usage
253+
# ```bash
254+
# validate_clusters
255+
256+
validate_clusters() {
257+
258+
echo -n "VALIDATING CLUSTERS ..."
259+
[[ $DEBUG -eq 1 ]] && echo -e "\n${CYAN}INFO${NC} - Validating clusters"
260+
mapfile -t clusterList < <(get_git_diff_files "./clusters" ".*\.ya\?ml$")
261+
for file in "${clusterList[@]}"; do
262+
local failed_checks
263+
[[ $DEBUG -eq 1 ]] && echo -e "${CYAN}INFO${NC} - Validating ${file}"
264+
kubeconform "${kubeconform_flags[@]}" "${kubeconform_config[@]}" "${file}" || ((failed_checks = 1))
265+
266+
if [[ ! $failed_checks -eq 0 ]]; then
267+
handle_error "kubeconform checks failed for: ${file}"
268+
((++numberOfClustersChecksFailed))
269+
else
270+
((++numberOfClusterFilesChanged))
271+
fi
272+
done
273+
if [[ $numberOfClustersChecksFailed -eq 0 ]]; then
274+
echo -e "[ ${GREEN}Done${NC} ]"
275+
else
276+
echo -e "[ ${RED}Failed${NC} ]"
277+
fi
278+
}
279+
280+
## Function: validate_kustomize
281+
#
282+
### Description
283+
# The `validate_kustomize` function is a Bash script that identifies and validates kustomize overlays that have been modified and staged for commit in a Git repository.
284+
# It uses the `kustomize` and `kubeconform` tools to perform validation on these overlays.
285+
#
286+
### Usage
287+
# ```bash
288+
# validate_kustomize
289+
290+
validate_kustomize() {
291+
echo -n "VALIDATING KUSTOMIZE ..."
292+
[[ $DEBUG -eq 1 ]] && echo -e "\n${CYAN}INFO${NC} - Validating kustomize overlays"
293+
mapfile -t kustomizeList < <(get_git_diff_files "./" "$kustomize_config")
294+
for file in "${kustomizeList[@]}"; do
295+
local failed_checks=0
296+
if [[ "$file" == *"$kustomize_config" ]]; then
297+
[[ $DEBUG -eq 1 ]] && echo -e "${CYAN}INFO${NC} - Validating kustomization ${file/%$kustomize_config/}"
298+
kustomize build "${file/%$kustomize_config/}" "${kustomize_flags[@]}" | kubeconform "${kubeconform_flags[@]}" "${kubeconform_config[@]}" || ((failed_checks = 1))
299+
300+
# if [[ ${PIPESTATUS[0]} != 0 ]]; then
301+
if [[ ! $failed_checks -eq 0 ]]; then
302+
handle_error "kustomize checks failed for: ${file}"
303+
((++numberOfKsChecksFailed))
304+
else
305+
((++numberOfKsFilesChanged))
306+
fi
307+
fi
308+
done
309+
310+
if [[ $numberOfKsChecksFailed -eq 0 ]]; then
311+
echo -e "[ ${GREEN}Done${NC} ]"
312+
else
313+
echo -e "[ ${RED}Failed${NC} ]"
314+
fi
315+
}
316+
317+
check_package "yq"
318+
check_package "kustomize"
319+
check_package "kubeconform"
320+
27321
set -o errexit
28322
set -o pipefail
29323

@@ -33,34 +327,48 @@ kustomize_config="kustomization.yaml"
33327

34328
# skip Kubernetes Secrets due to SOPS fields failing validation
35329
kubeconform_flags=("-skip=Secret")
36-
kubeconform_config=("-strict" "-ignore-missing-schemas" "-schema-location" "default" "-schema-location" "/tmp/flux-crd-schemas" "-verbose")
330+
kubeconform_config=("-strict" "-ignore-missing-schemas" "-schema-location" "default" "-schema-location" "/tmp/flux-crd-schemas")
331+
[[ $DEBUG -eq 1 ]] && kubeconform_config+=("-verbose")
37332

38-
echo "INFO - Downloading Flux OpenAPI schemas"
39-
mkdir -p /tmp/flux-crd-schemas/master-standalone-strict
40-
curl -sL https://github.com/fluxcd/flux2/releases/latest/download/crd-schemas.tar.gz | tar zxf - -C /tmp/flux-crd-schemas/master-standalone-strict
333+
# Define the directory and file paths
334+
SCHEMA_DIR="/tmp/flux-crd-schemas/master-standalone-strict"
335+
SCHEMA_TAR="$SCHEMA_DIR/crd-schemas.tar.gz"
336+
LATEST_VERSION_URL="https://api.github.com/repos/fluxcd/flux2/releases/latest"
337+
LATEST_VERSION_FILE="$SCHEMA_DIR/latest_version"
41338

42-
find . -type f -name '*.yaml' -print0 | while IFS= read -r -d $'\0' file;
43-
do
44-
echo "INFO - Validating $file"
45-
yq e 'true' "$file" > /dev/null
46-
done
339+
# Check if the directory exists and contains the schema files
340+
if [[ -d "$SCHEMA_DIR" && $(ls -A "$SCHEMA_DIR" 2>/dev/null) ]] && check_latest_schemas; then
341+
[[ $DEBUG -eq 1 ]] && echo -e "${CYAN}INFO${NC} - Flux OpenAPI schemas are already cached."
342+
else
343+
download_schemas
344+
fi
47345

48-
echo "INFO - Validating clusters"
49-
find ./clusters -maxdepth 2 -type f -name '*.yaml' -print0 | while IFS= read -r -d $'\0' file;
50-
do
51-
kubeconform "${kubeconform_flags[@]}" "${kubeconform_config[@]}" "${file}"
52-
if [[ ${PIPESTATUS[0]} != 0 ]]; then
53-
exit 1
54-
fi
55-
done
346+
print_initialization
56347

57-
echo "INFO - Validating kustomize overlays"
58-
find . -type f -name $kustomize_config -print0 | while IFS= read -r -d $'\0' file;
59-
do
60-
echo "INFO - Validating kustomization ${file/%$kustomize_config}"
61-
kustomize build "${file/%$kustomize_config}" "${kustomize_flags[@]}" | \
62-
kubeconform "${kubeconform_flags[@]}" "${kubeconform_config[@]}"
63-
if [[ ${PIPESTATUS[0]} != 0 ]]; then
348+
# Run validations based on user input
349+
if [[ ${#validate_options[@]} -eq 0 ]]; then
350+
validate_files
351+
validate_clusters
352+
validate_kustomize
353+
else
354+
for option in "${validate_options[@]}"; do
355+
case $option in
356+
files)
357+
validate_files
358+
;;
359+
clusters)
360+
validate_clusters
361+
;;
362+
kustomize)
363+
validate_kustomize
364+
;;
365+
*)
366+
echo "Unknown validation option: $option"
367+
show_help
64368
exit 1
65-
fi
66-
done
369+
;;
370+
esac
371+
done
372+
fi
373+
374+
print_summary

0 commit comments

Comments
 (0)