Skip to content

Commit 4311d3d

Browse files
ceeliasceelias
authored
Merge pull request #740 from sassoftware/shellcheck-fluentbit-scripts
Shellcheck fluentbit scripts
2 parents 6839989 + 04aadd5 commit 4311d3d

9 files changed

+301
-283
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
env:
1212
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
1313
SHELLCHECK_OPTS: -e SC1004 # exclude some shellcheck warnings.
14-
SHFMT_OPTS: -s # arguments to shfmt.
14+
SHFMT_OPTS: -s -i 4 -bn -sr -ln bash # shfmt: simplify (s), 4-space indentation (-i 4), braces on same line (-bn), space after redirects (-sr), bash dialect (-ln bash)
1515
with:
1616
sh_checker_only_diff: true # only run against files that were updated
1717
sh_checker_comment: true # include results as github comment

‎.shellcheckrc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Disables the check for sourced files
2+
disable=SC1091
3+
# Disables the check for expanding an array, as it predictably gives first element
4+
disable=SC2128

‎bin/helm-include.sh‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ function helm2ReleaseCheck {
6161
log_error "A Helm 2.x release of [$release] already exists"
6262
log_error "Helm [$HELM_VER_FULL] cannot manage the Helm 2.x release of [$release]"
6363
exit 1
64-
fi
64+
fi
6565
fi
6666
}
6767

@@ -72,7 +72,7 @@ function helmRepoAdd {
7272
## If this is an air gap deployment, do nothing
7373
if [ "$AIRGAP_DEPLOYMENT" == "true" ]; then
7474
return 0
75-
fi
75+
fi
7676

7777
HELM_FORCE_REPO_UPDATE=${HELM_FORCE_REPO_UPDATE:-true}
7878
if [[ ! $(helm repo list 2>/dev/null) =~ $repo[[:space:]] ]]; then
@@ -128,7 +128,8 @@ function get_helm_versionstring {
128128
if [ "$V4M_HELM_USE_LATEST" == "true" ]; then
129129
: # return null string
130130
else
131-
echo "--version $1"
131+
# Explicitly use printf instead of echo to avoid quote issues
132+
printf -- "--version %s" "$1"
132133
fi
133134

134135
return

‎logging/bin/deploy_fluentbit_azmonitor.sh‎

Lines changed: 111 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -1,199 +1,206 @@
1-
#! /bin/bash
1+
#!/bin/bash
22

33
# Copyright © 2020, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
44
# SPDX-License-Identifier: Apache-2.0
55

6-
cd "$(dirname $BASH_SOURCE)/../.."
6+
cd "$(dirname "$BASH_SOURCE")/../.." || exit 1
77
source logging/bin/common.sh
88

9-
this_script=`basename "$0"`
9+
this_script=$(basename "$0")
1010

1111
log_debug "Script [$this_script] has started [$(date)]"
1212

1313
# Enable Fluent Bit?
1414
FLUENT_BIT_ENABLED=${FLUENT_BIT_ENABLED:-true}
1515

1616
if [ "$FLUENT_BIT_ENABLED" != "true" ]; then
17-
log_info "Environment variable [FLUENT_BIT_ENABLED] is not set to 'true'; existing WITHOUT deploying Fluent Bit"
18-
exit
17+
log_info "Environment variable [FLUENT_BIT_ENABLED] is not set to 'true'; existing WITHOUT deploying Fluent Bit"
18+
exit 0
1919
fi
2020

21-
2221
set -e
2322

2423
HELM_DEBUG="${HELM_DEBUG:-false}"
24+
helmDebug=""
2525
if [ "$HELM_DEBUG" == "true" ]; then
26-
helmDebug="--debug"
26+
helmDebug="--debug"
2727
fi
2828

29-
helm2ReleaseCheck fb-$LOG_NS
29+
helm2ReleaseCheck "fb-$LOG_NS"
3030

3131
helmRepoAdd fluent https://fluent.github.io/helm-charts
3232

3333
# Confirm namespace exists
34-
if [ "$(kubectl get ns $LOG_NS -o name 2>/dev/null)" == "" ]; then
35-
log_error "The specified namespace [$LOG_NS] does not exist."
36-
exit 1
34+
if [ "$(kubectl get ns "$LOG_NS" -o name 2> /dev/null)" == "" ]; then
35+
log_error "The specified namespace [$LOG_NS] does not exist."
36+
exit 1
3737
fi
3838

3939
log_info "Deploying Fluent Bit (Azure Monitor)"
4040

41-
#Generate yaml file with all container-related keys#Generate yaml file with all container-related keys
42-
generateImageKeysFile "$FB_FULL_IMAGE" "logging/fb/fb_container_image.template"
43-
#Copy imageKeysFile since next call will replace existing one
41+
# Generate yaml file with all container-related keys
42+
generateImageKeysFile "$FB_FULL_IMAGE" "logging/fb/fb_container_image.template"
43+
# Copy imageKeysFile since next call will replace existing one
44+
#shellcheck disable=SC2154
4445
cp "$imageKeysFile" "$TMP_DIR/fb_imagekeysfile.yaml"
4546

4647
generateImageKeysFile "$FB_INITCONTAINER_FULL_IMAGE" "logging/fb/fb_initcontainer_image.template" "" "true"
4748

4849
# Fluent Bit user customizations
4950
FB_AZMONITOR_USER_YAML="${FB_AZMONITOR_USER_YAML:-$USER_DIR/logging/user-values-fluent-bit-azmonitor.yaml}"
5051
if [ ! -f "$FB_AZMONITOR_USER_YAML" ]; then
51-
log_debug "[$FB_AZMONITOR_USER_YAML] not found. Using $TMP_DIR/empty.yaml"
52-
FB_AZMONITOR_USER_YAML=$TMP_DIR/empty.yaml
52+
log_debug "[$FB_AZMONITOR_USER_YAML] not found. Using $TMP_DIR/empty.yaml"
53+
FB_AZMONITOR_USER_YAML="$TMP_DIR/empty.yaml"
5354
fi
5455

5556
if [ -f "$USER_DIR/logging/fluent-bit_config.configmap_azmonitor.yaml" ]; then
56-
# use copy in USER_DIR
57-
FB_CONFIGMAP="$USER_DIR/logging/fluent-bit_config.configmap_azmonitor.yaml"
57+
# use copy in USER_DIR
58+
FB_CONFIGMAP="$USER_DIR/logging/fluent-bit_config.configmap_azmonitor.yaml"
5859
else
59-
# use copy in repo
60-
FB_CONFIGMAP="logging/fb/fluent-bit_config.configmap_azmonitor.yaml"
60+
# use copy in repo
61+
FB_CONFIGMAP="logging/fb/fluent-bit_config.configmap_azmonitor.yaml"
6162
fi
62-
log_info "Using FB ConfigMap:" $FB_CONFIGMAP
63-
63+
log_info "Using FB ConfigMap: $FB_CONFIGMAP"
6464

6565
# Check/Create Connection Info Secret
66-
if [ "$(kubectl -n $LOG_NS get secret connection-info-azmonitor -o name 2>/dev/null)" == "" ]; then
67-
68-
export AZMONITOR_CUSTOMER_ID="${AZMONITOR_CUSTOMER_ID:-NotProvided}"
69-
export AZMONITOR_SHARED_KEY="${AZMONITOR_SHARED_KEY:-NotProvided}"
70-
71-
if [ "$AZMONITOR_CUSTOMER_ID" != "NotProvided" ] && [ "$AZMONITOR_SHARED_KEY" != "NotProvided" ]; then
72-
log_info "Creating secret [connection-info-azmonitor] in [$LOG_NS] namespace to hold Azure connection information."
73-
kubectl -n "$LOG_NS" create secret generic connection-info-azmonitor --from-literal=customer_id="$AZMONITOR_CUSTOMER_ID" --from-literal=shared_key="$AZMONITOR_SHARED_KEY"
74-
else
75-
log_error "Unable to create secret [$LOG_NS/connection-info-azmonitor] because missing required information: [AZMONITOR_CUSTOMER_ID: $AZMONITOR_CUSTOMER_ID ; AZMONITOR_SHARED_KEY: $AZMONITOR_SHARED_KEY]."
76-
log_error "You must provide this information via environment variables or create the secret [connection-info-azmonitor] before running this script."
77-
exit 1
78-
fi
66+
if [ "$(kubectl -n "$LOG_NS" get secret connection-info-azmonitor -o name 2> /dev/null)" == "" ]; then
67+
68+
export AZMONITOR_CUSTOMER_ID="${AZMONITOR_CUSTOMER_ID:-NotProvided}"
69+
export AZMONITOR_SHARED_KEY="${AZMONITOR_SHARED_KEY:-NotProvided}"
70+
71+
if [ "$AZMONITOR_CUSTOMER_ID" != "NotProvided" ] && [ "$AZMONITOR_SHARED_KEY" != "NotProvided" ]; then
72+
log_info "Creating secret [connection-info-azmonitor] in [$LOG_NS] namespace to hold Azure connection information."
73+
kubectl -n "$LOG_NS" create secret generic connection-info-azmonitor --from-literal=customer_id="$AZMONITOR_CUSTOMER_ID" --from-literal=shared_key="$AZMONITOR_SHARED_KEY"
74+
else
75+
log_error "Unable to create secret [$LOG_NS/connection-info-azmonitor] because missing required information: [AZMONITOR_CUSTOMER_ID: $AZMONITOR_CUSTOMER_ID ; AZMONITOR_SHARED_KEY: $AZMONITOR_SHARED_KEY]."
76+
log_error "You must provide this information via environment variables or create the secret [connection-info-azmonitor] before running this script."
77+
exit 1
78+
fi
7979
else
80-
log_info "Obtaining connection information from existing secret [$LOG_NS/connection-info-azmonitor]"
81-
export AZMONITOR_CUSTOMER_ID=$(kubectl -n "$LOG_NS" get secret connection-info-azmonitor -o=jsonpath="{.data.customer_id}" |base64 --decode)
82-
export AZMONITOR_SHARED_KEY=$(kubectl -n "$LOG_NS" get secret connection-info-azmonitor -o=jsonpath="{.data.shared_key}" |base64 --decode)
80+
log_info "Obtaining connection information from existing secret [$LOG_NS/connection-info-azmonitor]"
81+
AZMONITOR_CUSTOMER_ID=$(kubectl -n "$LOG_NS" get secret connection-info-azmonitor -o=jsonpath="{.data.customer_id}" | base64 --decode)
82+
export AZMONITOR_CUSTOMER_ID
83+
AZMONITOR_SHARED_KEY=$(kubectl -n "$LOG_NS" get secret connection-info-azmonitor -o=jsonpath="{.data.shared_key}" | base64 --decode)
84+
export AZMONITOR_SHARED_KEY
8385
fi
8486

8587
# Check for an existing Helm release of stable/fluent-bit
86-
if helm3ReleaseExists fbaz $LOG_NS; then
87-
log_info "Removing an existing release of deprecated stable/fluent-bit Helm chart from from the [$LOG_NS] namespace [$(date)]"
88-
helm $helmDebug delete -n "$LOG_NS" fbaz
89-
90-
if [ $(kubectl get servicemonitors -A |grep fluent-bit-v2 -c) -ge 1 ]; then
91-
log_debug "Updated serviceMonitor [fluent-bit-v2] appears to be deployed."
92-
elif [ $(kubectl get servicemonitors -A |grep fluent-bit -c) -ge 1 ]; then
93-
log_warn "You appear to have an obsolete service monitor in place for monitoring Fluent Bit."
94-
log_warn "Run monitoring/bin/deploy_monitoring_cluster.sh to deploy the current set of service monitors."
95-
fi
88+
if helm3ReleaseExists fbaz "$LOG_NS"; then
89+
log_info "Removing an existing release of deprecated stable/fluent-bit Helm chart from from the [$LOG_NS] namespace [$(date)]"
90+
helm "$helmDebug" delete -n "$LOG_NS" fbaz
91+
92+
num_service_monitors_v2=$(kubectl get servicemonitors -A | grep -c fluent-bit-v2 || true)
93+
if [ "$num_service_monitors_v2" -ge 1 ]; then
94+
log_debug "Updated serviceMonitor [fluent-bit-v2] appears to be deployed."
95+
else
96+
num_service_monitors=$(kubectl get servicemonitors -A | grep -c fluent-bit || true)
97+
if [ "$num_service_monitors" -ge 1 ]; then
98+
log_warn "You appear to have an obsolete service monitor in place for monitoring Fluent Bit."
99+
log_warn "Run monitoring/bin/deploy_monitoring_cluster.sh to deploy the current set of service monitors."
100+
fi
101+
fi
96102
else
97-
log_debug "No existing release of the deprecated stable/fluent-bit Helm chart was found"
103+
log_debug "No existing release of the deprecated stable/fluent-bit Helm chart was found"
98104
fi
99105

100106
# Multiline parser setup
101107
LOG_MULTILINE_ENABLED="${LOG_MULTILINE_ENABLED:-true}"
102108
if [ "$LOG_MULTILINE_ENABLED" == "true" ]; then
103-
LOG_MULTILINE_PARSER="docker, cri"
109+
LOG_MULTILINE_PARSER="docker, cri"
104110
else
105-
LOG_MULTILINE_PARSER=""
111+
LOG_MULTILINE_PARSER=""
106112
fi
107113

108114
# Create ConfigMap containing Fluent Bit configuration
109-
kubectl -n "$LOG_NS" apply -f $FB_CONFIGMAP
115+
kubectl -n "$LOG_NS" apply -f "$FB_CONFIGMAP"
110116

111117
# Create ConfigMap containing Viya-customized parsers (delete it first)
112118
kubectl -n "$LOG_NS" delete configmap fbaz-viya-parsers --ignore-not-found
113-
kubectl -n "$LOG_NS" create configmap fbaz-viya-parsers --from-file=logging/fb/viya-parsers.conf
119+
kubectl -n "$LOG_NS" create configmap fbaz-viya-parsers --from-file=logging/fb/viya-parsers.conf
114120

115121
TRACING_ENABLE="${TRACING_ENABLE:-false}"
122+
tracingValuesFile=""
116123
if [ "$TRACING_ENABLE" == "true" ]; then
117-
# Create ConfigMap containing tracing config
118-
kubectl -n "$LOG_NS" delete configmap fbaz-viya-tracing --ignore-not-found
119-
kubectl -n "$LOG_NS" create configmap fbaz-viya-tracing --from-file=logging/fb/viya-tracing.conf
124+
# Create ConfigMap containing tracing config
125+
kubectl -n "$LOG_NS" delete configmap fbaz-viya-tracing --ignore-not-found
126+
kubectl -n "$LOG_NS" create configmap fbaz-viya-tracing --from-file=logging/fb/viya-tracing.conf
120127

121-
tracingValuesFile="logging/fb/fluent-bit_helm_values_tracing.yaml"
122-
else
123-
# Create empty ConfigMap for tracing since it is expected to exist in main config
124-
kubectl -n "$LOG_NS" delete configmap fbaz-viya-tracing --ignore-not-found
125-
kubectl -n "$LOG_NS" create configmap fbaz-viya-tracing --from-file="$TMP_DIR"/empty.yaml
128+
tracingValuesFile="logging/fb/fluent-bit_helm_values_tracing.yaml"
129+
else
130+
# Create empty ConfigMap for tracing since it is expected to exist in main config
131+
kubectl -n "$LOG_NS" delete configmap fbaz-viya-tracing --ignore-not-found
132+
kubectl -n "$LOG_NS" create configmap fbaz-viya-tracing --from-file="$TMP_DIR"/empty.yaml
126133

127-
tracingValuesFile=$TMP_DIR/empty.yaml
134+
tracingValuesFile="$TMP_DIR/empty.yaml"
128135
fi
129136

130137
# Check for Kubernetes container runtime log format info
131-
KUBERNETES_RUNTIME_LOGFMT="${KUBERNETES_RUNTIME_LOGFMT}"
138+
KUBERNETES_RUNTIME_LOGFMT="${KUBERNETES_RUNTIME_LOGFMT:-}"
132139
if [ -z "$KUBERNETES_RUNTIME_LOGFMT" ]; then
133-
somenode=$(kubectl get nodes | awk 'NR==2 { print $1 }')
134-
runtime=$(kubectl get node $somenode -o jsonpath={.status.nodeInfo.containerRuntimeVersion} | awk -F: '{print $1}')
135-
log_debug "Kubernetes container runtime [$runtime] found on node [$somenode]"
136-
case $runtime in
140+
somenode=$(kubectl get nodes | awk 'NR==2 { print $1 }')
141+
runtime=$(kubectl get node "$somenode" -o "jsonpath={.status.nodeInfo.containerRuntimeVersion}" | awk -F: '{print $1}')
142+
log_debug "Kubernetes container runtime [$runtime] found on node [$somenode]"
143+
case $runtime in
137144
docker)
138-
KUBERNETES_RUNTIME_LOGFMT="docker"
139-
;;
140-
containerd|cri-o)
141-
KUBERNETES_RUNTIME_LOGFMT="criwithlog"
142-
;;
145+
KUBERNETES_RUNTIME_LOGFMT="docker"
146+
;;
147+
containerd | cri-o)
148+
KUBERNETES_RUNTIME_LOGFMT="criwithlog"
149+
;;
143150
*)
144-
log_warn "Unrecognized Kubernetes container runtime [$runtime]; using default parser"
145-
KUBERNETES_RUNTIME_LOGFMT="docker"
146-
;;
147-
esac
151+
log_warn "Unrecognized Kubernetes container runtime [$runtime]; using default parser"
152+
KUBERNETES_RUNTIME_LOGFMT="docker"
153+
;;
154+
esac
148155
fi
149156

150157
MON_NS="${MON_NS:-monitoring}"
151158

152159
# Create ConfigMap containing Kubernetes container runtime log format
153160
kubectl -n "$LOG_NS" delete configmap fbaz-env-vars --ignore-not-found
154161
kubectl -n "$LOG_NS" create configmap fbaz-env-vars \
155-
--from-literal=KUBERNETES_RUNTIME_LOGFMT=$KUBERNETES_RUNTIME_LOGFMT \
156-
--from-literal=LOG_MULTILINE_PARSER="${LOG_MULTILINE_PARSER}" \
157-
--from-literal=MON_NS="${MON_NS}"
162+
--from-literal=KUBERNETES_RUNTIME_LOGFMT="$KUBERNETES_RUNTIME_LOGFMT" \
163+
--from-literal=LOG_MULTILINE_PARSER="${LOG_MULTILINE_PARSER}" \
164+
--from-literal=MON_NS="${MON_NS}"
158165

159-
kubectl -n "$LOG_NS" label configmap fbaz-env-vars managed-by=v4m-es-script
166+
kubectl -n "$LOG_NS" label configmap fbaz-env-vars managed-by=v4m-es-script
160167

161168
# Check to see if we are upgrading from earlier version requiring root access
162-
if [ "$( kubectl -n $LOG_NS get configmap fbaz-dbmigrate-script -o name --ignore-not-found)" != "configmap/fbaz-dbmigrate-script" ]; then
163-
log_debug "Removing FB pods (if they exist) to allow migration."
164-
kubectl -n "$LOG_NS" delete daemonset v4m-fbaz --ignore-not-found
169+
if [ "$(kubectl -n "$LOG_NS" get configmap fbaz-dbmigrate-script -o name --ignore-not-found)" != "configmap/fbaz-dbmigrate-script" ]; then
170+
log_debug "Removing FB pods (if they exist) to allow migration."
171+
kubectl -n "$LOG_NS" delete daemonset v4m-fbaz --ignore-not-found
165172
fi
166173

167174
# Create ConfigMap containing Fluent Bit database migration script
168175
kubectl -n "$LOG_NS" delete configmap fbaz-dbmigrate-script --ignore-not-found
169-
kubectl -n "$LOG_NS" create configmap fbaz-dbmigrate-script --from-file logging/fb/migrate_fbstate_db.sh
170-
kubectl -n "$LOG_NS" label configmap fbaz-dbmigrate-script managed-by=v4m-es-script
176+
kubectl -n "$LOG_NS" create configmap fbaz-dbmigrate-script --from-file=logging/fb/migrate_fbstate_db.sh
177+
kubectl -n "$LOG_NS" label configmap fbaz-dbmigrate-script managed-by=v4m-es-script
171178

179+
## Get Helm Chart Name
180+
log_debug "Fluent Bit Helm Chart: repo [$FLUENTBIT_HELM_CHART_REPO] name [$FLUENTBIT_HELM_CHART_NAME] version [$FLUENTBIT_HELM_CHART_VERSION]"
181+
chart2install="$(get_helmchart_reference "$FLUENTBIT_HELM_CHART_REPO" "$FLUENTBIT_HELM_CHART_NAME" "$FLUENTBIT_HELM_CHART_VERSION")"
182+
versionstring="$(get_helm_versionstring "$FLUENTBIT_HELM_CHART_VERSION")"
172183

173-
## Get Helm Chart Name
174-
log_debug "Fluent Bit Helm Chart: repo [$FLUENTBIT_HELM_CHART_REPO] name [$FLUENTBIT_HELM_CHART_NAME] version [$FLUENTBIT_HELM_CHART_VERSION]"
175-
chart2install="$(get_helmchart_reference $FLUENTBIT_HELM_CHART_REPO $FLUENTBIT_HELM_CHART_NAME $FLUENTBIT_HELM_CHART_VERSION)"
176-
versionstring="$(get_helm_versionstring $FLUENTBIT_HELM_CHART_VERSION)"
177-
log_debug "Installing Helm chart from artifact [$chart2install]"
184+
log_debug "Installing Helm chart from artifact [$chart2install]"
178185

179186
# Deploy Fluent Bit via Helm chart
180-
helm $helmDebug upgrade --install v4m-fbaz --namespace $LOG_NS \
181-
$versionstring \
182-
--values $TMP_DIR/fb_imagekeysfile.yaml \
183-
--values $imageKeysFile \
184-
--values logging/fb/fluent-bit_helm_values_azmonitor.yaml \
185-
--values $FB_AZMONITOR_USER_YAML \
186-
--values $tracingValuesFile \
187-
--set fullnameOverride=v4m-fbaz \
188-
$chart2install
189-
190-
#pause to allow migration script to complete (if necessary)
187+
# shellcheck disable=SC2086
188+
helm $helmDebug upgrade --install v4m-fbaz --namespace "$LOG_NS" \
189+
$versionstring \
190+
--values "$TMP_DIR/fb_imagekeysfile.yaml" \
191+
--values "$imageKeysFile" \
192+
--values logging/fb/fluent-bit_helm_values_azmonitor.yaml \
193+
--values "$FB_AZMONITOR_USER_YAML" \
194+
--values "$tracingValuesFile" \
195+
--set fullnameOverride=v4m-fbaz \
196+
"$chart2install"
197+
# pause to allow migration script to complete (if necessary)
191198
sleep 20
192199

193-
#Container Security: Disable Token Automounting at ServiceAccount; enable for Pod
194-
disable_sa_token_automount $LOG_NS v4m-fbaz
200+
# Container Security: Disable Token Automounting at ServiceAccount; enable for Pod
201+
disable_sa_token_automount "$LOG_NS" v4m-fbaz
195202
# FB pods will restart after following call if automount is not already enabled
196-
enable_pod_token_automount $LOG_NS daemonset v4m-fbaz
203+
enable_pod_token_automount "$LOG_NS" daemonset v4m-fbaz
197204

198205
# Force restart of daemonset to ensure we pick up latest config changes
199206
# since Helm won't notice if the only changes are in the configMap

0 commit comments

Comments
 (0)