Skip to content

Commit f231c89

Browse files
authored
Merge pull request #30 from Yolean/y-assert-dogfooding
Essential ystack services' test coverage using y-assert
2 parents 0c554c1 + bf846d4 commit f231c89

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+735
-329
lines changed

.editorconfig

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
root = true
2+
3+
[*]
4+
charset = utf-8
5+
trim_trailing_whitespace = true
6+
indent_style = space
7+
indent_size = 2

README.md

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -97,11 +97,32 @@ docker volume rm ystack_admin 2> /dev/null || true
9797

9898
## Development
9999

100+
Using the [y-docker-compose](./bin/y-docker-compose) wrapper that extends [docker-compose.test.yml](./docker-compose.test.yml) that is used for CI with [docker-compose.dev-overrides.yml](./docker-compose.dev-overrides.yml). The k3s [image](./k3s/docker-image/) is the stock k3s image with y-stack's local registry config.
101+
102+
```
103+
y-docker-compose down
104+
y-docker-compose up --build -d master1
105+
y-docker-compose up --build -d ystack-proxy
106+
kubectl --kubeconfig=$YSTACK_HOME/devcluster/.kube/kubeconfig.yaml config rename-context default ystack-local
107+
y-kubie ctx -f $YSTACK_HOME/devcluster/.kube/kubeconfig.yaml
108+
```
109+
110+
To add monitoring support run `y-cluster-assert-install`.
111+
112+
For [dev loops](./examples/) and `y-assert` the docker stack replaces `y-kubefwd` (hard to use in CI)
113+
with container ports.
114+
You need `cat /etc/hosts | grep 127.0.0 | grep cluster.local` to have something like:
100115
```
101-
compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml'
102-
$compose down \
103-
;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true
104-
sudo rm test/.kube/kubeconfig.yaml
105-
$compose up --build -d ystack-proxy
106-
export KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml
116+
127.0.0.1 builds-registry.ystack.svc.cluster.local
117+
127.0.0.1 buildkitd.ystack.svc.cluster.local
118+
127.0.0.1 monitoring.ystack.svc.cluster.local
107119
```
120+
121+
Test using:
122+
```
123+
curl http://builds-registry.ystack.svc.cluster.local/v2/
124+
curl http://monitoring.ystack.svc.cluster.local:9090/api/v1/alertmanagers | jq '.data.activeAlertmanagers[0]'
125+
curl http://monitoring.ystack.svc.cluster.local:9093/api/v2/status
126+
```
127+
128+
Start a dev loop for actual asserts using `cd specs; y-skaffold --cache-artifacts=false dev` and start editing specs/*.spec.js. Run `y-assert` for CI-like runs until completion.

bin/.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,14 @@
22

33
# our executables are symlinked to real names
44
buildctl
5+
container-structure-test
6+
crane
57
helm
8+
kubie
9+
kustomize
610
minikube
711
mkcert
12+
promtool
813
rio
914
skaffold
1015
kubefwd

bin/y-assert

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ using ephemeral namespaces.
2222
# What cleanup solutions are there? Let's start with https://github.com/hjacobs/kube-janitor
2323
TTL="janitor/ttl: 23h"
2424

25+
[ -z "$MONITORING_HOST" ] && MONITORING_HOST=http://monitoring.ystack.svc.cluster.local
26+
27+
! curl -f -s --connect-timeout 3 $MONITORING_HOST:9090/ >/dev/null && echo "Failed to access the Prometheus endpoint" && exit 1
28+
! curl -f -s --connect-timeout 3 $MONITORING_HOST:9093/ >/dev/null && echo "Failed to access the Alertmanager endpoint" && exit 1
29+
2530
ctx=$1
2631
case $ctx in
2732
"--context="*) shift 1 ;;
@@ -32,6 +37,9 @@ case $ctx in
3237
esac
3338
skaffoldflags=$(echo $ctx | sed 's|^--|--kube-|')
3439

40+
[ -z "$PROFILES" ] && PROFILES="y-assert"
41+
skaffoldflags="$skaffoldflags -p $PROFILES"
42+
3543
CIRUN=$1
3644
[ -z "$CIRUN" ] && CIRUN="run"
3745

@@ -55,7 +63,21 @@ EOF
5563
fi
5664
skaffoldflags="$skaffoldflags -n $namespace"
5765

58-
set -x
66+
# At this point we should know how to identify relevant assert metrics
67+
[ -z "$assertlabels" ] && assertlabels="namespace=\"$namespace\"";
68+
5969
y-skaffold $skaffoldflags --cache-artifacts=false $CIRUN
6070

71+
echo "$(log) Prometheus queries for test results will use labels: $assertlabels"
6172
echo "$(log) cleanup-ish: kubectl $ctx get namespace -l $(echo $TTL | sed 's/: */=/') -o name | xargs kubectl $ctx delete"
73+
74+
query=assertions_failed{$assertlabels}
75+
76+
[ -z "$INTERVAL" ] && INTERVAL=5
77+
# This loop will continuously list errors and must fail on http errors or absence of the metric
78+
until curl -s --data "query=$query" $MONITORING_HOST:9090/api/v1/query \
79+
| jq -r '.data.result | if length > 0 then . else error("No results") end | .[] | . as {$metric, $values} | .value[1] | . as $vs | tonumber as $v | if $v > 0 then error($metric.namespace+" "+$metric.pod+" "+$vs) else empty end'
80+
do sleep $INTERVAL
81+
done
82+
83+
echo "Zero failures now with: curl -s --data query=$query $MONITORING_HOST:9090/api/v1/query"

bin/y-build-like-sync

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/usr/bin/env bash
2+
[ -z "$DEBUG" ] || set -x
3+
set -e
4+
5+
# Bash isn't a good tool for producing a tar from a build context
6+
# but this script is an experiment on how to compose an image
7+
# from a runtime and some static files using go-containerregistry's crane
8+
# The user shouldn't need to write Dockerfile or .dockerignore,
9+
# but we could probably use a generated Dockerfile + kaniko instead, or Buildpacks
10+
11+
# Settings
12+
DEFAULT_REGISTRY=builds-registry.ystack.svc.cluster.local
13+
[ -z "$BUILDS_REGISTRY" ] && BUILDS_REGISTRY=$DEFAULT_REGISTRY
14+
[ -z "$PUSH_REGISTRY" ] && PUSH_REGISTRY=$DEFAULT_REGISTRY
15+
16+
if [ "$(curl -s --connect-timeout 3 http://$BUILDS_REGISTRY/v2/)" != "{}" ]
17+
then
18+
echo "ERROR Skaffold need local access to the builds registry for digest lookup"
19+
echo "Registry: $BUILDS_REGISTRY"
20+
echo "Look for y-stack's ingress or port-forward utilities"
21+
exit 1
22+
fi
23+
24+
[ -z "$IMAGE" ] && echo "No IMAGE env (from for example Skaffold)" && exit 1
25+
26+
IMAGE=$IMAGE
27+
case "$IMAGE" in
28+
$BUILDS_REGISTRY/* ) ;;
29+
$PUSH_REGISTRY/* ) echo "Unlike y-build this script won't push to non-build registries" && exit 1 ;;
30+
* ) echo "Output is restricted to PUSH_REGISTRY=$PUSH_REGISTRY. Got: $IMAGE" && exit 1 ;;
31+
esac
32+
33+
RUNTIME_IMAGE=$1
34+
[ -z "$RUNTIME_IMAGE" ] && echo "First argument must be a runtime image to append the layer to" \
35+
&& echo "To improve build times use a runtime image in the target repo" && exit 1
36+
37+
# crane hangs for a long time if it doesn't know that the registry is plain http
38+
RUNTIME_IMAGE=$(echo $RUNTIME_IMAGE | sed 's|.local/|.local:80/|')
39+
IMAGE=$(echo $IMAGE | sed 's|.local/|.local:80/|')
40+
41+
# This is a PoC, let's make a lot of assumptions to simplify
42+
context=.
43+
src='**'
44+
# assuming a single manual sync, which is the reasonable use case for a runtime
45+
[ ! -f skaffold.yaml ] && echo "This composition example assumes a sync defined in a skaffold.yaml" && exit 1
46+
dest=$(cat skaffold.yaml | grep 'dest:' | awk '{ print $2 }')
47+
# this avoids "tar: Removing leading `/' from member names" and could come in handy if we can't use --transform
48+
dest=$(echo $dest | sed 's|^/||')
49+
50+
list=$(mktemp)
51+
(cd $context; git ls-files -c -o --exclude-standard -- . || find . -type f) > $list
52+
tar=$(mktemp)
53+
tar --transform "s|^|$dest/|" --show-transformed-names -cvhf $tar -T $list --mode='ug+rw' --group=65534 --owner=65532
54+
rm $list
55+
56+
set -x
57+
y-crane append --insecure -b $RUNTIME_IMAGE -f $tar -t $IMAGE
58+
set +x
59+
rm $tar

bin/y-build-nothing

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,20 +24,13 @@ case "$IMAGE" in
2424
* ) echo "Output is restricted to PUSH_REGISTRY=$PUSH_REGISTRY. Got: $IMAGE" && exit 1 ;;
2525
esac
2626

27-
[ -z "$ctx" ] && ctx="--context=$(kubectl config current-context)" && echo "No 'ctx' env, guessing: $ctx"
28-
29-
CRANE_IMAGE=gcr.io/go-containerregistry/crane:96cf69f03a3cf2d9c8850ce9d8e1f33da354826a@sha256:7b4f1fc7c681e49ce7bf6ed95bd30d6c749f4a1cd2467f994fc045e6433ebde9
3027
NOTHING_IMAGE=busybox@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209
3128

32-
BUILD_NAME=y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz")
33-
3429
# TODO copy is significantly faster if the source image is already in the target repo
3530

3631
# crane hangs for a long time if it doesn't know that the registry is plain http
3732
NOTHING_IMAGE=$(echo $NOTHING_IMAGE | sed 's|.local|.local:80|')
3833
IMAGE=$(echo $IMAGE | sed 's|.local|.local:80|')
3934

4035
set -x
41-
kubectl $ctx -n ystack run --restart=Never --attach \
42-
--image=$CRANE_IMAGE y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz") -- \
43-
cp $NOTHING_IMAGE $IMAGE
36+
y-crane cp $NOTHING_IMAGE $IMAGE

bin/y-cluster-assert-install

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/sh
2+
[ -z "$DEBUG" ] || set -x
3+
set -e
4+
5+
OPERATOR_VERSION=1b525b8a77f79e08b8653101f622e73083daf293
6+
KUBERNETES_ASSERT_VERSION=41207ccf8536ce022919d9aef86d2124b2c142b1
7+
8+
ctx=$1
9+
case $ctx in
10+
"--context="*) shift 1 ;;
11+
*) echo "Initial arg must be --context=" && exit 1 ;;
12+
esac
13+
14+
# If we're to make this script idempotent we must make sure that re-applying the operator's bundle isn't painfully slow
15+
kubectl $ctx create namespace monitoring
16+
17+
kubectl $ctx -n default apply -f https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml
18+
19+
kubectl-waitretry $ctx -n default --for=condition=Ready pod -l app.kubernetes.io/name=prometheus-operator
20+
21+
kubectl $ctx -n monitoring apply -k github.com/Yolean/kubernetes-assert/example-small?ref=$KUBERNETES_ASSERT_VERSION
22+
23+
kubectl-waitretry $ctx -n monitoring --for=condition=Ready pod --all

bin/y-cluster-install-prometheus-operator

Lines changed: 0 additions & 7 deletions
This file was deleted.

bin/y-cluster-provision-gke

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -94,19 +94,20 @@ esac
9494

9595
gcloud container clusters create "$NAME" $FLAGS "$@"
9696

97-
GKE_CONTEXT_NAME=$(kubectl config current-context)
97+
GKE_CONTEXT_NAME=$(kubectl $ctx config current-context)
98+
ctx="--context=${GKE_CONTEXT_NAME}"
9899

99100
echo "For outbound Internet access: y-cluster-provision-gke-network-outbound-enable"
100101

101-
kubectl create namespace ystack
102+
kubectl $ctx create namespace ystack
102103

103-
kubectl -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/
104+
kubectl $ctx -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/
104105

105106
### To be moved to registry provision script, but we do want the initial context name
106107

107108
[ -z "${BUILDS_BUCKET_NAME}" ] && BUILDS_BUCKET_NAME=${GKE_CONTEXT_NAME}_builds
108109

109-
kubectl create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME}
110+
kubectl $ctx create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME}
110111

111112
gcloud compute backend-buckets list
112113
gsutil ls
@@ -120,8 +121,8 @@ metadata:
120121
spec: {}
121122
EOF
122123

123-
kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
124+
kubectl $ctx apply -f gke/registry/builds-gcs-storagebucket.yaml || \
124125
gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME} || \
125126
echo "Warning: failed to create builds registry storage bucket"
126127

127-
y-cluster-install-prometheus-operator
128+
y-cluster-assert-install $ctx

bin/y-cluster-provision-k3s-multipass

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,11 @@ multipass exec "$VM_NAME" -- sudo cat /etc/rancher/k3s/k3s.yaml \
2626

2727
KUBECONFIG="$KUBECONFIG.tmp" kubectl config rename-context default local
2828

29-
KUBECONFIG="$KUBECONFIG.tmp" kubectl create namespace ystack
29+
KUBECONFIG="$KUBECONFIG.tmp" kubectl --context=local create namespace ystack
3030

3131
#KUBECONFIG="$KUBECONFIG.tmp" kubectl apply -k $YSTACK_HOME/metrics-server
3232

33-
KUBECONFIG="$KUBECONFIG.tmp" y-cluster-install-prometheus-operator
33+
KUBECONFIG="$KUBECONFIG.tmp" y-cluster-assert-install --context=local
3434

3535
y-kubeconfig-import "$KUBECONFIG.tmp"
3636

0 commit comments

Comments
 (0)