Skip to content

Commit ab53c58

Browse files
alextruninnalextruninn
committed
CLOUD-64700: [kms] Автоматизировать процедуру обновления docker и compute образов Vault+KMS
1 parent e29d98a commit ab53c58

19 files changed

+833
-0
lines changed

‎yandex/scripts/common.sh‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
init() {
2+
START_DIR=$(pwd)
3+
trap 'cd $START_DIR' EXIT
4+
5+
SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
6+
cd $SCRIPT_DIR
7+
SCRIPT_DIR=$(pwd)
8+
9+
if [[ -n $WORK_DIR ]]; then
10+
mkdir -p $WORK_DIR
11+
cd $WORK_DIR
12+
fi
13+
WORK_DIR=$(pwd)
14+
YCKMS_VERSION=${PATCH_BRANCH:-"$BASE_VERSION+yckms"}
15+
}
16+
17+
go_to_work_dir() {
18+
cd $WORK_DIR
19+
}
20+
21+
cleanup() {
22+
go_to_work_dir
23+
rm -rf vault
24+
rm -rf vault-kms-wrapper
25+
}
26+
27+
init_vault() {
28+
go_to_work_dir
29+
if [[ ! -d "vault" ]]; then
30+
echo "Cloning vault"
31+
git clone git@github.com:yandex-cloud/vault.git
32+
cd vault
33+
git remote add upstream git@github.com:hashicorp/vault.git
34+
else
35+
echo "Vault already cloned"
36+
cd vault
37+
git reset --hard
38+
fi
39+
40+
echo "Synchronizing vault with upstream"
41+
git checkout main
42+
git pull upstream main
43+
echo "Fetching tags"
44+
git fetch upstream --tags
45+
46+
go_to_work_dir
47+
}
48+
49+
init_vault_kms_wrapper() {
50+
go_to_work_dir
51+
if [[ ! -d "vault-kms-wrapper" ]]; then
52+
echo "Cloning vault-kms-wrapper"
53+
git clone git@github.com:yandex-cloud/vault-kms-wrapper.git
54+
cd vault-kms-wrapper
55+
else
56+
echo "Vault already cloned"
57+
cd vault-kms-wrapper
58+
fi
59+
60+
echo "Refreshing main"
61+
git reset --hard
62+
git checkout main
63+
git pull
64+
git fetch -p
65+
66+
go_to_work_dir
67+
}
68+
69+
get_kms_wrapper_version() {
70+
go_to_work_dir
71+
cd vault
72+
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
73+
git checkout $BASE_VERSION
74+
KMS_WRAPPER_VERSION=$(go list -m github.com/hashicorp/go-kms-wrapping/v2 | cut -f 2 -d " ")
75+
echo "Current go-kms-wrapping version: $KMS_WRAPPER_VERSION"
76+
git checkout $CURRENT_BRANCH
77+
go_to_work_dir
78+
}

‎yandex/scripts/push_compute/push_compute_image.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
set -e
3+
4+
SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
5+
. $SCRIPT_PATH/../common.sh
6+
. $SCRIPT_PATH/push_compute.cfg
7+
8+
init
9+
init_vault
10+
cd vault
11+
12+
if ! git checkout $YCKMS_VERSION; then
13+
echo >&2 "Cannot find branch $YCKMS_VERSION !"
14+
exit 1
15+
else
16+
echo "Checkout to branch '$YCKMS_VERSION' succeeded"
17+
fi
18+
19+
TOKEN=$(ycp --profile $PROFILE iam create-token)
20+
21+
cd yandex/compute
22+
23+
if [[ $REPLACE_DEFAULT_ENDPOINT == 1 ]]; then
24+
echo "$(jq --arg ENDPOINT "$ENDPOINT" --arg ZONE "$ZONE" --arg PLATFORM_ID "$PLATFORM_ID" \
25+
'.builders[0] += {"endpoint":$ENDPOINT, "zone":$ZONE, "platform_id":$PLATFORM_ID}' vault.packer.json)" > vault.packer.json
26+
fi
27+
28+
echo "$(jq --arg SOURCE_IMAGE_FAMILY "ubuntu-2004-lts" '.builders[0].source_image_family = $SOURCE_IMAGE_FAMILY' vault.packer.json)" > vault.packer.json
29+
30+
FOLDER_ID="$FOLDER_ID" TOKEN="$TOKEN" $SCRIPT_PATH/packer build vault.packer.json

‎yandex/scripts/push_compute/push_compute_sample.cfg‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#Requirements:
2+
# - yc and ycp
3+
# - jq
4+
# - packer (from https://wiki.yandex-team.ru/cloud/devel/assembly-workshop/#packer)
5+
# packer should be placed inside directory with script
6+
# Github already contains release version of $BASE_VERSION ($BASE_VERSION+$WRAPPER_SUFFIX)
7+
8+
#Version of patched vault from which will be build image(from git branch $BASE_VERSION+yckms)
9+
BASE_VERSION="v1.14.1"
10+
#Directory inside which patched version will be pulled from git
11+
WORK_DIR=./../tmp
12+
#If need to replace default endpoint(default is cloud.yandex.ru)
13+
REPLACE_DEFAULT_ENDPOINT=1
14+
ENDPOINT=api.il.nebius.cloud:443
15+
ZONE=il1-a
16+
PLATFORM_ID=standard-v3
17+
#Folder inside which image will be pulled
18+
FOLDER_ID=b48ans97du1s547k9e9m
19+
PROFILE=israel
20+
#Branch of git inside https://github.com/yandex-cloud/vault
21+
#from which will be build image(if not set $BASE_VERSION+yckms will be used)
22+
PATCH_BRANCH=test

‎yandex/scripts/push_docker/push_docker_image.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
set -e
3+
4+
SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
5+
. $SCRIPT_PATH/../common.sh
6+
. $SCRIPT_PATH/push_docker.cfg
7+
8+
init
9+
init_vault
10+
cd vault
11+
12+
if ! git checkout $YCKMS_VERSION; then
13+
echo >&2 "Cannot find branch $YCKMS_VERSION !"
14+
exit 1
15+
else
16+
echo "Checkout to branch '$YCKMS_VERSION' succeeded"
17+
fi
18+
19+
cd yandex
20+
cd docker
21+
22+
IAM_TOKEN=$(ycp --profile="$PROFILE" iam create-token)
23+
DOMAIN="cr.yandex"
24+
25+
if [[ "$PROFILE" == "israel" ]]; then
26+
DOMAIN="cr.cloudil.com"
27+
fi
28+
29+
if [[ "$PROFILE" == "preprod" ]]; then
30+
DOMAIN="cr.cloud-preprod.yandex.net"
31+
fi
32+
33+
docker login --username iam --password $IAM_TOKEN $DOMAIN
34+
docker build --platform linux/amd64 -t $DOMAIN/$REGISTRY_ID/vault:$BASE_VERSION-yckms -t $DOMAIN/$REGISTRY_ID/vault .
35+
docker push $DOMAIN/$REGISTRY_ID/vault:$BASE_VERSION-yckms
36+
docker logout

‎yandex/scripts/push_docker/push_docker_sample.cfg‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#Requirements:
2+
# - yc and ycp
3+
# - docker
4+
# Github already contains release version of $BASE_VERSION ($BASE_VERSION+$WRAPPER_SUFFIX)
5+
6+
#Version of patched vault from which will be build image(from git branch $BASE_VERSION+yckms)
7+
BASE_VERSION="v1.14.1"
8+
#Directory inside which patched version will be pulled from git
9+
WORK_DIR=./../tmp
10+
PROFILE=israel
11+
#Registry to which docker image will be pushed
12+
REGISTRY_ID=crls41le9juekjee2rv4
13+
#Branch of git inside https://github.com/yandex-cloud/vault
14+
#from which will be build image(if not set $BASE_VERSION+yckms will be used)
15+
PATCH_BRANCH=test

‎yandex/scripts/push_helm_chart/helm_chart_sample.cfg‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#Requirements:
2+
# - yc and ycp
3+
# - docker
4+
# - helm
5+
# - yq
6+
# - diff, patch, perl (should be included in default distributive)
7+
# - Build and pulled docker image of corresponding vault version(BASE_VERSION parameter)
8+
9+
# Docker image version of patched version that will be used for helm (Version is $BASE_VERSION-yckms)
10+
BASE_VERSION="v1.14.1"
11+
#Diractory in will be pulled vault-csi-provider and vault-k8s
12+
WORK_DIR=./../tmp
13+
PROFILE=israel
14+
#Registry inside which will be pushed helm image
15+
REGISTRY_ID=crls41le9juekjee2rv4

‎yandex/scripts/push_helm_chart/push_helm_chart.sh‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#!/bin/bash
2+
set -e
3+
4+
SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
5+
. $SCRIPT_PATH/helm_chart.cfg
6+
7+
if [[ -n $WORK_DIR ]]; then
8+
mkdir -p $WORK_DIR
9+
cd $WORK_DIR
10+
fi
11+
12+
IAM_TOKEN=$(ycp --profile="$PROFILE" iam create-token)
13+
DOMAIN="cr.yandex"
14+
15+
if [[ "$PROFILE" == "israel" ]]; then
16+
DOMAIN="cr.cloudil.com"
17+
fi
18+
19+
if [[ "$PROFILE" == "preprod" ]]; then
20+
DOMAIN="cr.cloud-preprod.yandex.net"
21+
fi
22+
23+
docker login --username iam --password $IAM_TOKEN $DOMAIN
24+
25+
git clone https://github.com/hashicorp/vault-helm.git ./vault-helm
26+
git clone ssh://git@bb.yandexcloud.net/cloud/mk8s-marketplace-helm.git ./mk8s-marketplace-helm
27+
28+
cd vault-helm
29+
30+
TAGS=$(git tag --sort="-version:refname")
31+
echo $TAGS
32+
33+
VERSION=$(echo "$BASE_VERSION" | cut -c 2-)
34+
CUT_VERSION="${VERSION%.*}"
35+
ACTUAL_TAG=''
36+
37+
for TAG in $TAGS
38+
do
39+
git checkout tags/$TAG values.yaml
40+
ACTUAL_VERSION=$(yq '.server.image.tag' values.yaml)
41+
ACTUAL_VERSION="${ACTUAL_VERSION%.*}"
42+
echo "Actual version = $ACTUAL_VERSION and version = $CUT_VERSION"
43+
if [ $CUT_VERSION = $ACTUAL_VERSION ]
44+
then
45+
ACTUAL_TAG=$TAG
46+
echo "Found actual version $ACTUAL_VERSION"
47+
break
48+
fi
49+
done
50+
51+
git checkout tags/$ACTUAL_TAG
52+
53+
sudo rm -r ./.git
54+
55+
CSI_PROVIDER_ACTUAL_TAG=$(yq '.csi.image.tag' values.yaml)
56+
K8S_ACTUAL_TAG=$(yq '.injector.image.tag' values.yaml)
57+
58+
cd ../
59+
60+
docker pull hashicorp/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG --platform amd64
61+
docker pull hashicorp/vault-k8s:$K8S_ACTUAL_TAG --platform amd64
62+
63+
CSI_PROVIDER_IMAGE_ID=$(docker images hashicorp/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG -q)
64+
K8S_IMAGE_ID=$(docker images hashicorp/vault-k8s:$K8S_ACTUAL_TAG -q)
65+
66+
docker tag $K8S_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-k8s:$K8S_ACTUAL_TAG
67+
docker tag $K8S_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-k8s:latest
68+
69+
docker tag $CSI_PROVIDER_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG
70+
docker tag $CSI_PROVIDER_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:latest
71+
72+
docker push $DOMAIN/$REGISTRY_ID/vault/vault-k8s:$K8S_ACTUAL_TAG
73+
docker push $DOMAIN/$REGISTRY_ID/vault/vault-k8s:latest
74+
75+
docker push $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG
76+
docker push $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:latest
77+
78+
rm -r ./mk8s-marketplace-helm/products/hashicorp-vault/chart
79+
cp -r ./vault-helm ./mk8s-marketplace-helm/products/hashicorp-vault/chart
80+
rm -r ./mk8s-marketplace-helm/products/hashicorp-vault/chart/test
81+
82+
CHART_VERSION="$(yq '.version' ./vault-helm/Chart.yaml)-1"
83+
replacement=$CHART_VERSION yq -i '.version = strenv(replacement)' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/Chart.yaml
84+
85+
cat >./mk8s-marketplace-helm/products/hashicorp-vault/chart/templates/kms-creds-secret.yaml <<EOF
86+
apiVersion: v1
87+
kind: Secret
88+
metadata:
89+
name: kms-creds
90+
namespace: {{ .Release.Namespace | quote }}
91+
labels:
92+
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
93+
type: Opaque
94+
data:
95+
"credentials.json": {{ .Values.yandexKmsAuthJson | b64enc }}
96+
EOF
97+
98+
/bin/bash $SCRIPT_PATH/update_values.sh
99+
100+
cd mk8s-marketplace-helm/products/hashicorp-vault/chart
101+
helm package .
102+
helm push ./vault-$(echo "$ACTUAL_TAG" | cut -c 2-)-1.tgz oci://$DOMAIN/$REGISTRY_ID/vault/chart

‎yandex/scripts/push_helm_chart/update_values.sh‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
3+
SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
4+
. $SCRIPT_PATH/release_sample.cfg
5+
6+
CSI_PROVIDER_ACTUAL_TAG=$(yq '.csi.image.tag' ./vault-helm/values.yaml)
7+
K8S_ACTUAL_TAG=$(yq '.injector.image.tag' ./vault-helm/values.yaml)
8+
9+
yq '.' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml > values.yaml.new
10+
11+
DOMAIN="cr.yandex"
12+
if [[ "$PROFILE" == "israel" ]]; then
13+
DOMAIN="cr.cloudil.com"
14+
fi
15+
if [[ "$PROFILE" == "preprod" ]]; then
16+
DOMAIN="cr.cloud-preprod.yandex.net"
17+
fi
18+
VAULT_VERSION="$(echo "$BASE_VERSION" | cut -c 2-)-yckms"
19+
20+
replacement=$VAULT_VERSION yq -i '(.. | select(key == "repository" and . == "hashicorp/vault") | parent).tag |= strenv(replacement)' values.yaml.new
21+
replacement="$DOMAIN/$REGISTRY_ID/vault/vault" yq -i '(.. | select(key == "repository" and . == "hashicorp/vault") | parent).repository |= strenv(replacement)' values.yaml.new
22+
replacement=$CSI_PROVIDER_ACTUAL_TAG yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-csi-provider") | parent).tag |= strenv(replacement)' values.yaml.new
23+
replacement="$DOMAIN/$REGISTRY_ID/vault/vault-csi-provider" yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-csi-provider") | parent).repository |= strenv(replacement)' values.yaml.new
24+
replacement=$K8S_ACTUAL_TAG yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-k8s") | parent).tag |= strenv(replacement)' values.yaml.new
25+
replacement="$DOMAIN/$REGISTRY_ID/vault/vault-k8s" yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-k8s") | parent).repository |= strenv(replacement)' values.yaml.new
26+
yq -i '(.. | select(key == "extraVolumes")) |= {"type": "secret", "name": "kms-creds"} ' values.yaml.new
27+
28+
yq '.' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml > values.yaml.noblanks
29+
diff -B values.yaml.noblanks values.yaml.new > ./patch.file
30+
patch ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml ./patch.file
31+
32+
#patch works bad, so it's needed to patch twice
33+
yq '.' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml > values.yaml.new
34+
35+
read -rd '' replacement << EOF
36+
37+
seal "yandexcloudkms" {
38+
kms_key_id = "{{ .Values.yandexKmsKeyId}}"
39+
service_account_key_file = "/vault/userconfig/kms-creds/credentials.json"
40+
}
41+
42+
EOF
43+
replacement=$replacement yq -i '(.. | select(key == "standalone")).config |= . + strenv(replacement)' values.yaml.new
44+
45+
yq '.' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml > values.yaml.noblanks
46+
diff -B values.yaml.noblanks values.yaml.new > ./patch.file
47+
patch ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml ./patch.file
48+
49+
echo 'yandexKmsAuthJson: ""' >> ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml
50+
echo 'yandexKmsKeyId: ""' >> ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml
51+
52+
perl -pe 'chomp if eof' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml >tmp.file
53+
mv tmp.file ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml
54+
55+
rm patch.file
56+
rm values.yaml.new
57+
rm values.yaml.noblanks
58+
rm ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml.orig

‎yandex/scripts/release/cleanup.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
set -e
3+
4+
SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
5+
. $SCRIPT_PATH/common.sh
6+
. $SCRIPT_PATH/release_sample.cfg
7+
8+
init
9+
cleanup

‎yandex/scripts/release/patch_version_base.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
set -e
3+
4+
START_DIR=$(pwd)
5+
trap 'cd $START_DIR' EXIT
6+
7+
SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
8+
cd "$SCRIPT_PATH"/..
9+
10+
sed -i '' 's/.*VersionMetadata.*=.*""/VersionMetadata = "yckms"/' version/version_base.go
11+
go fmt version/version_base.go

0 commit comments

Comments
 (0)