Skip to content

Commit 9021172

Browse files
geroplgeropl
authored
[werft] Extract build.js parts to enable reuse (#2987)
* [werft] Extract build.js parts to enable reuse * [werft] Exclude 'leeway vet'
1 parent cdbf26c commit 9021172

File tree

3 files changed

+116
-82
lines changed

3 files changed

+116
-82
lines changed

‎.werft/build.js

Lines changed: 23 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ const shell = require('shelljs');
22
const fs = require('fs');
33
const { werft, exec, gitTag } = require('./util/shell.js');
44
const { sleep } = require('./util/util.js');
5-
const { recreateNamespace } = require('./util/kubectl.js');
5+
const { wipeAndRecreateNamespace, setKubectlContextNamespace, deleteNonNamespaceObjects } = require('./util/kubectl.js');
6+
const { issueAndInstallCertficate } = require('./util/certs.js');
67

78
const GCLOUD_SERVICE_ACCOUNT_PATH = "/mnt/secrets/gcp-sa/service-account.json";
89

@@ -15,7 +16,7 @@ build(context, version)
1516
function parseVersion(context) {
1617
let buildConfig = context.Annotations || {};
1718
const explicitVersion = buildConfig.version;
18-
if(explicitVersion) {
19+
if (explicitVersion) {
1920
return explicitVersion;
2021
}
2122
let version = context.Name;
@@ -80,7 +81,8 @@ async function build(context, version) {
8081
};
8182
const imageRepo = publishRelease ? "gcr.io/gitpod-io/self-hosted" : "eu.gcr.io/gitpod-core-dev/build";
8283

83-
exec(`leeway vet --ignore-warnings`);
84+
// TODO https://github.com/gitpod-io/leeway/issues/47
85+
// exec(`leeway vet --ignore-warnings`);
8486
exec(`leeway build --werft=true -c ${cacheLevel} ${dontTest ? '--dont-test':''} -Dversion=${version} -DimageRepoBase=eu.gcr.io/gitpod-core-dev/dev dev:all`, buildEnv);
8587
if (publishRelease) {
8688
exec(`gcloud auth activate-service-account --key-file "/mnt/secrets/gcp-sa-release/service-account.json"`);
@@ -129,28 +131,25 @@ async function deployToDev(version, workspaceFeatureFlags, dynamicCPULimits, reg
129131
const registryProxyPort = `2${Math.floor(Math.random()*1000)}`;
130132
const registryNodePort = `${30000 + Math.floor(Math.random()*1000)}`;
131133

132-
try {
133-
const objs = shell
134-
.exec(`kubectl get pod -l component=workspace --namespace ${namespace} --no-headers -o=custom-columns=:metadata.name`)
135-
.split("\n")
136-
.map(o => o.trim())
137-
.filter(o => o.length > 0);
138-
139-
objs.forEach(o => {
140-
werft.log("prep", `deleting workspace ${o}`);
141-
exec(`kubectl delete pod --namespace ${namespace} ${o}`, {slice: 'prep'});
142-
});
134+
// trigger certificate issuing
135+
werft.log('certificate', "organizing a certificate for the preview environment...");
136+
let namespaceRecreatedResolve = undefined;
137+
let namespaceRecreatedPromise = new Promise((resolve) => {
138+
namespaceRecreatedResolve = resolve;
139+
});
140+
const certificatePromise = issueAndInstallCertficate(werft, ".werft/certs", GCLOUD_SERVICE_ACCOUNT_PATH, namespace, "gitpod-dev.com", domain, "34.76.116.244", namespaceRecreatedPromise, "proxy-config-certificates");
143141

144-
recreateNamespace(namespace, {slice: 'prep'});
145-
[
146-
"kubectl config current-context",
147-
`kubectl config set-context --current --namespace=${namespace}`
148-
].forEach(cmd => exec(cmd, {slice: 'prep'}));
142+
// re-create namespace
143+
try {
144+
wipeAndRecreateNamespace(namespace, {slice: 'prep'});
145+
setKubectlContextNamespace(namespace, {slice: 'prep'});
146+
namespaceRecreatedResolve(); // <-- signal for certificate
149147
werft.done('prep');
150148
} catch (err) {
151149
werft.fail('prep', err);
152150
}
153151

152+
// core-dev specific section start
154153
werft.log("secret", "copy secret into namespace")
155154
try {
156155
const auth = exec(`echo -n "_json_key:$(kubectl get secret gcp-sa-registry-auth --namespace=keys -o yaml \
@@ -180,39 +179,18 @@ async function deployToDev(version, workspaceFeatureFlags, dynamicCPULimits, reg
180179
} catch (err) {
181180
werft.fail('authProviders', err);
182181
}
182+
// core-dev specific section end
183183

184-
let certificatePromise = undefined;
185-
186-
// TODO(geropl): Now that the certs reside in a separate namespaces, start the actual certificate issuing _before_ the namespace cleanup
187-
werft.log('certificate', "organizing a certificate for the preview environment...");
188-
certificatePromise = issueAndInstallCertficate(namespace, domain);
189-
184+
// cleanup non-namespace objects
190185
werft.log("predeploy cleanup", "removing old unnamespaced objects - this might take a while");
191186
try {
192-
exec(`/usr/local/bin/helm3 delete gitpod-${destname} || echo gitpod-${destname} was not installed yet`, {slice: 'predeploy cleanup'});
193-
exec(`/usr/local/bin/helm3 delete jaeger-${destname} || echo jaeger-${destname} was not installed yet`, {slice: 'predeploy cleanup'});
194-
195-
let objs = [];
196-
["ws-scheduler", "node-daemon", "cluster", "workspace", "jaeger", "jaeger-agent", "ws-sync", "ws-manager-node", "ws-daemon", "registry-facade"].forEach(comp =>
197-
["ClusterRole", "ClusterRoleBinding", "PodSecurityPolicy"].forEach(kind =>
198-
shell
199-
.exec(`kubectl get ${kind} -l component=${comp} --no-headers -o=custom-columns=:metadata.name | grep ${namespace}-ns`)
200-
.split("\n")
201-
.map(o => o.trim())
202-
.filter(o => o.length > 0)
203-
.forEach(obj => objs.push({ 'kind': kind, 'obj': obj }))
204-
)
205-
)
206-
207-
objs.forEach(o => {
208-
werft.log("predeploy cleanup", `deleting old ${o.kind} ${o.obj}`);
209-
exec(`kubectl delete ${o.kind} ${o.obj}`, {slice: 'predeploy cleanup'});
210-
});
187+
deleteNonNamespaceObjects(namespace, destname, {slice: 'predeploy cleanup'})
211188
werft.done('predeploy cleanup');
212189
} catch (err) {
213190
werft.fail('predeploy cleanup', err);
214191
}
215192

193+
// versions
216194
werft.log("deploy", "extracting versions");
217195
try {
218196
// TODO [geropl] versions is not a core component yet
@@ -222,6 +200,7 @@ async function deployToDev(version, workspaceFeatureFlags, dynamicCPULimits, reg
222200
werft.fail('deploy', err);
223201
}
224202

203+
// deployment config
225204
let flags = "";
226205
flags+=` --namespace ${namespace}`;
227206
flags+=` --set components.imageBuilder.hostDindData=/mnt/disks/ssd0/docker-${namespace}`;
@@ -287,42 +266,6 @@ async function deployToDev(version, workspaceFeatureFlags, dynamicCPULimits, reg
287266
}
288267
}
289268

290-
async function issueAndInstallCertficate(namespace, domain) {
291-
// Always use 'terraform apply' to make sure the certificate is present and up-to-date
292-
await exec(`set -x \
293-
&& cd .werft/certs \
294-
&& terraform init \
295-
&& export GOOGLE_APPLICATION_CREDENTIALS="${GCLOUD_SERVICE_ACCOUNT_PATH}" \
296-
&& terraform apply -auto-approve \
297-
-var 'namespace=${namespace}' \
298-
-var 'dns_zone_domain=gitpod-dev.com' \
299-
-var 'domain=${domain}' \
300-
-var 'public_ip=34.76.116.244' \
301-
-var 'subdomains=["", "*.", "*.ws-dev."]'`, {slice: 'certificate', async: true});
302-
303-
werft.log('certificate', `waiting until certificate certs/${namespace} is ready...`)
304-
let notReadyYet = true;
305-
while (notReadyYet) {
306-
werft.log('certificate', `polling state of certs/${namespace}...`)
307-
const result = exec(`kubectl -n certs get certificate ${namespace} -o jsonpath="{.status.conditions[?(@.type == 'Ready')].status}"`, { silent: true, dontCheckRc: true });
308-
if (result.code === 0 && result.stdout === "True") {
309-
notReadyYet = false;
310-
break;
311-
}
312-
313-
sleep(5000);
314-
}
315-
316-
werft.log('certificate', `copying certificate from "certs/${namespace}" to "${namespace}/proxy-config-certificates"`);
317-
// certmanager is configured to create a secret in the namespace "certs" with the name "${namespace}".
318-
exec(`kubectl get secret ${namespace} --namespace=certs -o yaml \
319-
| yq d - 'metadata.namespace' \
320-
| yq d - 'metadata.uid' \
321-
| yq d - 'metadata.resourceVersion' \
322-
| yq d - 'metadata.creationTimestamp' \
323-
| sed 's/${namespace}/proxy-config-certificates/g' \
324-
| kubectl apply --namespace=${namespace} -f -`);
325-
}
326269

327270
/**
328271
* Publish Charts
@@ -347,6 +290,5 @@ async function publishHelmChart(imageRepoBase) {
347290
module.exports = {
348291
parseVersion,
349292
build,
350-
issueAndInstallCertficate,
351293
deployToDev
352294
}

‎.werft/util/certs.js

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
const { exec } = require('./shell');
2+
const { sleep } = require('./util.js');
3+
4+
async function issueAndInstallCertficate(werft, pathToTerraform, gcpSaPath, namespace, dnsZoneDomain, domain, ip, namespaceRecreatedPromise, certificateSecretName) {
5+
// Always use 'terraform apply' to make sure the certificate is present and up-to-date
6+
await exec(`set -x \
7+
&& cd ${pathToTerraform} \
8+
&& terraform init \
9+
&& export GOOGLE_APPLICATION_CREDENTIALS="${gcpSaPath}" \
10+
&& terraform apply -auto-approve \
11+
-var 'namespace=${namespace}' \
12+
-var 'dns_zone_domain=${dnsZoneDomain}' \
13+
-var 'domain=${domain}' \
14+
-var 'public_ip=${ip}' \
15+
-var 'subdomains=["", "*.", "*.ws-dev."]'`, {slice: 'certificate', async: true});
16+
17+
werft.log('certificate', `waiting until certificate certs/${namespace} is ready...`)
18+
let notReadyYet = true;
19+
while (notReadyYet) {
20+
werft.log('certificate', `polling state of certs/${namespace}...`)
21+
const result = exec(`kubectl -n certs get certificate ${namespace} -o jsonpath="{.status.conditions[?(@.type == 'Ready')].status}"`, { silent: true, dontCheckRc: true });
22+
if (result.code === 0 && result.stdout === "True") {
23+
notReadyYet = false;
24+
break;
25+
}
26+
27+
sleep(5000);
28+
}
29+
30+
werft.log('certificate', 'waiting for preview env namespace being re-created...');
31+
await namespaceRecreatedPromise;
32+
33+
werft.log('certificate', `copying certificate from "certs/${namespace}" to "${namespace}/${certificateSecretName}"`);
34+
// certmanager is configured to create a secret in the namespace "certs" with the name "${namespace}".
35+
exec(`kubectl get secret ${namespace} --namespace=certs -o yaml \
36+
| yq d - 'metadata.namespace' \
37+
| yq d - 'metadata.uid' \
38+
| yq d - 'metadata.resourceVersion' \
39+
| yq d - 'metadata.creationTimestamp' \
40+
| sed 's/${namespace}/${certificateSecretName}/g' \
41+
| kubectl apply --namespace=${namespace} -f -`);
42+
}
43+
44+
module.exports = {
45+
issueAndInstallCertficate,
46+
}

‎.werft/util/kubectl.js

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,30 @@
11
const { exec } = require('./shell');
22

33

4+
function setKubectlContextNamespace(namespace, shellOpts) {
5+
[
6+
"kubectl config current-context",
7+
`kubectl config set-context --current --namespace=${namespace}`
8+
].forEach(cmd => exec(cmd, shellOpts));
9+
}
10+
11+
function wipeAndRecreateNamespace(namespace, shellOpts) {
12+
deleteAllWorkspaces(namespace, shellOpts);
13+
14+
recreateNamespace(namespace, shellOpts);
15+
}
16+
17+
function deleteAllWorkspaces(namespace, shellOpts) {
18+
const objs = exec(`kubectl get pod -l component=workspace --namespace ${namespace} --no-headers -o=custom-columns=:metadata.name`)
19+
.split("\n")
20+
.map(o => o.trim())
21+
.filter(o => o.length > 0);
22+
23+
objs.forEach(o => {
24+
exec(`kubectl delete pod --namespace ${namespace} ${o}`, shellOpts);
25+
});
26+
}
27+
428
function recreateNamespace(namespace, shellOpts) {
529
const result = exec(`kubectl get namespace ${namespace}`, { ...shellOpts, dontCheckRc: true });
630
if (result.code === 0) {
@@ -25,7 +49,29 @@ function deleteNamespace(namespace, shellOpts, wait) {
2549
}
2650
}
2751

52+
function deleteNonNamespaceObjects(namespace, destname, shellOpts) {
53+
exec(`/usr/local/bin/helm3 delete gitpod-${destname} || echo gitpod-${destname} was not installed yet`, {slice: 'predeploy cleanup'});
54+
exec(`/usr/local/bin/helm3 delete jaeger-${destname} || echo jaeger-${destname} was not installed yet`, {slice: 'predeploy cleanup'});
55+
56+
let objs = [];
57+
["ws-scheduler", "node-daemon", "cluster", "workspace", "jaeger", "jaeger-agent", "ws-sync", "ws-manager-node", "ws-daemon", "registry-facade"].forEach(comp =>
58+
["ClusterRole", "ClusterRoleBinding", "PodSecurityPolicy"].forEach(kind =>
59+
exec(`kubectl get ${kind} -l component=${comp} --no-headers -o=custom-columns=:metadata.name | grep ${namespace}-ns`, { dontCheckRc: true })
60+
.split("\n")
61+
.map(o => o.trim())
62+
.filter(o => o.length > 0)
63+
.forEach(obj => objs.push({ 'kind': kind, 'obj': obj }))
64+
)
65+
)
66+
67+
objs.forEach(o => {
68+
exec(`kubectl delete ${o.kind} ${o.obj}`, shellOpts);
69+
});
70+
}
71+
2872

2973
module.exports = {
30-
recreateNamespace
74+
setKubectlContextNamespace,
75+
wipeAndRecreateNamespace,
76+
deleteNonNamespaceObjects,
3177
}

0 commit comments

Comments
 (0)