Skip to content

Commit 4b05e71

Browse files
authored
[jb-gha] improve nightly script and update file structure (#20220)
* [jb-gha] draft nightly script and update file structure * Update GHA with draft script * Update summary * Run leeway build * Fix build and share slack notification * Add force use dev latest images leeway script
1 parent eb279a9 commit 4b05e71

File tree

11 files changed

+452
-217
lines changed

11 files changed

+452
-217
lines changed

.github/workflows/jetbrains-auto-update-template.yml

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -33,39 +33,40 @@ jobs:
3333
identity_provider: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_PROVIDER || secrets.DEV_PREVIEW_PROVIDER }}
3434
service_account: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_SA || secrets.DEV_PREVIEW_SA }}
3535
leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
36-
- name: Find downloadUrl
37-
id: download-url
36+
- name: Install dependencies
3837
run: |
39-
downloadUrl=$(cat WORKSPACE.yaml | yq r - 'defaultArgs.${{ inputs.productId }}DownloadUrl')
40-
echo "::set-output name=downloadUrl::$downloadUrl"
41-
- name: Find IDE version to download
42-
id: ide-version
38+
cd ./components/ide/gha-update-image/
39+
yarn
40+
npm i -g bun
41+
- name: Find Nightly Target
42+
id: find-target
4343
run: |
44-
IDE_VERSIONS_JSON=$(bash ./components/ide/jetbrains/image/resolve-latest-ide-version.sh ${{ inputs.productCode }} ${{ steps.download-url.outputs.downloadUrl }})
45-
IDE_BUILD_VERSION=$(echo "$IDE_VERSIONS_JSON" | jq -r .IDE_BUILD_VERSION)
46-
IDE_VERSION=$(echo "$IDE_VERSIONS_JSON" | jq -r .IDE_VERSION)
47-
echo "IDE_BUILD_VERSION: $IDE_BUILD_VERSION"
48-
echo "IDE_VERSION: $IDE_VERSION"
49-
echo "::set-output name=ideBuildVersion::$IDE_BUILD_VERSION"
50-
echo "::set-output name=ideVersion::$IDE_VERSION"
44+
cd ./components/ide/gha-update-image/
45+
bun run index-jb-nightly.ts --task=1 --productCode=${{ inputs.productCode }}
46+
47+
if [ -f /tmp/__gh_output.txt ]
48+
then
49+
cat /tmp/__gh_output.txt >> $GITHUB_OUTPUT
50+
fi
5151
- name: Leeway build
52-
if: ${{ steps.ide-version.outputs.ideBuildVersion }}
52+
if: ${{ steps.find-target.outputs.buildNumber }}
5353
env:
5454
LEEWAY_MAX_PROVENANCE_BUNDLE_SIZE: "8388608"
5555
LEEWAY_REMOTE_CACHE_BUCKET: ${{ github.ref == 'refs/heads/main' && 'leeway-cache-main-c514a01' || 'leeway-cache-dev-3ac8ef5' }}
5656
run: |
5757
imageRepoBase=${{ github.ref == 'refs/heads/main' && 'eu.gcr.io/gitpod-core-dev/build' || 'eu.gcr.io/gitpod-dev-artifact/build' }}
58-
leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.ide-version.outputs.ideBuildVersion }} components/ide/jetbrains/image:${{ inputs.productId }}-latest -DjbBackendVersion=${{ steps.ide-version.outputs.ideVersion }}
58+
echo "Upgrade latest ${{ inputs.productId }} image with ${{ steps.find-target.outputs.editorSummary }}" >> $GITHUB_STEP_SUMMARY
59+
leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.find-target.outputs.buildNumber }} components/ide/jetbrains/image:${{ steps.find-target.outputs.image }}-latest -DjbBackendVersion=${{ steps.find-target.outputs.jbBackendVersion }}
5960
- name: Get previous job's status
6061
id: lastrun
6162
uses: filiptronicek/get-last-job-status@main
6263
- name: Slack Notification
63-
if: ${{ (success() && steps.lastrun.outputs.status == 'failed') || failure() }}
64+
if: ${{ (success() && steps.find-target.outputs.buildNumber) || failure() }}
6465
uses: rtCamp/action-slack-notify@v2
6566
env:
6667
SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }}
6768
SLACK_COLOR: ${{ job.status }}
68-
SLACK_TITLE: ${{ inputs.productId }}
69+
SLACK_TITLE: Upgrade latest ${{ inputs.productId }} image with ${{ steps.find-target.outputs.editorSummary }}
6970
SLACK_FOOTER: "<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Workflow logs>"
7071

7172
delete-runner:
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
scripts:
2+
- name: jb-use-dev-latest
3+
script: |
4+
ide_list=("intellij" "goland" "pycharm" "phpstorm" "rubymine" "webstorm" "rider" "clion" "rustrover")
5+
prop_list=("latestImage")
6+
7+
cf_patch=$(kubectl get cm ide-config -o=json | jq '.data."config.json"' |jq -r)
8+
for ide in "${ide_list[@]}"; do
9+
for prop in "${prop_list[@]}"; do
10+
cf_patch=$(echo "$cf_patch" | jq ".ideOptions.options.$ide.$prop = \"eu.gcr.io/gitpod-dev-artifact/build/ide/$ide:latest\"")
11+
done
12+
done
13+
cf_patch=$(echo "$cf_patch" |jq tostring)
14+
cf_patch="{\"data\": {\"config.json\": $cf_patch}}"
15+
# echo "$cf_patch"
16+
17+
kubectl patch cm ide-config --type=merge -p "$cf_patch"
18+
kubectl rollout restart deployment ide-service
19+
kubectl rollout restart deployment server
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
[
2+
{
3+
"image": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-d54058956db3274084249bdbc4f508ab111b8c51",
4+
"detail": {
5+
"pluginSinceBuild": "242.19533",
6+
"pluginUntilBuild":"242.*"
7+
}
8+
},
9+
{
10+
"image": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-d54058956db3274084249bdbc4f508ab111b8c51-latest",
11+
"detail": {
12+
"pluginSinceBuild": "242.19533",
13+
"pluginUntilBuild":"242.*"
14+
}
15+
}
16+
]
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// Copyright (c) 2024 Gitpod GmbH. All rights reserved.
2+
// Licensed under the GNU Affero General Public License (AGPL).
3+
// See License.AGPL.txt in the project root for license information.
4+
5+
// Update JetBrains latest editor images
6+
//
7+
// ```
8+
// bun run index-jb-nightly.ts --task=<id> --productCode=<code>
9+
// ```
10+
11+
import { $ } from "bun";
12+
import { appendGitHubOutput, pathToBackendPluginGradleLatest, readWorkspaceYaml } from "./lib/common";
13+
import { maybeCompatible, parseGradleProperties, parseGradlePropertiesFromTaskConfig } from "./lib/jb-helper/jb-helper";
14+
import { fetchProductReleases, ReleaseItem, releaseItemStr } from "./lib/jb-helper/jb-releases";
15+
import { getTaskFromArgs } from "./lib/jb-helper/jb-gradle-task-config";
16+
17+
$.nothrow(); // git likes to respond with non-zero codes, but it is alright for us
18+
19+
const task = getTaskFromArgs(true);
20+
21+
if (task.id !== 1) {
22+
throw new Error(`Only task 1 is supported, got ${task.id}`);
23+
}
24+
25+
console.log(`Updating nightly editor for ${task.productId} (${task.productType})`);
26+
27+
const { parsedObj: parsedWorkspaceYaml } = await readWorkspaceYaml();
28+
29+
const downloadUrl = parsedWorkspaceYaml.defaultArgs[task.productId + "DownloadUrl"] as string;
30+
31+
const latestGradle = parseGradleProperties(await Bun.file(pathToBackendPluginGradleLatest).text());
32+
33+
const platformVersionType = "build";
34+
35+
const releases = await fetchProductReleases({ productCode: task.productCode, productType: task.productType });
36+
37+
let maybeCompatibleRelease: ReleaseItem | undefined;
38+
for (const release of releases) {
39+
switch (platformVersionType) {
40+
case "build": {
41+
const ok = maybeCompatible(release, latestGradle);
42+
if (ok) {
43+
maybeCompatibleRelease = release;
44+
break;
45+
} else {
46+
console.error(`${releaseItemStr(release)} incompatible`);
47+
}
48+
}
49+
}
50+
if (maybeCompatibleRelease) {
51+
break;
52+
}
53+
}
54+
55+
if (maybeCompatibleRelease) {
56+
console.log(`${releaseItemStr(maybeCompatibleRelease)} maybe compatible`);
57+
}
58+
59+
const targetRelease = maybeCompatibleRelease || releases.find((e) => e.downloads.linux?.link === downloadUrl);
60+
61+
if (!targetRelease) {
62+
throw new Error(`No compatible release found`);
63+
}
64+
65+
console.log(`Preparing to use ${releaseItemStr(targetRelease)} as latest version for ${task.productId}`);
66+
67+
const targetConfig = parseGradlePropertiesFromTaskConfig(task, targetRelease);
68+
69+
// TODO: actually update nightly editor
70+
console.log(
71+
`Going to exec \`leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${targetConfig.platformVersion} components/ide/jetbrains/image:${task.productId}-latest -DjbBackendVersion=${targetRelease.version}\``,
72+
);
73+
74+
appendGitHubOutput(`buildNumber=${targetConfig.platformVersion}`);
75+
appendGitHubOutput(`image=${task.productId}`);
76+
appendGitHubOutput(`jbBackendVersion=${targetRelease.version}`);
77+
appendGitHubOutput(`editorSummary=${releaseItemStr(targetRelease)}`);

components/ide/gha-update-image/index-jb-platform-update.ts

Lines changed: 28 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -2,113 +2,37 @@
22
// Licensed under the GNU Affero General Public License (AGPL).
33
// See License.AGPL.txt in the project root for license information.
44

5-
// Update JetBrains Platform Version
5+
// Update JetBrains Plugins (gateway / backend) Platform Version
6+
//
7+
// ```
8+
// bun run index-jb-platform-update.ts --task=<id>
9+
// ```
610

7-
import path from "path";
8-
import { parseArgs } from "util";
9-
import { pathToProjectRoot } from "./lib/common";
10-
import { jbUpdatePlatformVersion } from "./lib/jb-update-platform-version";
11+
import { getTaskFromArgs } from "./lib/jb-helper/jb-gradle-task-config";
12+
import {
13+
parseGradleProperties,
14+
renderPropertiesTemplate,
15+
parseGradlePropertiesFromTaskConfig,
16+
} from "./lib/jb-helper/jb-helper";
17+
import { fetchProductReleases } from "./lib/jb-helper/jb-releases";
1118

12-
const targetInfo = [
13-
{
14-
id: 1,
15-
taskName: "Latest Backend Plugin",
16-
productCode: "IIU",
17-
productType: "eap,rc,release",
18-
xmlName: "IntelliJ IDEA",
19-
xmlChannels: ["IC-IU-EAP-licensing-RELEASE", "IC-IU-EAP-licensing-EAP", "IC-IU-RELEASE-licensing-RELEASE"],
20-
useXml: true,
21-
gradlePropertiesPath: path.resolve(
22-
pathToProjectRoot,
23-
"components/ide/jetbrains/backend-plugin/gradle-latest.properties",
24-
),
25-
gradlePropertiesTemplate: `# Code generated by gha-update-image/index-jb-platform-update.ts. DO NOT EDIT.
26-
# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
27-
# for insight into build numbers and IntelliJ Platform versions.
28-
# revert pluginSinceBuild if it's unnecessary
29-
pluginSinceBuild={{pluginSinceBuild}}
30-
pluginUntilBuild={{pluginUntilBuild}}
31-
# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl
32-
# See https://jb.gg/intellij-platform-builds-list for available build versions.
33-
pluginVerifierIdeVersions={{pluginVerifierIdeVersions}}
34-
# Version from "com.jetbrains.intellij.idea" which can be found at https://www.jetbrains.com/intellij-repository/snapshots
35-
platformVersion={{platformVersion}}
36-
`,
37-
},
38-
{
39-
id: 2,
40-
taskName: "Latest Frontend Plugin",
41-
productCode: "GW",
42-
productType: "eap,rc,release",
43-
xmlName: "Gateway",
44-
xmlChannels: ["GW-EAP-licensing-EAP", "GW-RELEASE-licensing-RELEASE", "GW-EAP-licensing-RELEASE"],
45-
useXml: true,
46-
useHumanreadableVersion: true,
47-
gradlePropertiesPath: path.resolve(
48-
pathToProjectRoot,
49-
"components/ide/jetbrains/gateway-plugin/gradle-latest.properties",
50-
),
51-
gradlePropertiesTemplate: `# Code generated by gha-update-image/index-jb-platform-update.ts. DO NOT EDIT.
52-
# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
53-
# for insight into build numbers and IntelliJ Platform versions.
54-
# revert pluginSinceBuild if it's unnecessary
55-
pluginSinceBuild={{pluginSinceBuild}}
56-
pluginUntilBuild={{pluginUntilBuild}}
57-
# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl
58-
# See https://jb.gg/intellij-platform-builds-list for available build versions.
59-
pluginVerifierIdeVersions={{pluginVerifierIdeVersions}}
60-
# Version from "com.jetbrains.gateway" which can be found at https://www.jetbrains.com/intellij-repository/snapshots
61-
platformVersion={{platformVersion}}
62-
`,
63-
},
64-
{
65-
id: 3,
66-
taskName: "Stable Frontend Plugin",
67-
productCode: "GW",
68-
productType: "release",
69-
xmlName: "Gateway",
70-
xmlChannels: ["GW-RELEASE-licensing-RELEASE"],
71-
useXml: true,
72-
useHumanreadableVersion: true,
73-
gradlePropertiesPath: path.resolve(
74-
pathToProjectRoot,
75-
"components/ide/jetbrains/gateway-plugin/gradle-stable.properties",
76-
),
77-
gradlePropertiesTemplate: `# Code generated by gha-update-image/index-jb-platform-update.ts. DO NOT EDIT.
78-
# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
79-
# for insight into build numbers and IntelliJ Platform versions.
80-
# revert pluginSinceBuild if it's unnecessary
81-
pluginSinceBuild={{pluginSinceBuild}}
82-
pluginUntilBuild={{pluginUntilBuild}}
83-
# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl
84-
# See https://jb.gg/intellij-platform-builds-list for available build versions.
85-
pluginVerifierIdeVersions={{pluginVerifierIdeVersions}}
86-
# Version from "com.jetbrains.gateway" which can be found at https://www.jetbrains.com/updates/updates.xml
87-
platformVersion={{platformVersion}}
88-
`,
89-
},
90-
];
19+
const task = getTaskFromArgs(false);
9120

92-
const { values } = parseArgs({
93-
args: Bun.argv,
94-
options: {
95-
task: {
96-
type: "string",
97-
},
98-
},
99-
strict: true,
100-
allowPositionals: true,
101-
});
21+
console.log("Updating", task.taskName);
10222

103-
const taskID = Number.parseInt(values.task ?? "NaN");
23+
const releases = await fetchProductReleases(task);
24+
const newProps = parseGradlePropertiesFromTaskConfig(task, releases[0]);
25+
console.log("New properties info:", newProps);
26+
const oldProps = parseGradleProperties(await Bun.file(task.gradlePropertiesPath).text());
27+
console.log("Old properties info:", oldProps);
10428

105-
const target = targetInfo.find((e) => e.id === taskID);
106-
if (!target) {
107-
throw new Error(
108-
`Invalid task id: ${taskID}, update cmd with \`--task="<name>"\`, available tasks: \n\t- ${targetInfo
109-
.map((e) => `(${e.id}) ${e.taskName}`)
110-
.join("\n\t- ")}`,
111-
);
29+
if (newProps.platformVersion === oldProps.platformVersion) {
30+
console.warn("PlatformVersion are the same, no need to update platform version");
31+
process.exit(0);
11232
}
113-
console.log("Updating", target.taskName);
114-
jbUpdatePlatformVersion(target);
33+
34+
const newGradleContent = renderPropertiesTemplate("gha-update-image/index-jb-platform-update.ts", task, newProps);
35+
36+
await Bun.write(task.gradlePropertiesPath, newGradleContent);
37+
38+
console.log("Updated platform version to:", newProps.platformVersion);

components/ide/gha-update-image/lib/common.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,14 @@ export const pathToBackendPluginGradleStable = path.resolve(
5353
"components/ide/jetbrains/backend-plugin/gradle-stable.properties",
5454
);
5555

56+
// gradle-latest.properties
57+
export const pathToBackendPluginGradleLatest = path.resolve(
58+
pathToProjectRoot,
59+
"components/ide/jetbrains/backend-plugin/gradle-latest.properties",
60+
);
61+
62+
export const pathToBackendPluginDir = path.resolve(pathToProjectRoot, "components/ide/jetbrains/backend-plugin");
63+
5664
// ide-configmap.json
5765
export const pathToConfigmap = path.resolve(
5866
pathToProjectRoot,

0 commit comments

Comments
 (0)