From 66e6ea63fe4783bc528e1c5643daea6e684a52b8 Mon Sep 17 00:00:00 2001 From: Huiwen Date: Sat, 14 Sep 2024 09:56:38 +0000 Subject: [PATCH 1/6] [jb-gha] draft nightly script and update file structure --- .../config/jb-backend-plugin-versions.json | 16 +++ .../ide/gha-update-image/index-jb-nightly.ts | 72 ++++++++++ .../index-jb-platform-update.ts | 132 ++++-------------- components/ide/gha-update-image/lib/common.ts | 8 ++ .../lib/jb-helper/jb-gradle-task-config.ts | 124 ++++++++++++++++ .../lib/jb-helper/jb-helper.ts | 118 ++++++++++++++++ .../lib/jb-helper/jb-releases.ts | 43 ++++++ .../gha-update-image/lib/jb-stable-version.ts | 1 + .../lib/jb-update-platform-version.ts | 96 ------------- 9 files changed, 410 insertions(+), 200 deletions(-) create mode 100644 components/ide/gha-update-image/config/jb-backend-plugin-versions.json create mode 100644 components/ide/gha-update-image/index-jb-nightly.ts create mode 100644 components/ide/gha-update-image/lib/jb-helper/jb-gradle-task-config.ts create mode 100644 components/ide/gha-update-image/lib/jb-helper/jb-helper.ts create mode 100644 components/ide/gha-update-image/lib/jb-helper/jb-releases.ts delete mode 100644 components/ide/gha-update-image/lib/jb-update-platform-version.ts diff --git a/components/ide/gha-update-image/config/jb-backend-plugin-versions.json b/components/ide/gha-update-image/config/jb-backend-plugin-versions.json new file mode 100644 index 00000000000000..d4390f9495cf8b --- /dev/null +++ b/components/ide/gha-update-image/config/jb-backend-plugin-versions.json @@ -0,0 +1,16 @@ +[ + { + "image": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-d54058956db3274084249bdbc4f508ab111b8c51", + "detail": { + "pluginSinceBuild": "242.19533", + "pluginUntilBuild":"242.*" + } + }, + { + "image": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-d54058956db3274084249bdbc4f508ab111b8c51-latest", + "detail": { + "pluginSinceBuild": "242.19533", + "pluginUntilBuild":"242.*" + } + } +] diff --git a/components/ide/gha-update-image/index-jb-nightly.ts b/components/ide/gha-update-image/index-jb-nightly.ts new file mode 100644 index 00000000000000..b344f48dafdc7b --- /dev/null +++ b/components/ide/gha-update-image/index-jb-nightly.ts @@ -0,0 +1,72 @@ +// Copyright (c) 2024 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License.AGPL.txt in the project root for license information. + +// Update JetBrains latest editor images +// +// ``` +// bun run index-jb-nightly.ts --task= --productCode= +// ``` + +import { $ } from "bun"; +import { pathToBackendPluginGradleLatest, readWorkspaceYaml } from "./lib/common"; +import { maybeCompatible, parseGradleProperties, parseGradlePropertiesFromTaskConfig } from "./lib/jb-helper/jb-helper"; +import { fetchProductReleases, ReleaseItem, releaseItemStr } from "./lib/jb-helper/jb-releases"; +import { getTaskFromArgs } from "./lib/jb-helper/jb-gradle-task-config"; + +$.nothrow(); // git likes to respond with non-zero codes, but it is alright for us + +const task = getTaskFromArgs(true); + +if (task.id !== 1) { + throw new Error(`Only task 1 is supported, got ${task.id}`); +} + +console.log(`Updating nightly editor for ${task.productId} (${task.productType})`); + +const { parsedObj: parsedWorkspaceYaml } = await readWorkspaceYaml(); + +const downloadUrl = parsedWorkspaceYaml.defaultArgs[task.productId + "DownloadUrl"] as string; + +const latestGradle = parseGradleProperties(await Bun.file(pathToBackendPluginGradleLatest).text()); + +const platformVersionType = "build"; + +const releases = await fetchProductReleases({ productCode: task.productCode, productType: task.productType }); + +let maybeCompatibleRelease: ReleaseItem | undefined; +for (const release of releases) { + switch (platformVersionType) { + case "build": { + const ok = maybeCompatible(release, latestGradle); + if (ok) { + maybeCompatibleRelease = release; + break; + } else { + console.error(`${releaseItemStr(release)} incompatible`); + } + } + } + if (maybeCompatibleRelease) { + break; + } +} + +if (maybeCompatibleRelease) { + console.log(`${releaseItemStr(maybeCompatibleRelease)} maybe compatible`); +} + +const targetRelease = maybeCompatibleRelease || releases.find((e) => e.downloads.linux?.link === downloadUrl); + +if (!targetRelease) { + throw new Error(`No compatible release found`); +} + +console.log(`Preparing to use ${releaseItemStr(targetRelease)} as latest version for ${task.productId}`); + +const targetConfig = parseGradlePropertiesFromTaskConfig(task, targetRelease); + +// TODO: actually update nightly editor +console.log( + `Going to exec \`leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${targetConfig.platformVersion} components/ide/jetbrains/image:${task.productId}-latest -DjbBackendVersion=${targetRelease.version}\``, +); diff --git a/components/ide/gha-update-image/index-jb-platform-update.ts b/components/ide/gha-update-image/index-jb-platform-update.ts index 3b0a5fa0104faa..30afd6f7e45101 100644 --- a/components/ide/gha-update-image/index-jb-platform-update.ts +++ b/components/ide/gha-update-image/index-jb-platform-update.ts @@ -2,113 +2,37 @@ // Licensed under the GNU Affero General Public License (AGPL). // See License.AGPL.txt in the project root for license information. -// Update JetBrains Platform Version +// Update JetBrains Plugins (gateway / backend) Platform Version +// +// ``` +// bun run index-jb-platform-update.ts --task= +// ``` -import path from "path"; -import { parseArgs } from "util"; -import { pathToProjectRoot } from "./lib/common"; -import { jbUpdatePlatformVersion } from "./lib/jb-update-platform-version"; +import { getTaskFromArgs } from "./lib/jb-helper/jb-gradle-task-config"; +import { + parseGradleProperties, + renderPropertiesTemplate, + parseGradlePropertiesFromTaskConfig, +} from "./lib/jb-helper/jb-helper"; +import { fetchProductReleases } from "./lib/jb-helper/jb-releases"; -const targetInfo = [ - { - id: 1, - taskName: "Latest Backend Plugin", - productCode: "IIU", - productType: "eap,rc,release", - xmlName: "IntelliJ IDEA", - xmlChannels: ["IC-IU-EAP-licensing-RELEASE", "IC-IU-EAP-licensing-EAP", "IC-IU-RELEASE-licensing-RELEASE"], - useXml: true, - gradlePropertiesPath: path.resolve( - pathToProjectRoot, - "components/ide/jetbrains/backend-plugin/gradle-latest.properties", - ), - gradlePropertiesTemplate: `# Code generated by gha-update-image/index-jb-platform-update.ts. DO NOT EDIT. -# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html -# for insight into build numbers and IntelliJ Platform versions. -# revert pluginSinceBuild if it's unnecessary -pluginSinceBuild={{pluginSinceBuild}} -pluginUntilBuild={{pluginUntilBuild}} -# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl -# See https://jb.gg/intellij-platform-builds-list for available build versions. -pluginVerifierIdeVersions={{pluginVerifierIdeVersions}} -# Version from "com.jetbrains.intellij.idea" which can be found at https://www.jetbrains.com/intellij-repository/snapshots -platformVersion={{platformVersion}} -`, - }, - { - id: 2, - taskName: "Latest Frontend Plugin", - productCode: "GW", - productType: "eap,rc,release", - xmlName: "Gateway", - xmlChannels: ["GW-EAP-licensing-EAP", "GW-RELEASE-licensing-RELEASE", "GW-EAP-licensing-RELEASE"], - useXml: true, - useHumanreadableVersion: true, - gradlePropertiesPath: path.resolve( - pathToProjectRoot, - "components/ide/jetbrains/gateway-plugin/gradle-latest.properties", - ), - gradlePropertiesTemplate: `# Code generated by gha-update-image/index-jb-platform-update.ts. DO NOT EDIT. -# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html -# for insight into build numbers and IntelliJ Platform versions. -# revert pluginSinceBuild if it's unnecessary -pluginSinceBuild={{pluginSinceBuild}} -pluginUntilBuild={{pluginUntilBuild}} -# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl -# See https://jb.gg/intellij-platform-builds-list for available build versions. -pluginVerifierIdeVersions={{pluginVerifierIdeVersions}} -# Version from "com.jetbrains.gateway" which can be found at https://www.jetbrains.com/intellij-repository/snapshots -platformVersion={{platformVersion}} -`, - }, - { - id: 3, - taskName: "Stable Frontend Plugin", - productCode: "GW", - productType: "release", - xmlName: "Gateway", - xmlChannels: ["GW-RELEASE-licensing-RELEASE"], - useXml: true, - useHumanreadableVersion: true, - gradlePropertiesPath: path.resolve( - pathToProjectRoot, - "components/ide/jetbrains/gateway-plugin/gradle-stable.properties", - ), - gradlePropertiesTemplate: `# Code generated by gha-update-image/index-jb-platform-update.ts. DO NOT EDIT. -# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html -# for insight into build numbers and IntelliJ Platform versions. -# revert pluginSinceBuild if it's unnecessary -pluginSinceBuild={{pluginSinceBuild}} -pluginUntilBuild={{pluginUntilBuild}} -# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl -# See https://jb.gg/intellij-platform-builds-list for available build versions. -pluginVerifierIdeVersions={{pluginVerifierIdeVersions}} -# Version from "com.jetbrains.gateway" which can be found at https://www.jetbrains.com/updates/updates.xml -platformVersion={{platformVersion}} -`, - }, -]; +const task = getTaskFromArgs(false); -const { values } = parseArgs({ - args: Bun.argv, - options: { - task: { - type: "string", - }, - }, - strict: true, - allowPositionals: true, -}); +console.log("Updating", task.taskName); -const taskID = Number.parseInt(values.task ?? "NaN"); +const releases = await fetchProductReleases(task); +const newProps = parseGradlePropertiesFromTaskConfig(task, releases[0]); +console.log("New properties info:", newProps); +const oldProps = parseGradleProperties(await Bun.file(task.gradlePropertiesPath).text()); +console.log("Old properties info:", oldProps); -const target = targetInfo.find((e) => e.id === taskID); -if (!target) { - throw new Error( - `Invalid task id: ${taskID}, update cmd with \`--task=""\`, available tasks: \n\t- ${targetInfo - .map((e) => `(${e.id}) ${e.taskName}`) - .join("\n\t- ")}`, - ); +if (newProps.platformVersion === oldProps.platformVersion) { + console.warn("PlatformVersion are the same, no need to update platform version"); + process.exit(0); } -console.log("Updating", target.taskName); -jbUpdatePlatformVersion(target); + +const newGradleContent = renderPropertiesTemplate("gha-update-image/index-jb-platform-update.ts", task, newProps); + +await Bun.write(task.gradlePropertiesPath, newGradleContent); + +console.log("Updated platform version to:", newProps.platformVersion); diff --git a/components/ide/gha-update-image/lib/common.ts b/components/ide/gha-update-image/lib/common.ts index 08fae08e03ef2f..3d5777bfb38e72 100644 --- a/components/ide/gha-update-image/lib/common.ts +++ b/components/ide/gha-update-image/lib/common.ts @@ -53,6 +53,14 @@ export const pathToBackendPluginGradleStable = path.resolve( "components/ide/jetbrains/backend-plugin/gradle-stable.properties", ); +// gradle-latest.properties +export const pathToBackendPluginGradleLatest = path.resolve( + pathToProjectRoot, + "components/ide/jetbrains/backend-plugin/gradle-latest.properties", +); + +export const pathToBackendPluginDir = path.resolve(pathToProjectRoot, "components/ide/jetbrains/backend-plugin"); + // ide-configmap.json export const pathToConfigmap = path.resolve( pathToProjectRoot, diff --git a/components/ide/gha-update-image/lib/jb-helper/jb-gradle-task-config.ts b/components/ide/gha-update-image/lib/jb-helper/jb-gradle-task-config.ts new file mode 100644 index 00000000000000..1e21bbfe46d2b8 --- /dev/null +++ b/components/ide/gha-update-image/lib/jb-helper/jb-gradle-task-config.ts @@ -0,0 +1,124 @@ +import path from "path"; +import { AllProductCodes, AllProductIDs, TargetInfo } from "./jb-helper"; +import { pathToProjectRoot } from "../common"; +import { parseArgs } from "util"; + +export type TaskInfo = TargetInfo & { id: number; taskName: string }; + +export const latestBackendPluginGradleTarget: TaskInfo = { + id: 1, + taskName: "Latest Backend Plugin", + productId: "intellij", + productCode: "IIU", + productType: "eap,rc,release", + usePlatformVersionType: "build", + gradlePropertiesPath: path.resolve( + pathToProjectRoot, + "components/ide/jetbrains/backend-plugin/gradle-latest.properties", + ), + gradlePropertiesTemplate: `# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html +# for insight into build numbers and IntelliJ Platform versions. +# revert pluginSinceBuild if it's unnecessary +pluginSinceBuild={{pluginSinceBuild}} +pluginUntilBuild={{pluginUntilBuild}} +# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl +# See https://jb.gg/intellij-platform-builds-list for available build versions. +pluginVerifierIdeVersions={{pluginVerifierIdeVersions}} +# Version from "com.jetbrains.intellij.idea" which can be found at https://www.jetbrains.com/intellij-repository/snapshots +platformVersion={{platformVersion}} +`, +}; + +export const latestGatewayPluginGradleTarget: TaskInfo = { + id: 2, + taskName: "Latest Frontend Plugin", + productId: "gateway", + productCode: "GW", + productType: "eap,rc,release", + usePlatformVersionType: "version", + gradlePropertiesPath: path.resolve( + pathToProjectRoot, + "components/ide/jetbrains/gateway-plugin/gradle-latest.properties", + ), + gradlePropertiesTemplate: `# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html +# for insight into build numbers and IntelliJ Platform versions. +# revert pluginSinceBuild if it's unnecessary +pluginSinceBuild={{pluginSinceBuild}} +pluginUntilBuild={{pluginUntilBuild}} +# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl +# See https://jb.gg/intellij-platform-builds-list for available build versions. +pluginVerifierIdeVersions={{pluginVerifierIdeVersions}} +# Version from "com.jetbrains.gateway" which can be found at https://www.jetbrains.com/intellij-repository/snapshots +platformVersion={{platformVersion}} +`, +}; + +export const stableGatewayPluginGradleTarget: TaskInfo = { + id: 3, + taskName: "Stable Frontend Plugin", + productId: "gateway", + productCode: "GW", + productType: "release", + usePlatformVersionType: "version", + gradlePropertiesPath: path.resolve( + pathToProjectRoot, + "components/ide/jetbrains/gateway-plugin/gradle-stable.properties", + ), + gradlePropertiesTemplate: `# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html +# for insight into build numbers and IntelliJ Platform versions. +# revert pluginSinceBuild if it's unnecessary +pluginSinceBuild={{pluginSinceBuild}} +pluginUntilBuild={{pluginUntilBuild}} +# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl +# See https://jb.gg/intellij-platform-builds-list for available build versions. +pluginVerifierIdeVersions={{pluginVerifierIdeVersions}} +# Version from "com.jetbrains.gateway" which can be found at https://www.jetbrains.com/updates/updates.xml +platformVersion={{platformVersion}} +`, +}; + +export const AllTasks = [ + latestBackendPluginGradleTarget, + latestGatewayPluginGradleTarget, + stableGatewayPluginGradleTarget, +]; + +export const getTaskFromArgs = (requireProductCode: boolean) => { + const { values } = parseArgs({ + args: Bun.argv, + options: { + task: { + type: "string", + }, + productCode: { + type: "string", + }, + }, + allowPositionals: true, + }); + + const taskID = Number.parseInt(values.task ?? "NaN"); + + const target = AllTasks.find((e) => e.id === taskID); + if (!target) { + throw new Error( + `Invalid task id: ${taskID}, update cmd with \`--task=""\`, available tasks: \n\t- ${AllTasks.map( + (e) => `(${e.id}) ${e.taskName}`, + ).join("\n\t- ")}`, + ); + } + if (requireProductCode && !values.productCode) { + throw new Error(`\`--productCode\` is required, available codes: ${AllProductCodes.join(", ")}`); + } + if (values.productCode) { + const index = AllProductCodes.indexOf(values.productCode as any); + if (index === -1) { + throw new Error( + `Invalid \`--productCode\`: ${values.productCode}, available codes: ${AllProductCodes.join(", ")}`, + ); + } + target.productCode = values.productCode as any; + target.productId = AllProductIDs[index]; + } + return target; +}; diff --git a/components/ide/gha-update-image/lib/jb-helper/jb-helper.ts b/components/ide/gha-update-image/lib/jb-helper/jb-helper.ts new file mode 100644 index 00000000000000..a985163639837b --- /dev/null +++ b/components/ide/gha-update-image/lib/jb-helper/jb-helper.ts @@ -0,0 +1,118 @@ +import semver from "semver"; +import { ReleaseItem } from "./jb-releases"; + +export type UsePlatformVersionType = + | "build" // use `build` field from ReleaseItem + | "version" // use `version` field from ReleaseItem + | "build-snapshot"; // use major `build` with `-EAP-CANDIDATE-SNAPSHOT` + +export const AllProductCodes = ["IIU", "GW", "GO", "PCP", "PS", "RM", "WS", "RD", "CL", "RR"] as const; +export type ProductCodes = (typeof AllProductCodes)[number]; +export const AllProductIDs = [ + "intellij", + "gateway", + "goland", + "pycharm", + "phpstorm", + "rubymine", + "webstorm", + "rider", + "clion", + "resharper", +] as const; +export type ProductIDs = (typeof AllProductIDs)[number]; + +export interface TargetInfo { + /** + * @example intellij goland + */ + productId: ProductIDs; + /** + * @example IIU GO + */ + productCode: ProductCodes; + /** + * @example eap,rc,release + */ + productType: string; + usePlatformVersionType: "build" | "version" | "build-snapshot"; + gradlePropertiesPath: string; + gradlePropertiesTemplate: string; +} + +export interface GradleProperties { + pluginSinceBuild: string; + pluginUntilBuild: string; + pluginVerifierIdeVersions: string; + platformVersion: string; +} + +export const parseGradleProperties = (content: string) => { + const properties: Record = {}; + content.split("\n").forEach((line) => { + if (line.startsWith("#")) { + return; + } + const [key, value] = line.split("="); + if (key && value) { + properties[key.trim()] = value.trim(); + } + }); + return properties as any as GradleProperties; +}; + +export function parseGradlePropertiesFromTaskConfig( + info: Pick, + targetBuild: ReleaseItem, +): GradleProperties { + const build = targetBuild.build; + const buildSem = semver.parse(build); + if (!buildSem) { + throw new Error(`Invalid build version ${build}`); + } + let platformVersion = build; + switch (info.usePlatformVersionType) { + case "build": + platformVersion = build; + break; + case "version": + platformVersion = targetBuild.version; + break; + case "build-snapshot": + platformVersion = `${buildSem.major}.${buildSem.minor}-EAP-CANDIDATE-SNAPSHOT`; + break; + } + return { + pluginSinceBuild: `${buildSem.major}.${buildSem.minor}`, + pluginUntilBuild: `${buildSem.major}.*`, + pluginVerifierIdeVersions: targetBuild.majorVersion, + platformVersion, + }; +} + +export function renderPropertiesTemplate( + whoami: string, + info: Pick, + properties: GradleProperties, +) { + const { gradlePropertiesTemplate } = info; + let newContent = gradlePropertiesTemplate; + Object.entries(properties).forEach(([key, value]) => { + newContent = newContent.replace(`{{${key}}}`, value); + }); + return `# Code generated by ${whoami}. DO NOT EDIT.\n${newContent}`; +} + +export const maybeCompatible = ( + newRelease: Pick, + oldVersion: Pick, +) => { + const sinceBuild = semver.parse(`${oldVersion.pluginSinceBuild}.0`); + const newBuild = semver.parse(newRelease.build); + if (!sinceBuild || !newBuild) { + return false; + } + const versionOK = semver.gte(newBuild, sinceBuild, false); + const majorOK = semver.major(newBuild) === semver.major(sinceBuild); + return versionOK && majorOK; +}; diff --git a/components/ide/gha-update-image/lib/jb-helper/jb-releases.ts b/components/ide/gha-update-image/lib/jb-helper/jb-releases.ts new file mode 100644 index 00000000000000..880c10c7d1bcb4 --- /dev/null +++ b/components/ide/gha-update-image/lib/jb-helper/jb-releases.ts @@ -0,0 +1,43 @@ +import { z } from "zod"; +import axios from "axios"; + +export const productReleaseZod = z.record( + z.string(), + z.array( + z.object({ + date: z.string(), + type: z.string(), + downloads: z.object({ + linux: z + .object({ + link: z.string(), + }) + .optional(), + }), + notesLink: z.string().nullish(), + whatsnew: z.string().nullish(), + majorVersion: z.string(), // 2024.2 + build: z.string(), // 242.20224.159 + version: z.string(), // 2024.2.1 + }), + ), +); + +export type ReleaseItem = z.infer[string][number]; + +export const releaseItemStr = (release: ReleaseItem) => { + return `${release.version}(${release.type},${release.build})`; +}; + +export async function fetchProductReleases(info: { productCode: string; productType: string }) { + const { productCode, productType } = info; + // https://data.services.jetbrains.com/products/releases?code=GW&type=eap,rc,release&platform=linux + const url = `https://data.services.jetbrains.com/products/releases?code=${productCode}&type=${productType}&platform=linux`; + console.log(`Fetching product releases on ${url}`); + const response = await axios.get(url); + const data = productReleaseZod.parse(response.data); + if (!data[productCode] || data[productCode].length <= 0) { + throw new Error(`No data found for ${productCode} in ${url}`); + } + return data[productCode]; +} diff --git a/components/ide/gha-update-image/lib/jb-stable-version.ts b/components/ide/gha-update-image/lib/jb-stable-version.ts index 0d5fef58adeffe..3dd5203ff1dea4 100644 --- a/components/ide/gha-update-image/lib/jb-stable-version.ts +++ b/components/ide/gha-update-image/lib/jb-stable-version.ts @@ -188,6 +188,7 @@ export const getStableVersionsInfo = async (ides: JetBrainsIDE[]) => { return { buildVersion, majorVersion, updatedIDEs }; }; +// TODO: remove me and use jb-gradle-task-config.ts export const upgradeStableVersionsInWorkspaceaAndGradle = async () => { try { const { buildVersion, majorVersion, updatedIDEs } = await getStableVersionsInfo(ides); diff --git a/components/ide/gha-update-image/lib/jb-update-platform-version.ts b/components/ide/gha-update-image/lib/jb-update-platform-version.ts deleted file mode 100644 index 39dd8939a17cd7..00000000000000 --- a/components/ide/gha-update-image/lib/jb-update-platform-version.ts +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) 2024 Gitpod GmbH. All rights reserved. -// Licensed under the GNU Affero General Public License (AGPL). -// See License.AGPL.txt in the project root for license information. - -import axios from "axios"; -import { z } from "zod"; -import semver from "semver"; - -interface TargetInfo { - productCode: string; - productType: string; - xmlName: string; - xmlChannels: string[]; - useXml: boolean; - useHumanreadableVersion: boolean; - gradlePropertiesPath: string; - gradlePropertiesTemplate: string; -} - -// ## Product Release -// - For stable builds https://data.services.jetbrains.com/products/releases?code=IIU&type=eap,rc,release&platform=linux - -const productReleaseZod = z.record( - z.string(), - z.array( - z.object({ - date: z.string(), - type: z.string(), - downloads: z.object({ - linux: z - .object({ - link: z.string(), - }) - .optional(), - }), - notesLink: z.string().nullish(), - whatsnew: z.string().nullish(), - majorVersion: z.string(), // 2024.2 - build: z.string(), // 242.20224.159 - version: z.string(), // 2024.2.1 - }), - ), -); - -async function fetchLatestVersionFromProductReleases(info: TargetInfo) { - const { productCode, productType, useXml, useHumanreadableVersion } = info; - // https://data.services.jetbrains.com/products/releases?code=GW&type=eap,rc,release&platform=linux - const url = `https://data.services.jetbrains.com/products/releases?code=${productCode}&type=${productType}&platform=linux`; - const response = await axios.get(url); - const data = productReleaseZod.parse(response.data); - if (!data[productCode] || data[productCode].length <= 0) { - throw new Error(`No data found for ${productCode} in ${url}`); - } - const latestBuild = data[productCode][0]; - const build = latestBuild.build; - const buildSem = semver.parse(build); - if (!buildSem) { - throw new Error(`Invalid build version ${build}`); - } - const latestPlatformVersion = !useXml ? `${buildSem.major}.${buildSem.minor}-EAP-CANDIDATE-SNAPSHOT` : build; - const latestHumanreadablePlatfromVersion = latestBuild.version; - return { - pluginSinceBuild: `${buildSem.major}.${buildSem.minor}`, - pluginUntilBuild: `${buildSem.major}.*`, - pluginVerifierIdeVersions: latestBuild.majorVersion, - platformVersion: useHumanreadableVersion ? latestHumanreadablePlatfromVersion : latestPlatformVersion, - }; -} - -// ## Updates XML -// - For latest builds (2024.2) https://www.jetbrains.com/updates/updates.xml -// TODO: Parse XML when necessary - -// ## - -export async function jbUpdatePlatformVersion(info: TargetInfo) { - let { gradlePropertiesPath, gradlePropertiesTemplate } = info; - const newBuildInfo = await fetchLatestVersionFromProductReleases(info); - console.log("new build info:", newBuildInfo); - const oldContent = await Bun.file(gradlePropertiesPath).text(); - const regexGroups = /platformVersion=(.*?)$/gm.exec(oldContent); - const oldPlatformVersion = regexGroups ? regexGroups[1] : ""; - if (!oldPlatformVersion) { - throw new Error("Failed to find old platform version"); - } - console.log("old platform version:", oldPlatformVersion); - if (newBuildInfo.platformVersion === oldPlatformVersion) { - console.log("No need to update platform version"); - return; - } - Object.entries(newBuildInfo).forEach(([key, value]) => { - gradlePropertiesTemplate = gradlePropertiesTemplate.replace(`{{${key}}}`, value); - }); - await Bun.write(gradlePropertiesPath, gradlePropertiesTemplate); - console.log("Updated platform version to:", newBuildInfo.platformVersion); -} From 1a29e600e77d7a8044062e2543e8493280aae36a Mon Sep 17 00:00:00 2001 From: Huiwen Date: Sat, 14 Sep 2024 10:20:33 +0000 Subject: [PATCH 2/6] Update GHA with draft script --- .../jetbrains-auto-update-template.yml | 31 ++++++++++++++++++- .../ide/gha-update-image/index-jb-nightly.ts | 6 +++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jetbrains-auto-update-template.yml b/.github/workflows/jetbrains-auto-update-template.yml index 94babe05053dd7..aea4e303ee3230 100644 --- a/.github/workflows/jetbrains-auto-update-template.yml +++ b/.github/workflows/jetbrains-auto-update-template.yml @@ -20,6 +20,35 @@ jobs: with: task: ${{ inputs.productId }} + draft-script: + runs-on: ${{ needs.create-runner.outputs.label }} + container: + image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:pd-test-new-preview-gha.24525 + needs: [ create-runner ] + steps: + - uses: actions/checkout@v2 + - name: Setup Environment + uses: ./.github/actions/setup-environment + with: + identity_provider: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_PROVIDER || secrets.DEV_PREVIEW_PROVIDER }} + service_account: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_SA || secrets.DEV_PREVIEW_SA }} + leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }} + - name: Install dependencies + run: | + cd ./components/ide/gha-update-image/ + yarn + npm i -g bun + - name: Find Nightly Target + id: target + run: | + cd ./components/ide/gha-update-image/ + bun run index-jb-nightly.ts --task=1 --productCode=${{ inputs.productCode }} + + if [ -f /tmp/__gh_output.txt ] + then + cat /tmp/__gh_output.txt >> $GITHUB_OUTPUT + fi + update-jetbrains: runs-on: ${{ needs.create-runner.outputs.label }} container: @@ -55,7 +84,7 @@ jobs: LEEWAY_REMOTE_CACHE_BUCKET: ${{ github.ref == 'refs/heads/main' && 'leeway-cache-main-c514a01' || 'leeway-cache-dev-3ac8ef5' }} run: | imageRepoBase=${{ github.ref == 'refs/heads/main' && 'eu.gcr.io/gitpod-core-dev/build' || 'eu.gcr.io/gitpod-dev-artifact/build' }} - leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.ide-version.outputs.ideBuildVersion }} components/ide/jetbrains/image:${{ inputs.productId }}-latest -DjbBackendVersion=${{ steps.ide-version.outputs.ideVersion }} + echo "leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.ide-version.outputs.ideBuildVersion }} components/ide/jetbrains/image:${{ inputs.productId }}-latest -DjbBackendVersion=${{ steps.ide-version.outputs.ideVersion }}" - name: Get previous job's status id: lastrun uses: filiptronicek/get-last-job-status@main diff --git a/components/ide/gha-update-image/index-jb-nightly.ts b/components/ide/gha-update-image/index-jb-nightly.ts index b344f48dafdc7b..9b4c5b715abd67 100644 --- a/components/ide/gha-update-image/index-jb-nightly.ts +++ b/components/ide/gha-update-image/index-jb-nightly.ts @@ -9,7 +9,7 @@ // ``` import { $ } from "bun"; -import { pathToBackendPluginGradleLatest, readWorkspaceYaml } from "./lib/common"; +import { appendGitHubOutput, pathToBackendPluginGradleLatest, readWorkspaceYaml } from "./lib/common"; import { maybeCompatible, parseGradleProperties, parseGradlePropertiesFromTaskConfig } from "./lib/jb-helper/jb-helper"; import { fetchProductReleases, ReleaseItem, releaseItemStr } from "./lib/jb-helper/jb-releases"; import { getTaskFromArgs } from "./lib/jb-helper/jb-gradle-task-config"; @@ -70,3 +70,7 @@ const targetConfig = parseGradlePropertiesFromTaskConfig(task, targetRelease); console.log( `Going to exec \`leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${targetConfig.platformVersion} components/ide/jetbrains/image:${task.productId}-latest -DjbBackendVersion=${targetRelease.version}\``, ); + +appendGitHubOutput(`buildNumber=${targetConfig.platformVersion}`); +appendGitHubOutput(`image=${task.productId}`); +appendGitHubOutput(`jbBackendVersion=${targetRelease.version}`); From bce6ae488d4d5ea56f2a910bbb6a8639de41c9d0 Mon Sep 17 00:00:00 2001 From: Huiwen Date: Sat, 14 Sep 2024 10:40:12 +0000 Subject: [PATCH 3/6] Update summary --- .../jetbrains-auto-update-template.yml | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/jetbrains-auto-update-template.yml b/.github/workflows/jetbrains-auto-update-template.yml index aea4e303ee3230..8f7f223f7ca3c2 100644 --- a/.github/workflows/jetbrains-auto-update-template.yml +++ b/.github/workflows/jetbrains-auto-update-template.yml @@ -47,6 +47,7 @@ jobs: if [ -f /tmp/__gh_output.txt ] then cat /tmp/__gh_output.txt >> $GITHUB_OUTPUT + cat /tmp/__gh_output.txt >> $GITHUB_STEP_SUMMARY fi update-jetbrains: @@ -85,17 +86,22 @@ jobs: run: | imageRepoBase=${{ github.ref == 'refs/heads/main' && 'eu.gcr.io/gitpod-core-dev/build' || 'eu.gcr.io/gitpod-dev-artifact/build' }} echo "leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.ide-version.outputs.ideBuildVersion }} components/ide/jetbrains/image:${{ inputs.productId }}-latest -DjbBackendVersion=${{ steps.ide-version.outputs.ideVersion }}" - - name: Get previous job's status - id: lastrun - uses: filiptronicek/get-last-job-status@main - - name: Slack Notification - if: ${{ (success() && steps.lastrun.outputs.status == 'failed') || failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }} - SLACK_COLOR: ${{ job.status }} - SLACK_TITLE: ${{ inputs.productId }} - SLACK_FOOTER: "" + echo "leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.ide-version.outputs.ideBuildVersion }} components/ide/jetbrains/image:${{ inputs.productId }}-latest -DjbBackendVersion=${{ steps.ide-version.outputs.ideVersion }}" >> $GITHUB_STEP_SUMMARY + - name: Mock slack content + run: | + content=$(cat $GITHUB_STEP_SUMMARY) + echo "Content from summary: $content" + # - name: Get previous job's status + # id: lastrun + # uses: filiptronicek/get-last-job-status@main + # - name: Slack Notification + # if: ${{ (success() && steps.lastrun.outputs.status == 'failed') || failure() }} + # uses: rtCamp/action-slack-notify@v2 + # env: + # SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }} + # SLACK_COLOR: ${{ job.status }} + # SLACK_TITLE: ${{ inputs.productId }} + # SLACK_FOOTER: "" delete-runner: if: always() From 3632d720a031d9df58a87dc940d7367058d229cf Mon Sep 17 00:00:00 2001 From: Huiwen Date: Sat, 14 Sep 2024 10:54:16 +0000 Subject: [PATCH 4/6] Run leeway build --- .../jetbrains-auto-update-template.yml | 46 +++---------------- .../ide/gha-update-image/index-jb-nightly.ts | 1 + 2 files changed, 7 insertions(+), 40 deletions(-) diff --git a/.github/workflows/jetbrains-auto-update-template.yml b/.github/workflows/jetbrains-auto-update-template.yml index 8f7f223f7ca3c2..821dcbeb6a797b 100644 --- a/.github/workflows/jetbrains-auto-update-template.yml +++ b/.github/workflows/jetbrains-auto-update-template.yml @@ -20,7 +20,7 @@ jobs: with: task: ${{ inputs.productId }} - draft-script: + update-jetbrains: runs-on: ${{ needs.create-runner.outputs.label }} container: image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:pd-test-new-preview-gha.24525 @@ -39,7 +39,7 @@ jobs: yarn npm i -g bun - name: Find Nightly Target - id: target + id: find-target run: | cd ./components/ide/gha-update-image/ bun run index-jb-nightly.ts --task=1 --productCode=${{ inputs.productCode }} @@ -47,50 +47,16 @@ jobs: if [ -f /tmp/__gh_output.txt ] then cat /tmp/__gh_output.txt >> $GITHUB_OUTPUT - cat /tmp/__gh_output.txt >> $GITHUB_STEP_SUMMARY fi - - update-jetbrains: - runs-on: ${{ needs.create-runner.outputs.label }} - container: - image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:pd-test-new-preview-gha.24525 - needs: [ create-runner ] - steps: - - uses: actions/checkout@v2 - - name: Setup Environment - uses: ./.github/actions/setup-environment - with: - identity_provider: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_PROVIDER || secrets.DEV_PREVIEW_PROVIDER }} - service_account: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_SA || secrets.DEV_PREVIEW_SA }} - leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }} - - name: Find downloadUrl - id: download-url - run: | - downloadUrl=$(cat WORKSPACE.yaml | yq r - 'defaultArgs.${{ inputs.productId }}DownloadUrl') - echo "::set-output name=downloadUrl::$downloadUrl" - - name: Find IDE version to download - id: ide-version - run: | - IDE_VERSIONS_JSON=$(bash ./components/ide/jetbrains/image/resolve-latest-ide-version.sh ${{ inputs.productCode }} ${{ steps.download-url.outputs.downloadUrl }}) - IDE_BUILD_VERSION=$(echo "$IDE_VERSIONS_JSON" | jq -r .IDE_BUILD_VERSION) - IDE_VERSION=$(echo "$IDE_VERSIONS_JSON" | jq -r .IDE_VERSION) - echo "IDE_BUILD_VERSION: $IDE_BUILD_VERSION" - echo "IDE_VERSION: $IDE_VERSION" - echo "::set-output name=ideBuildVersion::$IDE_BUILD_VERSION" - echo "::set-output name=ideVersion::$IDE_VERSION" - name: Leeway build - if: ${{ steps.ide-version.outputs.ideBuildVersion }} + if: ${{ steps.find-target.outputs.buildNumber }} env: LEEWAY_MAX_PROVENANCE_BUNDLE_SIZE: "8388608" LEEWAY_REMOTE_CACHE_BUCKET: ${{ github.ref == 'refs/heads/main' && 'leeway-cache-main-c514a01' || 'leeway-cache-dev-3ac8ef5' }} run: | imageRepoBase=${{ github.ref == 'refs/heads/main' && 'eu.gcr.io/gitpod-core-dev/build' || 'eu.gcr.io/gitpod-dev-artifact/build' }} - echo "leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.ide-version.outputs.ideBuildVersion }} components/ide/jetbrains/image:${{ inputs.productId }}-latest -DjbBackendVersion=${{ steps.ide-version.outputs.ideVersion }}" - echo "leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.ide-version.outputs.ideBuildVersion }} components/ide/jetbrains/image:${{ inputs.productId }}-latest -DjbBackendVersion=${{ steps.ide-version.outputs.ideVersion }}" >> $GITHUB_STEP_SUMMARY - - name: Mock slack content - run: | - content=$(cat $GITHUB_STEP_SUMMARY) - echo "Content from summary: $content" + echo "Upgrade latest ${{ inputs.productId }} image with ${{ steps.find-target.outputs.editorSummary }}" >> $GITHUB_STEP_SUMMARY + leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.find-target.outputs.buildNumber }} components/ide/jetbrains/image:${{ steps.find-target.outputs.image }}-latest -DjbBackendVersion=${{ steps.find-target.outputs.jbBackendVersion }} # - name: Get previous job's status # id: lastrun # uses: filiptronicek/get-last-job-status@main @@ -100,7 +66,7 @@ jobs: # env: # SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }} # SLACK_COLOR: ${{ job.status }} - # SLACK_TITLE: ${{ inputs.productId }} + # SLACK_TITLE: Upgrade latest ${{ inputs.productId }} image with ${{ steps.find-target.outputs.editorSummary }} # SLACK_FOOTER: "" delete-runner: diff --git a/components/ide/gha-update-image/index-jb-nightly.ts b/components/ide/gha-update-image/index-jb-nightly.ts index 9b4c5b715abd67..8815a152c36c1c 100644 --- a/components/ide/gha-update-image/index-jb-nightly.ts +++ b/components/ide/gha-update-image/index-jb-nightly.ts @@ -74,3 +74,4 @@ console.log( appendGitHubOutput(`buildNumber=${targetConfig.platformVersion}`); appendGitHubOutput(`image=${task.productId}`); appendGitHubOutput(`jbBackendVersion=${targetRelease.version}`); +appendGitHubOutput(`editorSummary=${releaseItemStr(targetRelease)}`); From 463d5c8c5570232e8d9a79808e2310d513556d33 Mon Sep 17 00:00:00 2001 From: Huiwen Date: Sat, 14 Sep 2024 11:03:44 +0000 Subject: [PATCH 5/6] Fix build and share slack notification --- .../jetbrains-auto-update-template.yml | 22 +++++++++---------- .../lib/jb-helper/jb-helper.ts | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/jetbrains-auto-update-template.yml b/.github/workflows/jetbrains-auto-update-template.yml index 821dcbeb6a797b..05511405c09999 100644 --- a/.github/workflows/jetbrains-auto-update-template.yml +++ b/.github/workflows/jetbrains-auto-update-template.yml @@ -57,17 +57,17 @@ jobs: imageRepoBase=${{ github.ref == 'refs/heads/main' && 'eu.gcr.io/gitpod-core-dev/build' || 'eu.gcr.io/gitpod-dev-artifact/build' }} echo "Upgrade latest ${{ inputs.productId }} image with ${{ steps.find-target.outputs.editorSummary }}" >> $GITHUB_STEP_SUMMARY leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.find-target.outputs.buildNumber }} components/ide/jetbrains/image:${{ steps.find-target.outputs.image }}-latest -DjbBackendVersion=${{ steps.find-target.outputs.jbBackendVersion }} - # - name: Get previous job's status - # id: lastrun - # uses: filiptronicek/get-last-job-status@main - # - name: Slack Notification - # if: ${{ (success() && steps.lastrun.outputs.status == 'failed') || failure() }} - # uses: rtCamp/action-slack-notify@v2 - # env: - # SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }} - # SLACK_COLOR: ${{ job.status }} - # SLACK_TITLE: Upgrade latest ${{ inputs.productId }} image with ${{ steps.find-target.outputs.editorSummary }} - # SLACK_FOOTER: "" + - name: Get previous job's status + id: lastrun + uses: filiptronicek/get-last-job-status@main + - name: Slack Notification + if: ${{ (success() && steps.find-target.outputs.buildNumber) || failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }} + SLACK_COLOR: ${{ job.status }} + SLACK_TITLE: Upgrade latest ${{ inputs.productId }} image with ${{ steps.find-target.outputs.editorSummary }} + SLACK_FOOTER: "" delete-runner: if: always() diff --git a/components/ide/gha-update-image/lib/jb-helper/jb-helper.ts b/components/ide/gha-update-image/lib/jb-helper/jb-helper.ts index a985163639837b..c423d60a84810a 100644 --- a/components/ide/gha-update-image/lib/jb-helper/jb-helper.ts +++ b/components/ide/gha-update-image/lib/jb-helper/jb-helper.ts @@ -18,7 +18,7 @@ export const AllProductIDs = [ "webstorm", "rider", "clion", - "resharper", + "rustrover", ] as const; export type ProductIDs = (typeof AllProductIDs)[number]; From 42ede24297ec065ebaf8da0cf0004944346e4cf0 Mon Sep 17 00:00:00 2001 From: Huiwen Date: Sat, 14 Sep 2024 11:50:09 +0000 Subject: [PATCH 6/6] Add force use dev latest images leeway script --- components/ide/gha-update-image/BUILD.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 components/ide/gha-update-image/BUILD.yaml diff --git a/components/ide/gha-update-image/BUILD.yaml b/components/ide/gha-update-image/BUILD.yaml new file mode 100644 index 00000000000000..9fdc9ceaf9286d --- /dev/null +++ b/components/ide/gha-update-image/BUILD.yaml @@ -0,0 +1,19 @@ +scripts: + - name: jb-use-dev-latest + script: | + ide_list=("intellij" "goland" "pycharm" "phpstorm" "rubymine" "webstorm" "rider" "clion" "rustrover") + prop_list=("latestImage") + + cf_patch=$(kubectl get cm ide-config -o=json | jq '.data."config.json"' |jq -r) + for ide in "${ide_list[@]}"; do + for prop in "${prop_list[@]}"; do + cf_patch=$(echo "$cf_patch" | jq ".ideOptions.options.$ide.$prop = \"eu.gcr.io/gitpod-dev-artifact/build/ide/$ide:latest\"") + done + done + cf_patch=$(echo "$cf_patch" |jq tostring) + cf_patch="{\"data\": {\"config.json\": $cf_patch}}" + # echo "$cf_patch" + + kubectl patch cm ide-config --type=merge -p "$cf_patch" + kubectl rollout restart deployment ide-service + kubectl rollout restart deployment server