From 093193d4c600ad18214fbc2f9caed2a3722cec0b Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Sat, 3 May 2025 12:21:53 -0700 Subject: [PATCH 01/12] generalize --- pxtlib/github.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 706299bb0eaa..338d777db0ba 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -1210,6 +1210,15 @@ namespace pxt.github { return null for (const upgr of rules) { + const gh = /^gh:(.*)/.exec(upgr) + if (gh) { + const parsed = parseRepoId(gh[1]) + const repo_s = stringifyRepo(parsed) + pxt.debug(`upgrading ${id} to ${repo_s}}`) + const np : string = upgradedPackageReference(cfg, repo_s) + if (np) return np + else return repo_s + } const m = /^min:(.*)/.exec(upgr) const minV = m && pxt.semver.tryParse(m[1]); if (minV) { From aed333b3f9aa6d1b2dc5a7ae3d031afd5dbf785a Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Mon, 5 May 2025 11:52:07 -0700 Subject: [PATCH 02/12] updates --- pxtlib/github.ts | 3 ++- pxtlib/package.ts | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 338d777db0ba..179c0d8ba0c7 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -722,6 +722,7 @@ namespace pxt.github { p.tag = await db.latestVersionAsync(p.slug, config) } const cached = await db.loadPackageAsync(p.fullName, p.tag) + // TODO: what about min/gh? const dv = upgradedDisablesVariants(config, repoWithTag) if (dv) { const cfg = Package.parseAndValidConfig(cached.files[pxt.CONFIG_NAME]) @@ -1210,7 +1211,7 @@ namespace pxt.github { return null for (const upgr of rules) { - const gh = /^gh:(.*)/.exec(upgr) + const gh = /^move:(.*)/.exec(upgr) if (gh) { const parsed = parseRepoId(gh[1]) const repo_s = stringifyRepo(parsed) diff --git a/pxtlib/package.ts b/pxtlib/package.ts index ae4ec165720d..779941baf8e4 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -654,7 +654,10 @@ namespace pxt { // if we are installing this script, we haven't yet downloaded the config // do upgrade later - if (this.level == 0 && !isInstall) { + // TODO: why only level 0? + //if (this.level == 0 && !isInstall) { + + if (!isInstall) { await this.upgradePackagesAsync(); } From 5de13d39189d091ef49a2e4a05df7b09670c94f5 Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Mon, 5 May 2025 12:56:34 -0700 Subject: [PATCH 03/12] remove whitespace --- pxtlib/github.ts | 4 ++-- pxtlib/package.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 179c0d8ba0c7..3bbc6975068f 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -1216,8 +1216,8 @@ namespace pxt.github { const parsed = parseRepoId(gh[1]) const repo_s = stringifyRepo(parsed) pxt.debug(`upgrading ${id} to ${repo_s}}`) - const np : string = upgradedPackageReference(cfg, repo_s) - if (np) return np + const np: string = upgradedPackageReference(cfg, repo_s) + if (np) return np else return repo_s } const m = /^min:(.*)/.exec(upgr) diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 779941baf8e4..7092029c5e6f 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -675,7 +675,7 @@ namespace pxt { // we are installing the script, and we've download the original version and we haven't upgraded it yet // do upgrade and reload as needed - if (this.level == 0 && isInstall) { + if (isInstall) { const fixes = await this.upgradePackagesAsync(); if (fixes) { From 918c99e649f05f388e0809de87b577c6974a5a92 Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Mon, 5 May 2025 13:25:30 -0700 Subject: [PATCH 04/12] deal with mono repo --- pxtlib/github.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 3bbc6975068f..22562ab99afc 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -1213,8 +1213,15 @@ namespace pxt.github { for (const upgr of rules) { const gh = /^move:(.*)/.exec(upgr) if (gh) { - const parsed = parseRepoId(gh[1]) - const repo_s = stringifyRepo(parsed) + const orig_parsed = parseRepoId(id) + const new_parsed = parseRepoId(gh[1]) + // to deal with mono repo, we should preserve filename if + // new repo doesn't override it + if (orig_parsed.fileName && !new_parsed.fileName) { + new_parsed.fileName = orig_parsed.fileName + new_parsed.fullName = join(new_parsed.owner, new_parsed.project, new_parsed.fileName) + } + const repo_s = stringifyRepo(new_parsed) pxt.debug(`upgrading ${id} to ${repo_s}}`) const np: string = upgradedPackageReference(cfg, repo_s) if (np) return np From 72adfaee095250465babab80892640bbee959493 Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Mon, 5 May 2025 13:36:22 -0700 Subject: [PATCH 05/12] cleanup --- pxtlib/github.ts | 1 - pxtlib/package.ts | 3 --- 2 files changed, 4 deletions(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 22562ab99afc..49f19cbcaf60 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -722,7 +722,6 @@ namespace pxt.github { p.tag = await db.latestVersionAsync(p.slug, config) } const cached = await db.loadPackageAsync(p.fullName, p.tag) - // TODO: what about min/gh? const dv = upgradedDisablesVariants(config, repoWithTag) if (dv) { const cfg = Package.parseAndValidConfig(cached.files[pxt.CONFIG_NAME]) diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 7092029c5e6f..15a5bb35a39c 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -654,9 +654,6 @@ namespace pxt { // if we are installing this script, we haven't yet downloaded the config // do upgrade later - // TODO: why only level 0? - //if (this.level == 0 && !isInstall) { - if (!isInstall) { await this.upgradePackagesAsync(); } From bb83f57a5359ac0cfea5464a6b48e53d021b9eb8 Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Tue, 6 May 2025 15:31:03 -0700 Subject: [PATCH 06/12] simplify form --- pxtlib/github.ts | 20 +++++++++----------- pxtlib/package.ts | 1 + 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 49f19cbcaf60..2262a2e48194 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -1210,17 +1210,15 @@ namespace pxt.github { return null for (const upgr of rules) { - const gh = /^move:(.*)/.exec(upgr) - if (gh) { - const orig_parsed = parseRepoId(id) - const new_parsed = parseRepoId(gh[1]) - // to deal with mono repo, we should preserve filename if - // new repo doesn't override it - if (orig_parsed.fileName && !new_parsed.fileName) { - new_parsed.fileName = orig_parsed.fileName - new_parsed.fullName = join(new_parsed.owner, new_parsed.project, new_parsed.fileName) - } - const repo_s = stringifyRepo(new_parsed) + const move = /^move:([^#\/:]+)\/([^#\/:]+)$/.exec(upgr); + if (move) { + const repo = parseRepoId(id) + repo.owner = move[1] + repo.project = move[2] + repo.slug = join(repo.owner, repo.project) + repo.fullName = join(repo.owner, repo.project, repo.fileName) + repo.tag = undefined + const repo_s = stringifyRepo(repo) pxt.debug(`upgrading ${id} to ${repo_s}}`) const np: string = upgradedPackageReference(cfg, repo_s) if (np) return np diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 15a5bb35a39c..216638da3be0 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -680,6 +680,7 @@ namespace pxt { Object.keys(fixes).forEach(key => pxt.tickEvent("package.doubleload", { "extension": key })) pxt.log(`upgraded, downloading again`); pxt.debug(fixes); + // TODO: we should cache await this.downloadAsync(); } } From 320ed83577aea4e6854bd423e53c8c657cc722cf Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Thu, 8 May 2025 08:38:53 -0700 Subject: [PATCH 07/12] get version tag --- pxtlib/github.ts | 31 ++++++++++++++++++------------- pxtlib/package.ts | 19 ++++++++++++------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 2262a2e48194..54e22b8ee07a 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -1204,25 +1204,30 @@ namespace pxt.github { return null } - export function upgradedPackageReference(cfg: PackagesConfig, id: string) { + export async function upgradedPackageReferenceAsync(cfg: PackagesConfig, id: string) { const rules = upgradeRules(cfg, id) if (!rules) return null for (const upgr of rules) { - const move = /^move:([^#\/:]+)\/([^#\/:]+)$/.exec(upgr); + const move = /^([^#\/:]+)\/([^#\/:]+)$/.exec(upgr); if (move) { - const repo = parseRepoId(id) - repo.owner = move[1] - repo.project = move[2] - repo.slug = join(repo.owner, repo.project) - repo.fullName = join(repo.owner, repo.project, repo.fileName) - repo.tag = undefined - const repo_s = stringifyRepo(repo) - pxt.debug(`upgrading ${id} to ${repo_s}}`) - const np: string = upgradedPackageReference(cfg, repo_s) - if (np) return np - else return repo_s + const owner = move[1]; + const project = move[2]; + const tag = await latestVersionAsync(`github:${owner}/${project}`,cfg) + if (tag) { + const repo = parseRepoId(id) + repo.owner = owner + repo.project = project + repo.slug = join(repo.owner, repo.project) + repo.fullName = join(repo.owner, repo.project, repo.fileName) + repo.tag = tag + const repo_s = stringifyRepo(repo) + pxt.debug(`upgrading ${id} to ${repo_s}}`) + const np: string = await upgradedPackageReferenceAsync(cfg, repo_s) + if (np) return np + else return repo_s + } } const m = /^min:(.*)/.exec(upgr) const minV = m && pxt.semver.tryParse(m[1]); diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 216638da3be0..562b3c17b4ba 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -388,17 +388,22 @@ namespace pxt { .then(packagesConfig => { let numfixes = 0 let fixes: pxt.Map = {}; + const promises: Promise[] = [] U.iterMap(this.config.dependencies, (pkg, ver) => { - if (pxt.github.isGithubId(ver)) { - const upgraded = pxt.github.upgradedPackageReference(packagesConfig, ver) - if (upgraded && upgraded != ver) { - pxt.log(`upgrading dep ${pkg}: ${ver} -> ${upgraded}`); - fixes[ver] = upgraded; - this.config.dependencies[pkg] = upgraded - numfixes++ + const doit = async () => { + if (pxt.github.isGithubId(ver)) { + const upgraded = await pxt.github.upgradedPackageReferenceAsync(packagesConfig, ver) + if (upgraded && upgraded != ver) { + pxt.log(`upgrading dep ${pkg}: ${ver} -> ${upgraded}`); + fixes[ver] = upgraded; + this.config.dependencies[pkg] = upgraded + numfixes++ + } } } + promises.push(doit()) }) + Promise.all(promises) if (numfixes) this.saveConfig() return numfixes && fixes; From 0baa9da621bd31a03308e0ecc39c978db9913135 Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Thu, 8 May 2025 09:40:20 -0700 Subject: [PATCH 08/12] correct format issues --- pxtlib/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 54e22b8ee07a..2fe4c893b61e 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -1210,11 +1210,11 @@ namespace pxt.github { return null for (const upgr of rules) { - const move = /^([^#\/:]+)\/([^#\/:]+)$/.exec(upgr); + const move = /^move:([^#\/:]+)\/([^#\/:]+)$/.exec(upgr); if (move) { const owner = move[1]; const project = move[2]; - const tag = await latestVersionAsync(`github:${owner}/${project}`,cfg) + const tag = await latestVersionAsync(`https://github.com/${owner}/${project}`,cfg) if (tag) { const repo = parseRepoId(id) repo.owner = owner From 36d8319661c42336795b83a2cd082c382d388270 Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Thu, 8 May 2025 11:45:39 -0700 Subject: [PATCH 09/12] correct bad async call --- pxtlib/package.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 562b3c17b4ba..9683f54ab8ac 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -385,7 +385,7 @@ namespace pxt { if (!this.config) this.loadConfig(); return pxt.packagesConfigAsync() - .then(packagesConfig => { + .then(async packagesConfig => { let numfixes = 0 let fixes: pxt.Map = {}; const promises: Promise[] = [] @@ -403,7 +403,7 @@ namespace pxt { } promises.push(doit()) }) - Promise.all(promises) + await Promise.all(promises) if (numfixes) this.saveConfig() return numfixes && fixes; From 6f2a87b457c73d158791189cb1686671069c0d2c Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Thu, 8 May 2025 11:50:07 -0700 Subject: [PATCH 10/12] remove whitespace --- pxtlib/package.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 9683f54ab8ac..e154b59f6715 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -390,7 +390,7 @@ namespace pxt { let fixes: pxt.Map = {}; const promises: Promise[] = [] U.iterMap(this.config.dependencies, (pkg, ver) => { - const doit = async () => { + const doit = async () => { if (pxt.github.isGithubId(ver)) { const upgraded = await pxt.github.upgradedPackageReferenceAsync(packagesConfig, ver) if (upgraded && upgraded != ver) { From f7e6fa92c067b3b4a34fff24ad35c14fa377459f Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Thu, 8 May 2025 12:34:33 -0700 Subject: [PATCH 11/12] more fixes --- pxtlib/github.ts | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 2fe4c893b61e..5914092d5209 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -1210,23 +1210,25 @@ namespace pxt.github { return null for (const upgr of rules) { - const move = /^move:([^#\/:]+)\/([^#\/:]+)$/.exec(upgr); - if (move) { - const owner = move[1]; - const project = move[2]; - const tag = await latestVersionAsync(`https://github.com/${owner}/${project}`,cfg) - if (tag) { + const mv = /^move:(.*)$/.exec(upgr); + if (mv) { + const new_repo = parseRepoId(mv[1]) + if (new_repo) { + if (!new_repo.tag) { + new_repo.tag = await latestVersionAsync(mv[1],cfg) + } const repo = parseRepoId(id) - repo.owner = owner - repo.project = project - repo.slug = join(repo.owner, repo.project) - repo.fullName = join(repo.owner, repo.project, repo.fileName) - repo.tag = tag - const repo_s = stringifyRepo(repo) - pxt.debug(`upgrading ${id} to ${repo_s}}`) - const np: string = await upgradedPackageReferenceAsync(cfg, repo_s) + if (!new_repo.fileName && repo.fileName) { + new_repo.fileName = repo.fileName + new_repo.fullName = join(new_repo.owner, new_repo.project, new_repo.fileName) + } + const new_repo_s = stringifyRepo(new_repo) + pxt.debug(`upgrading ${id} to ${new_repo_s}}`) + const np: string = await upgradedPackageReferenceAsync(cfg, new_repo_s) if (np) return np - else return repo_s + else return new_repo_s + } else { + pxt.log(`cannot parse move target: ${mv[1]}`) } } const m = /^min:(.*)/.exec(upgr) From 1995244cfed06843160c6384155a16bc8167bca8 Mon Sep 17 00:00:00 2001 From: Thomas Ball Date: Fri, 9 May 2025 15:36:49 -0700 Subject: [PATCH 12/12] cleanup --- pxtlib/package.ts | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/pxtlib/package.ts b/pxtlib/package.ts index e154b59f6715..c82e95e8530d 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -657,12 +657,6 @@ namespace pxt { this.parseConfig(str); } - // if we are installing this script, we haven't yet downloaded the config - // do upgrade later - if (!isInstall) { - await this.upgradePackagesAsync(); - } - if (isInstall) { await this.downloadAsync(); if (this.level !== 0 && !this.isAssetPack()) { @@ -675,20 +669,7 @@ namespace pxt { } } - // we are installing the script, and we've download the original version and we haven't upgraded it yet - // do upgrade and reload as needed - if (isInstall) { - const fixes = await this.upgradePackagesAsync(); - - if (fixes) { - // worst case scenario with double load - Object.keys(fixes).forEach(key => pxt.tickEvent("package.doubleload", { "extension": key })) - pxt.log(`upgraded, downloading again`); - pxt.debug(fixes); - // TODO: we should cache - await this.downloadAsync(); - } - } + await this.upgradePackagesAsync() if (appTarget.simulator && appTarget.simulator.dynamicBoardDefinition) { if (this.level == 0) {