Skip to content

Commit 44fd417

Browse files
committed
Revert "aliPublish: bulk-list dist-runtime to avoid per-version S3 calls"
This reverts commit c0f8824.
1 parent c0f8824 commit 44fd417

File tree

1 file changed

+6
-23
lines changed

1 file changed

+6
-23
lines changed

publish/aliPublishS3

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -636,24 +636,6 @@ def sync(pub, architectures, s3Client, bucket, baseUrl, basePrefix, rules,
636636
if nv
637637
)
638638

639-
# Bulk-list all dist-runtime entries upfront to avoid one S3 call per
640-
# package version in process_package below.
641-
t_runtime_start = time()
642-
_rt_prefix = "%s/%s/dist-runtime/" % (basePrefix, arch)
643-
_rt_prefix_len = len(_rt_prefix)
644-
runtimeDepsCache: dict = {}
645-
for _page in s3Client.get_paginator("list_objects_v2").paginate(
646-
Bucket=bucket, Prefix=_rt_prefix,
647-
):
648-
for _item in _page.get("Contents", ()):
649-
if _item["Key"].endswith("/"):
650-
continue
651-
_pkg, _ver_dir, _dep_file = _item["Key"][_rt_prefix_len:].split("/", 2)
652-
runtimeDepsCache.setdefault((_pkg, _ver_dir), []).append(
653-
{"name": _dep_file, "type": "file"}
654-
)
655-
info("TIMING: %s: bulk-listing dist-runtime took %.1fs", arch, time() - t_runtime_start)
656-
657639
# Get versions for all valid packages and filter them according to the rules
658640
def process_package(pkgName):
659641
result = []
@@ -684,13 +666,14 @@ def sync(pub, architectures, s3Client, bucket, baseUrl, basePrefix, rules,
684666

685667
# At this point we have filtered in the package: let's see its dependencies!
686668
# Note that a package always depends on itself (list cannot be empty).
687-
runtimeDeps = runtimeDepsCache.get((pkgName, f"{pkgName}-{pkgVer}"), [])
669+
distPath = f"{arch}/dist-runtime/{pkgName}/{pkgName}-{pkgVer}"
670+
runtimeDeps = list(listDir(distPath))
688671
if not runtimeDeps:
689-
error("%s / %s / %s: cannot list dependencies from dist-runtime: skipping",
690-
arch, pkgName, pkgVer)
672+
error("%s / %s / %s: cannot list dependencies from %s: skipping",
673+
arch, pkgName, pkgVer, distPath)
691674
continue
692-
debug("%s / %s / %s: listing all dependencies from dist-runtime cache",
693-
arch, pkgName, pkgVer)
675+
debug("%s / %s / %s: listing all dependencies under %s",
676+
arch, pkgName, pkgVer, distPath)
694677
for depTar in runtimeDeps:
695678
if depTar["type"] != "file":
696679
continue

0 commit comments

Comments
 (0)