Skip to content

Commit 1512441

Browse files
CarreauCarreau
committed
config file
1 parent 037bc07 commit 1512441

File tree

2 files changed

+146
-11
lines changed

2 files changed

+146
-11
lines changed

‎all_repos.txt

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
jupyter-incubator/sparkmagic : https://pypi.org/project/sparkmagic
2+
jupyter-widgets/ipywidgets : https://pypi.org/project/ipywidgets
3+
jupyter-server/jupyter-resource-usage : https://pypi.org/project/jupyter-resource-usage
4+
jupyter-server/jupyter_server : https://pypi.org/project/jupyter-server
5+
jupyter-server/pytest-jupyter : https://pypi.org/project/pytest-jupyter
6+
jupyter-server/jupyter_server_mathjax : https://pypi.org/project/jupyter-server-mathjax
7+
jupyter-server/gateway_provisioners : https://pypi.org/project/gateway-provisioners
8+
jupyter-server/jupyter_releaser : https://pypi.org/project/jupyter-releaser
9+
jupyter-server/fps : https://pypi.org/project/fps
10+
jupyter-server/jupyter_server_terminals : https://pypi.org/project/jupyter-server-terminals
11+
jupyter-server/jupyter_ydoc : https://pypi.org/project/jupyter-ydoc
12+
jupyter-server/jupyter_server_fileid : https://pypi.org/project/jupyter-server-fileid
13+
jupyter-server/jupyter-scheduler : https://pypi.org/project/jupyter-scheduler
14+
jupyter-server/pycrdt : https://pypi.org/project/pycrdt
15+
jupyter-server/pycrdt-websocket : https://pypi.org/project/pycrdt-websocket
16+
ipython/ipython : https://pypi.org/project/ipython
17+
ipython/traitlets : https://pypi.org/project/traitlets
18+
ipython/ipython_genutils : https://pypi.org/project/ipython_genutils
19+
ipython/ipyparallel : https://pypi.org/project/ipyparallel
20+
ipython/ipykernel : https://pypi.org/project/ipykernel
21+
ipython/ipynb : https://pypi.org/project/ipynb
22+
ipython/rlipython : https://pypi.org/project/rlipython
23+
ipython/disp : https://pypi.org/project/disp
24+
ipython/matplotlib-inline : https://pypi.org/project/matplotlib-inline
25+
ipython/comm : https://pypi.org/project/comm
26+
ipython/ipython-pygments-lexers : https://pypi.org/project/ipython-pygments-lexers
27+
jupyterhub/escapism : https://pypi.org/project/escapism
28+
jupyterhub/pamela : https://pypi.org/project/pamela
29+
jupyterhub/autodoc-traits : https://pypi.org/project/autodoc-traits
30+
jupyter-book/jupyter-book : https://pypi.org/project/jupyter-book
31+
jupyter/nbviewer : https://pypi.org/project/nbviewer
32+
jupyter/nbgrader : https://pypi.org/project/nbgrader
33+
jupyter/terminado : https://pypi.org/project/terminado
34+
jupyter/jupyter_core : https://pypi.org/project/jupyter-core
35+
jupyter/nbformat : https://pypi.org/project/nbformat
36+
jupyter/jupyter_client : https://pypi.org/project/jupyter-client
37+
jupyter/notebook : https://pypi.org/project/notebook
38+
jupyter/nbconvert : https://pypi.org/project/nbconvert
39+
jupyter/jupyter_console : https://pypi.org/project/jupyter-console
40+
jupyter/jupyter_kernel_test : https://pypi.org/project/jupyter-kernel-test
41+
jupyter/jupyter : https://pypi.org/project/jupyter
42+
jupyter/nbdime : https://pypi.org/project/nbdime
43+
jupyter/jupyterlab_geojson : https://pypi.org/project/jupyterlab-geojson
44+
jupyter/jupyter-sphinx : https://pypi.org/project/jupyter-sphinx
45+
jupyter/jupyter-packaging : https://pypi.org/project/jupyter-packaging
46+
jupyter/jupyterlab-fasta : https://pypi.org/project/jupyterlab-fasta
47+
jupyter/nbclassic : https://pypi.org/project/nbclassic
48+
jupyter/nbclient : https://pypi.org/project/nbclient
49+
jupyter/papyri : https://pypi.org/project/papyri
50+
jupyter/notebook_shim : https://pypi.org/project/notebook-shim
51+
jupyter/jupyter_events : https://pypi.org/project/jupyter-events
52+
jupyterlab/jupyterlab : https://pypi.org/project/jupyterlab
53+
jupyterlab/pytest-check-links : https://pypi.org/project/pytest-check-links
54+
jupyterlab/jupyterlab_server : https://pypi.org/project/jupyterlab-server
55+
jupyterlab/jupyterlab-git : https://pypi.org/project/jupyterlab-git
56+
jupyterlab/jupyterlab-github : https://pypi.org/project/jupyterlab-github
57+
jupyterlab/jupyterlab-latex : https://pypi.org/project/jupyterlab-latex
58+
jupyterlab/jupyterlab-telemetry : https://pypi.org/project/jupyterlab-telemetry
59+
jupyterlab/jupyterlab_pygments : https://pypi.org/project/jupyterlab-pygments
60+
jupyterlab/jupyterlab_apod : https://pypi.org/project/jupyterlab-apod
61+
jupyterlab/jupyterlab-plugin-playground : https://pypi.org/project/jupyterlab-plugin-playground
62+
jupyterlab/jupyterlab-translate : https://pypi.org/project/jupyterlab-translate
63+
jupyterlab/hatch-jupyter-builder : https://pypi.org/project/hatch-jupyter-builder
64+
jupyterlab/jupyter-collaboration : https://pypi.org/project/jupyter-collaboration
65+
66+
# All these are from the same repo:
67+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-ar-SA
68+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-ca-ES
69+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-cs-CZ
70+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-da-DK
71+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-de-DE
72+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-el-GR
73+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-es-ES
74+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-et-EE
75+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-fi-FI
76+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-fr-FR
77+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-he-IL
78+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-hu-HU
79+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-hy-AM
80+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-id-ID
81+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-it-IT
82+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-ja-JP
83+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-ko-KR
84+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-lt-LT
85+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-nl-NL
86+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-no-NO
87+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-pl-PL
88+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-pt-BR
89+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-ro-RO
90+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-ru-RU
91+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-tr-TR
92+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-uk-UA
93+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-vi-VN
94+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-zh-CN
95+
jupyterlab/language-packs : https://pypi.org/project/jupyterlab-language-pack-zh-TW
96+
97+
# repo with no packages:
98+
https://github.com/binderhub-ci-repos/lfs:
99+
https://github.com/jupyter/design:

‎tools/all_repos.py

Lines changed: 47 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -132,38 +132,75 @@ async def get_package_maintainers(package: str) -> list[str]:
132132
"""
133133
url = f"https://pypi.org/project/{package}/"
134134
if package in cache:
135+
print("c", end="", flush=True)
135136
return cache[package]
136137
response = await asks.get(url)
137138
if response.status_code == 200:
138139
html = response.text
139140
soup = BeautifulSoup(html, "html.parser")
140141
maintainers = soup.find_all("span", class_="sidebar-section__maintainer")
141142
if not maintainers:
143+
print("x", end="", flush=True)
142144
return set(["unknown (blocked by fastly?)"])
143145
res = set(a.text.strip() for a in maintainers)
144146
cache[package] = res
147+
print(".", end="", flush=True)
145148
return res
149+
print("f", end="", flush=True)
146150
return set(["unknown (status code: " + str(response.status_code) + ")"])
147151

148152

149-
async def main():
153+
async def main(config_file: str = "all_repos.txt"):
154+
from pathlib import Path
155+
156+
items = Path(config_file).read_text().splitlines()
157+
known_mapping = []
158+
for item in items:
159+
if item.startswith("#") or not item.strip():
160+
continue
161+
github_name, pypi_name = item.split(":", maxsplit=1)
162+
# pypi name may be empty for repo with no packages.
163+
# and one repo can create multiple pypi packages.
164+
known_mapping.append((github_name.strip(), pypi_name.strip()))
165+
166+
# get all packages in the pypi jupyter org
150167
packages = get_packages(f"https://pypi.org/org/jupyter/")
168+
packages_urls = [f"https://pypi.org/project/{p}" for p in packages]
151169
print(f"Found {len(packages)} packages in the pypi jupyter org")
152170

171+
missing_from_pypi_org = set([p for _, p in known_mapping]) - set(packages_urls)
172+
if missing_from_pypi_org:
173+
print(
174+
"Repos missing from pypi org – they are listed on the config file, with a corresponding Pypi pacakge, but the package is not part of Pypi org:"
175+
)
176+
for repo in missing_from_pypi_org:
177+
print(f" {repo}")
178+
179+
missing_from_github_org = set(packages_urls) - set([p for _, p in known_mapping])
180+
if missing_from_github_org:
181+
print(
182+
"Packages missing from github org, they are on PyPI, but I don't know the source github repo...:"
183+
)
184+
for repo in sorted(missing_from_github_org):
185+
print(f" {repo}")
186+
153187
map = {p.lower().replace("-", "_"): p for p in packages}
154188

155189
todo = []
156190
async for org, repo in list_repos(default_orgs):
157-
lowname = repo.lower().replace("-", "_")
158-
if lowname in map:
159-
print(
160-
f"{org}/{repo}".ljust(40),
161-
f"https://pypi.org/project/{map[lowname]}",
162-
" in jupyter org",
163-
)
164-
del map[lowname]
165-
else:
191+
org_repo = f"{org}/{repo}"
192+
candidates = [v for k, v in known_mapping if k == org_repo]
193+
if not candidates:
194+
print(f"Missing: no candidate for {org_repo}")
166195
todo.append((org, repo))
196+
continue
197+
for candidate in candidates:
198+
if candidate in packages_urls:
199+
pass
200+
# print(f"OK: {org_repo} -> {candidate}"")
201+
else:
202+
print(f"Missing: {org_repo} -> {candidate}")
203+
todo.append((org, repo))
167204

168205
print()
169206
print("check potentially matching Pypi names:")
@@ -186,7 +223,6 @@ async def _loc(targets, org, repo):
186223
maintainers,
187224
)
188225
)
189-
print(".", end="", flush=True)
190226

191227
nursery.start_soon(_loc, targets, org, repo)
192228

0 commit comments

Comments
 (0)