updats

Author: Carreau

all_repos.txt

@@ -1,3 +1,4 @@
+# github.repo : pypi url (or empty)
 jupyter-incubator/sparkmagic             : https://pypi.org/project/sparkmagic
 jupyter-widgets/ipywidgets               : https://pypi.org/project/ipywidgets
 jupyter-server/jupyter-resource-usage    : https://pypi.org/project/jupyter-resource-usage
@@ -97,3 +98,67 @@ jupyterlab/language-packs :  https://pypi.org/project/jupyterlab-language-pack-z
 # repo with no packages:
 https://github.com/binderhub-ci-repos/lfs:
 https://github.com/jupyter/design:
+
+# security forks:
+# is it safe to list those URL ? 
+# REDACTED
+
+# .github repos
+jupyter-standard/.github:
+jupyter-server/.github:
+jupyter-xeus/.github:
+jupyter-standards/.github:
+ipython/.github:
+ipython/.github-1:
+jupyterhub/.github:
+voila-dashboards/.github:
+jupyter/.github:
+jupyterlab/.github:
+
+# .github.io repos
+ipython/ipython.github.com:
+jupyter-xeus/jupyter-xeus.github.io:
+jupyterhub/jupyterhub.github.io:
+voila-gallery/voila-gallery.github.io:
+voila-dashboards/voila-dashboards.github.io:
+jupyter/jupyter.github.io:
+
+# team compass
+jupyter/docs-team-compass:
+jupyter/foundations-and-standards-team-compass:
+jupyter/ipython-components:
+jupyter/kernels-team-compass:
+jupyter/notebook-team-compass:
+jupyter/software-steering-council-team-compass:
+jupyter/team-compass-template:
+jupyter-book/team-compass:
+jupyter-governance/ec-team-compass:
+jupyter-server/team-compass:
+jupyter-widgets/team-compass:
+jupyterhub/team-compass:
+jupyterlab/frontends-team-compass:
+
+# misc websites:
+
+jupyter/cdn.jupyter.org:
+jupyter/colaboratory.jupyter.org:
+jupyter/nbviewer.org-deploy:
+jupyter/try.jupyter.org:
+jupyter-book/mystmd.org:
+jupyterhub/mybinder.org-deploy:
+jupyterhub/mybinder.org-user-guide:
+voila-gallery/voila-gallery.org-deploy:
+
+# various tutorials
+jupyter/ngcm-tutorial:
+jupyter/scipy-advanced-tutorial:
+jupyter/strata-sv-2015-tutorial:
+jupyter/tutorial-dashboards-declarativewidgets:
+jupyter/tutorial-devteam-jupyterhub-2017:
+jupyter/tutorial-devteam-projectmgt-2017:
+jupyter-widgets/tutorial:
+jupyter-widgets/tutorial-jupyterlite:
+jupyterhub/jupyterhub-tutorial:
+jupyterlab/jupytercon-jupyterlab-tutorial:
+jupyterlab/scipy2018-jupyterlab-tutorial:
+jupyterlab/scipy2019-jupyterlab-tutorial:

tools/all_repos.py

@@ -184,18 +184,32 @@ async def main(config_file: str = "all_repos.txt"):
         for repo in sorted(missing_from_github_org):
             print(f"  {repo}")
 
-    map = {p.lower().replace("-", "_"): p for p in packages}
-
     todo = []
+    # now we loop over all the org/repo and check if:
+    # - it is in the jupyter org:
+    # - it has a pypi package (or not) in the mapping, if not,
+    #   it's ok it's not supposed to have one.
+    # - if it has:
+    #   all repos in the mapping should be in the Pypi org
+    # - if not, add to todo list.
+
     async for org, repo in list_repos(default_orgs):
         org_repo = f"{org}/{repo}"
+        listed = [k for k, _ in known_mapping]
+        if not listed:
+            # not listed in config. We search by default.
+            todo.append((org, repo))
+            continue
         candidates = [v for k, v in known_mapping if k == org_repo]
         if not candidates:
-            print(f"Missing: no candidate for {org_repo}")
+            # not listed in config. We search by default.
             todo.append((org, repo))
             continue
         for candidate in candidates:
-            if candidate in packages_urls:
+            if candidate == "":
+                continue
+                # not supposed to have a Pypi package
+            elif candidate in packages_urls:
                 pass
                 # print(f"OK: {org_repo} -> {candidate}"")
             else:
@@ -207,7 +221,7 @@ async def main(config_file: str = "all_repos.txt"):
 
     async with trio.open_nursery() as nursery:
         targets = []
-        semaphore = trio.Semaphore(10)  # Throttle to 10 concurrent requests
+        semaphore = trio.Semaphore(15)  # Throttle to 10 concurrent requests
         for org, repo in todo:
 
             async def _loc(targets, org, repo):
@@ -252,9 +266,6 @@ async def _loc(targets, org, repo):
             corg = org
         if status != 200:
             print(f"https://github.com/{org}/{repo}")
-    print()
-    print("Packages with no repos.")
-    print(map)
 
 
 trio.run(main)