Improve download notebook (and fix a bug)

Signed-off-by: m09 <[email protected]>

Author: m09

notebooks/Download repositories.ipynb

@@ -26,15 +26,38 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from logging import getLogger\n",
     "from os import makedirs\n",
     "from os.path import join as path_join\n",
     "\n",
+    "from coloredlogs import install as coloredlogs_install\n",
+    "\n",
+    "\n",
+    "coloredlogs_install()\n",
+    "logger = getLogger(\"downloader\")\n",
+    "\n",
     "\n",
     "git_data_dir = path_join(\"/devfest\", \"repos\", \"git-data\")\n",
     "makedirs(git_data_dir, exist_ok=True)\n",
     "repos_json = path_join(git_data_dir, \"repos.json\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To use GitHub API, we need a token. Please create one in your [GitHub account settings](https://github.com/settings/tokens) (the basic permissions are fine), and fill it here:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "TOKEN = "
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -50,12 +73,6 @@
     "from tqdm import tqdm_notebook as tqdm\n",
     "\n",
     "\n",
-    "# Generate a personal access token here: https://github.com/settings/tokens\n",
-    "TOKEN = # See comment above, please generate a token and put it here\n",
-    "MAX_SIZE = 50 * 1024\n",
-    "REPOS_NUMBER = 50\n",
-    "\n",
-    "\n",
     "next_pattern = re_compile('<(https://api.github.com/user/[^/]+/repos\\?[^>]*page=\\d+[^>]*)>; rel=\"next\"')\n",
     "last_pattern = re_compile('<https://api.github.com/user/[^/]+/repos\\?[^>]*page=(\\d+)[^>]*>; rel=\"last\"')\n",
     "\n",
@@ -70,15 +87,11 @@
     "    return int(match.group(1)) if match is not None else None\n",
     "\n",
     "\n",
-    "def filter_repos(repos: List[Dict[str, Any]], max_size: int, limit: int):\n",
-    "    return [repo\n",
-    "            for repo in sorted(repos,\n",
-    "                               key=itemgetter(\"stars\"),\n",
-    "                               reverse=True)\n",
-    "            if repo[\"size\"] <= max_size][:limit]\n",
-    "\n",
-    "\n",
-    "def list_repositories(user: str, token: str, max_size: int, limit: int):\n",
+    "def list_repositories(user: str,\n",
+    "                      token: str,\n",
+    "                      max_size_mb: int,\n",
+    "                      repos_number: int\n",
+    "                     ) -> List[Dict[str, Any]]:\n",
     "    repos_list_headers = dict(Authorization=\"token %s\" % token)\n",
     "    repos_url = \"https://api.github.com/users/%s/repos\" % user\n",
     "\n",
@@ -90,9 +103,9 @@
     "    def get_page_url(page: int):\n",
     "        return \"%s?page=%d\" % (repos_url, page)\n",
     "\n",
-    "    print(\"Retrieving repos list for user %s\" % user)\n",
+    "    logger.info(\"Retrieving repos list for user %s\" % user)\n",
     "    repos = []\n",
-    "    for page in tqdm(range(total_pages)):\n",
+    "    for page in tqdm(range(1, total_pages + 1)):\n",
     "        request = requests.get(get_page_url(page),\n",
     "                               headers=repos_list_headers)\n",
     "        request.raise_for_status()\n",
@@ -107,8 +120,22 @@
     "                stars=repo[\"stargazers_count\"]\n",
     "            ))\n",
     "\n",
-    "    print(\"Filtering to keep the top %d repositories\" % limit)\n",
-    "    filtered_repos = filter_repos(repos, max_size, limit)\n",
+    "    if max_size_mb is not None:\n",
+    "        logger.info(\n",
+    "            \"Filtering to keep only repositories under %.2f MB\",\n",
+    "            max_size_mb\n",
+    "        )\n",
+    "        repos = [repo for repo in repos\n",
+    "                 if repo[\"size\"] <= max_size_mb * 1024]\n",
+    "\n",
+    "    if repos_number is not None:\n",
+    "        logger.info(\n",
+    "            \"Filtering to keep only the %d most popular repositories\",\n",
+    "            repos_number\n",
+    "        )\n",
+    "        repos = [repo for repo in sorted(repos,\n",
+    "                                         key=itemgetter(\"stars\"),\n",
+    "                                         reverse=True)][:repos_number]\n",
     "\n",
     "    def get_repo_sha_url(user: str, repo: str, branch: str):\n",
     "        return \"https://api.github.com/repos/%s/%s/commits/%s\" % (\n",
@@ -117,12 +144,12 @@
     "            branch\n",
     "        )\n",
     "\n",
-    "    print(\"Getting SHA1 for each repository\")\n",
+    "    logger.info(\"Getting SHA1 for each repository\")\n",
     "    repo_sha_headers = dict(\n",
     "        Authorization=\"token %s\" % token,\n",
     "        Accept = \"application/vnd.github.VERSION.sha\"\n",
     "    )\n",
-    "    for repo in tqdm(filtered_repos):\n",
+    "    for repo in tqdm(repos):\n",
     "        request_sha = requests.get(\n",
     "            get_repo_sha_url(user, repo[\"name\"], repo[\"branch\"]),\n",
     "            headers=repo_sha_headers)\n",
@@ -132,11 +159,20 @@
     "        else:\n",
     "            request_sha.raise_for_status()\n",
     "        repo[\"sha\"] = request_sha.text\n",
-    "    return filtered_repos\n",
+    "    return repos\n",
     "\n",
     "\n",
     "with open(repos_json, \"w\", encoding=\"utf8\") as fh:\n",
-    "    json_dump(list_repositories(\"apache\", TOKEN, MAX_SIZE, REPOS_NUMBER), fh)"
+    "    json_dump(\n",
+    "        list_repositories(\n",
+    "            user=\"apache\",\n",
+    "            # Generate a personal access token here\n",
+    "            # https://github.com/settings/tokens\n",
+    "            token=TOKEN,\n",
+    "            max_size_mb=50,\n",
+    "            repos_number=50),\n",
+    "        fh\n",
+    "    )"
    ]
   },
   {