Merge pull request #16 from m09/refactor-path-and-download

Refactor the paths used and change download notebook

Author: m09

Makefile

@@ -30,7 +30,7 @@ gitbase: bblfshd
 			--link devfest_bblfshd:devfest_bblfshd \
 			--env BBLFSH_ENDPOINT=devfest_bblfshd:9432 \
 			--env MAX_MEMORY=1024 \
-			--volume $(PWD)/repos:/opt/repos \
+			--volume $(PWD)/repos/git-data:/opt/repos \
 			srcd/gitbase:v0.24.0-rc2
 
 jupyter-image:

README.md

@@ -5,10 +5,11 @@ Workshop given at [DevFest Nantes 2019](https://devfest.gdgnantes.com/sessions/u
 Slides: on [gDrive](https://docs.google.com/presentation/d/1vF0JMagmXXzn-h-OaJu6CsDt78oSQSg58YFJsBUaHxk/edit#slide=id.g4f0d75b8b4_0_0)
 
 OSS tools covered:
- - [gitbase](https://docs.sourced.tech/gitbase)
- - [bblfsh](https://doc.bblf.sh)
- - [BigARTM](http://bigartm.org)
- - [OpenNMT](http://opennmt.net)
+
+- [gitbase](https://docs.sourced.tech/gitbase)
+- [bblfsh](https://doc.bblf.sh)
+- [BigARTM](http://bigartm.org)
+- [OpenNMT](http://opennmt.net)
 
  <details>
 <summary>Abstract</summary>
@@ -32,8 +33,10 @@ OSS tools covered:
 
 </details>
 
+Slides: on [gDrive](https://docs.google.com/presentation/d/1vF0JMagmXXzn-h-OaJu6CsDt78oSQSg58YFJsBUaHxk/edit#slide=id.g4f0d75b8b4_0_0)
 
 ## Prerequisites
+
 - Docker
 
 ## Dependencies
@@ -72,7 +75,7 @@ docker run \
     --link devfest_bblfshd:devfest_bblfshd \
     --env BBLFSH_ENDPOINT=devfest_bblfshd:9432 \
     --env MAX_MEMORY=1024 \
-    --volume $(pwd)/repos:/opt/repos \
+    --volume $(pwd)/repos/git-data:/opt/repos \
     srcd/gitbase:v0.24.0-rc2
 ```
 
@@ -104,8 +107,8 @@ To only launch the 3 required containers
 ```shell
 make
 ```
-</details>
 
+</details>
 
 ## Workflow
 
@@ -121,10 +124,8 @@ TBD
 
 [Notebook 2: project and developer similarities](#link to local jupyther)
 
-
 ### 3. Function Name Suggestion
 
 TBD
 
 [Notebook 2: function name suggestion](#link to local jupyther)
-

notebooks/.gitignore

@@ -1,3 +1,2 @@
 .ipynb_checkpoints
 __pycache__
-full

notebooks/Download repositories.ipynb

@@ -20,79 +20,123 @@
     "We iterate as long as the API gives us a pointer to another response page. We filter forks to focus on original repositories."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from os import makedirs\n",
+    "from os.path import join as path_join\n",
+    "\n",
+    "\n",
+    "git_data_dir = path_join(\"/devfest\", \"repos\", \"git-data\")\n",
+    "makedirs(git_data_dir, exist_ok=True)\n",
+    "repos_json = path_join(git_data_dir, \"repos.json\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "from json import dump as json_dump\n",
+    "from operator import itemgetter\n",
     "from re import compile as re_compile\n",
-    "from typing import Optional\n",
+    "from typing import Any, Dict, List, Optional\n",
     "\n",
     "import requests\n",
+    "from tqdm import tqdm_notebook as tqdm\n",
     "\n",
     "\n",
     "# Generate a personal access token here: https://github.com/settings/tokens\n",
     "TOKEN = # See comment above, please generate a token and put it here\n",
+    "MAX_SIZE = 50 * 1024\n",
+    "REPOS_NUMBER = 50\n",
     "\n",
     "\n",
     "next_pattern = re_compile('<(https://api.github.com/user/[^/]+/repos\\?[^>]*page=\\d+[^>]*)>; rel=\"next\"')\n",
+    "last_pattern = re_compile('<https://api.github.com/user/[^/]+/repos\\?[^>]*page=(\\d+)[^>]*>; rel=\"last\"')\n",
     "\n",
     "\n",
     "def parse_next(link_header: str) -> Optional[str]:\n",
     "    match = next_pattern.search(link_header)\n",
     "    return match.group(1) if match is not None else None\n",
     "\n",
     "\n",
-    "def list_repositories(user: str):\n",
-    "    headers = dict(Authorization=\"token {token}\".format(token=TOKEN))\n",
-    "    url = \"https://api.github.com/users/{user}/repos\".format(user=user)\n",
-    "    while url is not None:\n",
-    "        request = requests.get(url, headers=headers)\n",
-    "        request.raise_for_status()\n",
-    "        for repo in request.json():\n",
-    "            if not repo[\"fork\"]:\n",
-    "                yield repo[\"name\"], repo[\"clone_url\"], repo[\"size\"], repo[\"stargazers_count\"]\n",
-    "        url = parse_next(request.headers[\"Link\"])\n",
-    "\n",
-    "\n",
-    "with open('output/repos.json', 'w') as fh:\n",
-    "    json_dump(list(list_repositories(\"apache\")), fh)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Filtering for repos we want to analyze\n",
+    "def parse_last(link_header: str) -> Optional[int]:\n",
+    "    match = last_pattern.search(link_header)\n",
+    "    return int(match.group(1)) if match is not None else None\n",
     "\n",
-    "We'll keep the most popular repos by stars that are under a given size threshold."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from json import load as json_load\n",
-    "from operator import itemgetter\n",
-    "from pprint import pprint\n",
     "\n",
+    "def filter_repos(repos: List[Dict[str, Any]], max_size: int, limit: int):\n",
+    "    return [repo\n",
+    "            for repo in sorted(repos,\n",
+    "                               key=itemgetter(\"stars\"),\n",
+    "                               reverse=True)\n",
+    "            if repo[\"size\"] <= max_size][:limit]\n",
     "\n",
-    "MAX_SIZE = 50 * 1024\n",
     "\n",
+    "def list_repositories(user: str, token: str, max_size: int, limit: int):\n",
+    "    repos_list_headers = dict(Authorization=\"token %s\" % token)\n",
+    "    repos_url = \"https://api.github.com/users/%s/repos\" % user\n",
     "\n",
-    "filtered = []\n",
-    "with open('output/repos.json', 'r') as fh:\n",
-    "    repos = json_load(fh)\n",
-    "    filtered = [(name, clone_url)\n",
-    "                for name, clone_url, size, _ in sorted(repos, key=itemgetter(3), reverse=True)\n",
-    "                if size <= MAX_SIZE]\n",
+    "    request_total = requests.get(repos_url,\n",
+    "                                 headers=repos_list_headers)\n",
+    "    total_pages = parse_last(request_total.headers[\"Link\"])\n",
+    "    assert total_pages is not None\n",
     "\n",
+    "    def get_page_url(page: int):\n",
+    "        return \"%s?page=%d\" % (repos_url, page)\n",
     "\n",
-    "pprint(filtered)"
+    "    print(\"Retrieving repos list for user %s\" % user)\n",
+    "    repos = []\n",
+    "    for page in tqdm(range(total_pages)):\n",
+    "        request = requests.get(get_page_url(page),\n",
+    "                               headers=repos_list_headers)\n",
+    "        request.raise_for_status()\n",
+    "        for repo in request.json():\n",
+    "            if repo[\"fork\"]:\n",
+    "                continue\n",
+    "            repos.append(dict(\n",
+    "                name=repo[\"name\"],\n",
+    "                branch=repo[\"default_branch\"],\n",
+    "                clone_url=repo[\"clone_url\"],\n",
+    "                size=repo[\"size\"],\n",
+    "                stars=repo[\"stargazers_count\"]\n",
+    "            ))\n",
+    "\n",
+    "    print(\"Filtering to keep the top %d repositories\" % limit)\n",
+    "    filtered_repos = filter_repos(repos, max_size, limit)\n",
+    "\n",
+    "    def get_repo_sha_url(user: str, repo: str, branch: str):\n",
+    "        return \"https://api.github.com/repos/%s/%s/commits/%s\" % (\n",
+    "            user,\n",
+    "            repo,\n",
+    "            branch\n",
+    "        )\n",
+    "\n",
+    "    print(\"Getting SHA1 for each repository\")\n",
+    "    repo_sha_headers = dict(\n",
+    "        Authorization=\"token %s\" % token,\n",
+    "        Accept = \"application/vnd.github.VERSION.sha\"\n",
+    "    )\n",
+    "    for repo in tqdm(filtered_repos):\n",
+    "        request_sha = requests.get(\n",
+    "            get_repo_sha_url(user, repo[\"name\"], repo[\"branch\"]),\n",
+    "            headers=repo_sha_headers)\n",
+    "        if request_sha.status_code == 409:\n",
+    "            # Repo is empty\n",
+    "            continue\n",
+    "        else:\n",
+    "            request_sha.raise_for_status()\n",
+    "        repo[\"sha\"] = request_sha.text\n",
+    "    return filtered_repos\n",
+    "\n",
+    "\n",
+    "with open(repos_json, \"w\", encoding=\"utf8\") as fh:\n",
+    "    json_dump(list_repositories(\"apache\", TOKEN, MAX_SIZE, REPOS_NUMBER), fh)"
    ]
   },
   {
@@ -101,19 +145,28 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from json import load as json_load\n",
     "from multiprocessing.pool import ThreadPool\n",
     "\n",
     "\n",
     "PARALLEL_DOWNLOADS = 10\n",
-    "REPOS_NUMBER = 50\n",
     "\n",
     "\n",
-    "def clone_repo(clone_url: str):\n",
-    "    !cd /devfest/repos && git clone -q {clone_url}\n",
+    "def clone_repo(name: str, clone_url: str, sha):\n",
+    "    !cd {git_data_dir} \\\n",
+    "        && git clone -q {clone_url} {name} \\\n",
+    "        && cd {name} \\\n",
+    "        && git checkout -q {sha}\n",
     "\n",
     "\n",
-    "with ThreadPool(PARALLEL_DOWNLOADS) as pool:\n",
-    "    pool.map(clone_repo, [clone_url for _, clone_url in filtered[:REPOS_NUMBER]])"
+    "with ThreadPool(PARALLEL_DOWNLOADS) as pool, \\\n",
+    "        open(repos_json, encoding=\"utf8\") as fh:\n",
+    "    repos = json_load(fh)\n",
+    "    pool.starmap(clone_repo,\n",
+    "                 [(repo[\"name\"],\n",
+    "                   repo[\"clone_url\"],\n",
+    "                   repo[\"sha\"])\n",
+    "                  for repo in repos])"
    ]
   }
  ],