Added --issues flag // #1

Author: lourot

.npmignore

@@ -1,3 +1,4 @@
+/.ghuser.io.json
 /.nyc_output/
 /.travis.yml
 /coverage/

README.md

@@ -16,7 +16,7 @@ List **all** GitHub repos a user has contributed to **since the beginning of tim
 $ github-contribs AurelienLourot
 ✔ Fetched first day at GitHub: 2015-04-04.
 ⚠ Be patient. The whole process might take up to an hour... Consider using --since and/or --until
-✔ Fetched all commits and PRs.
+✔ Fetched all commits and PRs. Consider using --issues to fetch issues as well.
 35 repo(s) found:
 AurelienLourot/lsankidb
 reframejs/reframe
@@ -65,21 +65,44 @@ Instead we noticed that the "Contribution Activity" on the
 * https://github.com/users/AurelienLourot/created_repositories?from=2018-05-17&to=2018-05-17
 * https://github.com/users/AurelienLourot/created_pull_requests?from=2018-05-17&to=2018-05-17
 * https://github.com/users/AurelienLourot/created_pull_request_reviews?from=2018-05-17&to=2018-05-17
+* https://github.com/users/AurelienLourot/created_issues?from=2018-07-10&to=2018-07-10
 
 So we're doing the same :)
 
+> **NOTE**: it seems like `created_issues` URLs don't deliver "hot issues" (issues which received
+> more comments than others):
+>
+> ```bash
+> $ curl -s "https://github.com/users/AurelienLourot/created_issues?from=2015-09-23&to=2015-09-23"
+> <div class="profile-rollup-content">
+> </div>
+> ```
+>
+> To get these, we also query the profile itself:
+>
+> ```bash
+> $ curl -s "https://github.com/AurelienLourot?from=2015-09-23" | grep issues/
+>         <a class="text-gray-dark" href="/jfrog/build-info/issues/60">Publish properties aren&#39;t used by build-info-extractor-gradle?</a>
+> ```
+
 ### Why is it so slow?
 
 We hit a [rate limit](https://en.wikipedia.org/wiki/Rate_limiting). And since it's not an official
 API, we can't use a token to raise the limit.
 
 ### Isn't it likely to break?
 
-Yes, it is since that interface isn't public. We're monitoring it and will react as fast as we can
-when it breaks.
+Yes, it is since that interface isn't public. We're monitoring it<sup>[1](#footnote1)</sup> and will
+react as fast as we can when it breaks.
+
+<a name="footnote1"><sup>1</sup></a> [ghuser.io](https://github.com/AurelienLourot/ghuser.io) runs
+this tool every day.<br/>
 
 ## Changelog
 
+**2.2.0** (2018-08-09):
+  * [#1](https://github.com/AurelienLourot/github-contribs/issues/1) - Added `--issues` flag.
+
 **2.1.0** (2018-06-25):
   * Exported helper function `prevDay()`.
 

cli.js

@@ -10,7 +10,7 @@
 
   const cli = meow(`
 usage:
-  $ github-contribs [--quiet|--verbose] [--since YYYY-MM-DD] [--until YYYY-MM-DD] USER
+  $ github-contribs [--quiet|--verbose] [--since YYYY-MM-DD] [--until YYYY-MM-DD] [--issues] USER
   $ github-contribs --help
   $ github-contribs --version
 
@@ -22,12 +22,14 @@ optional arguments:
   --verbose            show debugging information
   --since YYYY-MM-DD   limit the results (default: first day at GitHub)
   --until YYYY-MM-DD   limit the results (default: today)
+  --issues             fetches not only commits and PRs but also issues
   --version            show program's version number and exit
   --help               show this help message and exit
 `, {
     boolean: [
       'quiet',
       'verbose',
+      'issues',
     ],
     string: [
       'since',
@@ -51,8 +53,10 @@ optional arguments:
   }
 
   const user = cli.input[0];
-  const repos = await githubContribs.fetch(user, cli.flags.since, cli.flags.until,
-                                           !cli.flags.quiet && ora, cli.flags.verbose && console);
+  const repos = await githubContribs.fetch(
+    user, cli.flags.since, cli.flags.until, !cli.flags.quiet && ora, cli.flags.verbose && console,
+    cli.flags.issues
+  );
   if (!cli.flags.quiet) {
     console.log(`${repos.size} repo(s) found:`);
   }

docs/advanced.md

@@ -14,7 +14,7 @@ $ github-contribs AurelienLourot
 GitHub API key found.
 ✔ Fetched first day at GitHub: 2015-04-04.
 ⚠ Be patient. The whole process might take up to an hour... Consider using --since and/or --until
-✔ Fetched all commits and PRs.
+✔ Fetched all commits and PRs. Consider using --issues to fetch issues as well.
 35 repo(s) found:
 AurelienLourot/lsankidb
 reframejs/reframe

index.js

@@ -7,7 +7,7 @@
   const htmlparser = require('htmlparser');
   const moment = require('moment');
 
-  const fetchContribs = async (user, since, until, ora, console) => {
+  const fetchContribs = async (user, since, until, ora, console, alsoIssues) => {
     ora = ora || (() => {
       return {
         start() { return this; },
@@ -21,7 +21,7 @@
     };
 
     const joinDate = await getFirstDayAtGithub(user, ora);
-    const result = await getContribs(user, joinDate, since, until, ora, console);
+    const result = await getContribs(user, joinDate, since, until, ora, console, alsoIssues);
     return result;
   };
 
@@ -82,7 +82,7 @@
     return result;
   };
 
-  const getContribs = async (user, joinDate, since, until, ora, console) => {
+  const getContribs = async (user, joinDate, since, until, ora, console, alsoIssues) => {
     const commitsHtmlToRepos = html => {
       const repos = new Set();
 
@@ -116,7 +116,7 @@
       return repos;
     };
 
-    const prsHtmlToRepos = html => {
+    const issuesHtmlToRepos = html => {
       const repos = new Set();
 
       const handler = new htmlparser.DefaultHandler((error, dom) => {});
@@ -149,6 +149,39 @@
       return repos;
     };
 
+    const hotIssuesHtmlToRepos = html => {
+      const repos = new Set();
+
+      const regex = /<a.*href="\/(.*)\/(.*)\/issues\//g;
+      let linkToIssue;
+      while ((linkToIssue = regex.exec(html))) {
+        const owner = linkToIssue[1];
+        const name = linkToIssue[2];
+        repos.add(`${owner}/${name}`);
+      }
+
+      return repos;
+    };
+
+    const progressMsg = (isDone, alsoIssues, numOfQueriedDays, numOfDaysToQuery) => {
+      let result = (isDone && 'Fetched') || 'Fetching';
+      result += ' all commits';
+      result += (alsoIssues && ', PRs and issues') || ' and PRs';
+
+      if (isDone) {
+        result += '.';
+      } else if (numOfQueriedDays && numOfDaysToQuery) {
+        result += ` [${numOfQueriedDays}/${numOfDaysToQuery}]`;
+      } else {
+        result += '...';
+      }
+
+      if (!alsoIssues) {
+        result += ' Consider using --issues to fetch issues as well.';
+      }
+      return result;
+    };
+
     let oldestDate = joinDate;
     if (since) {
       oldestDate = new Date(Math.max(oldestDate, stringToDate(since)));
@@ -175,22 +208,50 @@
             `https://github.com/users/${user}/created_commits?from=${currDateStr}&to=${currDateStr}`
           );
           const userCommitsHtml = await userCommits.text();
+          const commitsRepos = commitsHtmlToRepos(userCommitsHtml);
+
           const userPRs = await fetchRetry(
             `https://github.com/users/${user}/created_pull_requests?from=${currDateStr}&to=${currDateStr}`,
           );
           const userPRsHtml = await userPRs.text();
-          const commitsRepos = commitsHtmlToRepos(userCommitsHtml);
-          const prsRepos = prsHtmlToRepos(userPRsHtml);
+          const prsRepos = issuesHtmlToRepos(userPRsHtml);
+
+          let issuesRepos = [];
+          let hotIssuesRepos = [];
+          if (alsoIssues) {
+            const userIssues = await fetchRetry(
+              `https://github.com/users/${user}/created_issues?from=${currDateStr}&to=${currDateStr}`,
+            );
+            const userIssuesHtml = await userIssues.text();
+            issuesRepos = issuesHtmlToRepos(userIssuesHtml);
+
+            const userHotIssues = await fetchRetry(
+              `https://github.com/${user}?from=${currDateStr}`,
+            );
+            const userHotIssuesHtml = await userHotIssues.text();
+            hotIssuesRepos = hotIssuesHtmlToRepos(userHotIssuesHtml);
+          }
+
           progressSpinner.stop(); // temporary stop for logging
           for (const repo of commitsRepos) {
-            console.log(`${currDateStr}: (commits) ${repo}`);
+            console.log(`${currDateStr}: (commits)    ${repo}`);
             result.add(repo);
           }
           for (const repo of prsRepos) {
-            console.log(`${currDateStr}: (PRs)     ${repo}`);
+            console.log(`${currDateStr}: (PRs)        ${repo}`);
+            result.add(repo);
+          }
+          for (const repo of issuesRepos) {
+            console.log(`${currDateStr}: (issues)     ${repo}`);
             result.add(repo);
           }
-          progressSpinner.start(`Fetching all commits and PRs [${++numOfQueriedDays}/${numOfDaysToQuery}]`);
+          for (const repo of hotIssuesRepos) {
+            console.log(`${currDateStr}: (hot issues) ${repo}`);
+            result.add(repo);
+          }
+          progressSpinner.start(
+            progressMsg(false, alsoIssues, ++numOfQueriedDays, numOfDaysToQuery)
+          );
         })();
       };
     })();
@@ -205,9 +266,9 @@
     ora(warning).warn();
 
     const result = new Set();
-    const progressSpinner = ora('Fetching all commits and PRs...').start();
+    const progressSpinner = ora(progressMsg(false, alsoIssues)).start();
     await new PromisePool(getContribsOnOneDay, 5).start();
-    progressSpinner.succeed('Fetched all commits and PRs.');
+    progressSpinner.succeed(progressMsg(true, alsoIssues));
     return result;
   };
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ghuser/github-contribs",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "List all GitHub repos a user has contributed to since the beginning of time.",
   "license": "Unlicense",
   "repository": {
@@ -72,6 +72,6 @@
       ".js"
     ],
     "check-coverage": true,
-    "lines": 80
+    "lines": 84
   }
 }

test.js

@@ -1,35 +1,55 @@
 import test from 'ava';
 import m from '.';
 
+const ora = string => {
+  if (string) {
+    console.log(string);
+  }
+  return {
+    start(string) {
+      if (string) {
+        console.log(string);
+      }
+      return this;
+    },
+    stop() {},
+    succeed(string) {
+      if (string) {
+        console.log(string);
+      }
+    },
+    warn() {},
+  };
+};
+
 test('fetches commits and PRs', async t => {
   /* AurelienLourot had the following activity:
      * 2017-08-26: nothing
      * 2017-08-27: 14 commits in AurelienLourot/mybeir.ut
      * 2017-08-28: 1 PR in tt-gf/ant-ivy */
 
-  const ora = string => {
-    if (string) {
-      console.log(string);
-    }
-    return {
-      start(string) {
-        if (string) {
-          console.log(string);
-        }
-        return this;
-      },
-      stop() {},
-      succeed(string) {
-        if (string) {
-          console.log(string);
-        }
-      },
-      warn() {},
-    };
-  };
-
   const result = await m.fetch('AurelienLourot', '2017-08-26', '2017-08-28', ora, console);
   t.is(result.size, 2);
   t.true(result.has('AurelienLourot/mybeir.ut'));
   t.true(result.has('tt-gf/ant-ivy'));
 });
+
+test('fetches issues', async t => {
+  /* RichardLitt had the following activity:
+     * 2018-08-07: 1 issue in orbitdb/welcome */
+
+  const result = await m.fetch('RichardLitt', '2018-08-07', '2018-08-07', ora, console, true);
+  t.is(result.size, 1);
+  t.true(result.has('orbitdb/welcome'));
+});
+
+test('fetches hot issues', async t => {
+  /* AurelienLourot had the following activity:
+     * 2015-09-23: 1 commit in AurelienLourot/AurelienLourot.github.io
+                   1 hot issue in jfrog/build-info */
+
+  const result = await m.fetch('AurelienLourot', '2015-09-23', '2015-09-23', ora, console, true);
+  t.is(result.size, 2);
+  t.true(result.has('AurelienLourot/AurelienLourot.github.io'));
+  t.true(result.has('jfrog/build-info'));
+});