Dockerhub, Appsody releases, and star/watcher/forks for /stacks and /appsody repos

.gitignore

@@ -0,0 +1,3 @@
+/appsody_reports
+/node_modules
+.DS_Store

README.md

@@ -1,2 +1,27 @@
 # utils
 Repository to store various utilities related to Appsody project
+
+
+### Data Pulling scripts
+
+In order to execute any of these scripts, you'll need to first export your GitHub API Key to an environment variable called `GITHUB_API_KEY`.
+
+  - run_all.sh
+
+    Runs all of the `.js` scripts in this folder consecutively, then runs `../data_pulling_utils/json_to_csv.py` which creates a `.csv` file for any comparisons made against the previous month (if the `.json` files from last month are found). 
+    In order for the comparison to be made, the data from the previous month would have had to have been collected on the same day (e.g 01/12/19 and 01/01/2020); gathering data after the 28th day of the month wouldn't be recommended.
+
+  - appsody_dockerhub.js
+
+    Gets the pull count for each of the repositories on [appsody's dockerhub](https://hub.docker.com/u/appsody), and when the repository was last updated.
+
+  - appsody_releases.js
+
+    Gets the the download count for each of the binaries for every release of appsody from [the GitHub releases](https://github.com/appsody/appsody/releases).
+
+  - appsody_stars_watchers_forks.js
+
+    Gets the number of stars, watchers, and forks for the [appsody/stacks repository](https://github.com/appsody/stacks) and the [appsody/appsody repository](https://github.com/appsody/appsody).
+
+
+All of the results are stored in an `appsody_reports/DATE/` folder.

data_pulling_scripts/appsody_dockerhub.js

@@ -0,0 +1,41 @@
+const axios = require('axios');
+const tools = require('../data_pulling_utils/tools');
+
+
+
+retrieveData();
+
+ async function retrieveData() {
+
+
+    const AuthStr = `token ${process.env.GITHUB_API_KEY}`
+    const URL = 'https://hub.docker.com/v2/repositories/appsody/?page=1&page_size=100';
+
+    axios.get(URL, { headers: { Authorization: AuthStr } })
+    .then(response => {
+
+        dockerHubResultsArray = [];
+        for (let item of response.data.results) {
+
+            const date = new Date(item.last_updated)
+            var readableTimestamp = tools.addZeroPrefix(date.getDate()) + "-" + tools.addZeroPrefix((date.getMonth() +1)) + "-" + tools.addZeroPrefix(date.getFullYear());
+
+            const single = {
+                "name": item.name,
+                "pull_count": item.pull_count,
+                "last_updated": readableTimestamp
+            }
+            dockerHubResultsArray.push(single);
+        }
+
+        tools.createLogFile("dockerhub_appsody.json", dockerHubResultsArray, function(err) {
+            console.log(err);
+         });
+
+         tools.createComparisonFile("dockerhub_appsody", dockerHubResultsArray, "name");
+
+    })
+    .catch((error) => {
+        console.log('error ' + error);
+    });
+ }

data_pulling_scripts/appsody_releases.js

@@ -0,0 +1,40 @@
+const axios = require('axios');
+const tools = require('../data_pulling_utils/tools');
+
+retrieveData();
+
+ async function retrieveData() {
+
+
+    const AuthStr = `token ${process.env.GITHUB_API_KEY}`
+    const URL = 'https://api.github.com/repos/appsody/appsody/releases';
+
+    axios.get(URL, { headers: { Authorization: AuthStr } })
+    .then(response => {
+        releaseResultsArray = [];
+        for (let item of response.data) {
+
+            let name = item.tag_name
+
+            for (let asset of item.assets) {
+                const single = {
+                "release": name,
+                "cli_binary": asset.name,
+                "download_count": asset.download_count
+                }
+
+                releaseResultsArray.push(single);
+            }
+        }
+
+        tools.createLogFile("releases_appsody.json", releaseResultsArray, function(err) {
+            console.log(err);
+         });
+
+         tools.createComparisonFile("releases_appsody", releaseResultsArray, "cli_binary");
+
+    })
+    .catch((error) => {
+        console.log('error ' + error);
+    });
+ }

data_pulling_scripts/appsody_stars_watchers_forks.js

@@ -0,0 +1,76 @@
+const { graphql } = require('@octokit/graphql')
+const tools = require('../data_pulling_utils/tools');
+
+
+async function asyncForEach(array, callback) {
+  for (let index = 0; index < array.length; index++) {
+      await callback(array[index], index, array);
+  }
+}
+
+const queryTotals = async (repo, org) => {
+  const graphqlWithAuth = graphql.defaults({
+      headers: {
+          authorization: `token ${process.env.GITHUB_API_KEY}`
+      }
+  })
+  let data
+  try {
+      data = await graphqlWithAuth(`
+  {
+    repository(name: "${repo}", owner: "${org}") {
+      forks {
+        totalCount
+      }
+      stargazers {
+        totalCount
+      }
+      watchers {
+        totalCount
+      }
+    }
+  }`)
+  } catch (err) {
+      if (err.name === 'HttpError') {
+          setTimeout(() => {
+              queryTotals(repo, org)
+          }, 10000)
+      } else {
+          throw (err)
+      }
+  } finally {
+
+      const single = {
+          "repo": repo,
+          "stars": data.repository.stargazers.totalCount,
+          "watchers": data.repository.watchers.totalCount,
+          "forks": data.repository.forks.totalCount,
+      }
+      return single;
+  }
+}
+
+var queries = [
+  ['appsody', 'stacks'],
+  ['appsody', 'appsody']
+];
+results = [];
+
+const callQueries = async () => {
+
+  const start = async () => {
+      await asyncForEach(queries, async (q) => {
+          let res = await (queryTotals(q[1], q[0]));
+          results.push(res)
+      });
+
+      tools.createLogFile(`stars_watchers_forks.json`, results, function(err) {
+          console.log(err);
+      });
+
+      tools.createComparisonFile("stars_watchers_forks", results, "repo");
+  }
+  start();
+}
+
+callQueries();

data_pulling_scripts/run_all.sh

@@ -0,0 +1,6 @@
+for entry in ./*.js
+do
+  echo "$entry"
+  `node $entry`
+done
+`python ../data_pulling_utils/json_to_csv.py ../appsody_reports/`

data_pulling_utils/json_to_csv.py

@@ -0,0 +1,34 @@
+from os import walk, path, remove
+
+from datetime import date    
+
+import csv, json, sys, logging
+
+
+if sys.argv[1] is not None:
+
+  today = date.today().isoformat()
+  targetFolder = sys.argv[1] + today + "/"
+
+  f = []
+  for (dirpath, dirnames, filenames) in walk(targetFolder):
+      f.extend(filenames)
+      break
+
+  for file in f:
+    if ".csv" not in file and "_comparison" in file:
+      fileInput = targetFolder + file
+      fileOutput = fileInput[:-5] + ".csv"
+
+      inputFile = open(fileInput)
+      outputFile = open(fileOutput, 'w')
+      data = json.load(inputFile)
+      inputFile.close()
+
+      output = csv.writer(outputFile)
+
+      output.writerow(data[0].keys())  # header row
+
+      for row in data:
+          output.writerow(row.values())
+      output.writerow("")