Add Cloud Functions

.circleci/config.yml

@@ -42,9 +42,12 @@ jobs:
           command: |-
             mkdir -p /home/node/.npm-global
             ./.circleci/npm-install-retry.js
+            cd functions
+            ../.circleci/npm-install-retry.js
+            cd ..
           environment:
             NPM_CONFIG_PREFIX: /home/node/.npm-global
-      - run: cp defaultsettings.json settings.json
+      - run: cp functions/settings.tmpl.json functions/settings.json
       - run: npm test
       - run: node_modules/.bin/codecov
   node10:

.eslintignore

@@ -1 +1,2 @@
 node_modules/*
+functions/node_modules/*

README.md

@@ -16,7 +16,9 @@ animal, that's why you need LabelCat.
 1.  `cd LabelCat`
 1.  `npm install`
 1.  `npm link .`
-1.  `cp defaultsettings.json settings.json` (`settings.json` is where you
+1.  `cd functions`
+
+    `cp settings.tmpl.json settings.json` (`settings.json` is where you
     customize the app)
 1.  Modify `settings.json` as necessary.
 
@@ -169,6 +171,14 @@ Examples:
 
         labelcat createModel 123ABCD456789 firstModel
 
+## Deploy Cloud Functions
+1.  `cd Functions`
+1.  `npm install`
+1.  `cp settings.tmpl.json settings.json`
+1.  Modify `settings.json` as necessary.
+1. `gcloud alpha functions deploy handleNewIssue --trigger-http --runtime nodejs8`
+1. `gcloud functions deploy triage --runtime nodejs8 --trigger-resource YOUR-PUB/SUB-TOPIC-NAME --trigger-event google.pubsub.topic.publish
+`
 ## Contributing
 
 See [CONTRIBUTING][3].

bin/labelcat.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 const util = require('../src/util.js');
-const settings = require('../settings.json'); // eslint-disable-line node/no-missing-require
+const settings = require('../functions/settings.json'); // eslint-disable-line node/no-missing-require
 
 require(`yargs`)
   .demand(1)

functions/.gcloudignore

@@ -0,0 +1,16 @@
+# This file specifies files that are *not* uploaded to Google Cloud Platform
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+node_modules

functions/index.js

@@ -0,0 +1,142 @@
+const settings = require('./settings.json'); // eslint-disable-line node/no-missing-require
+const crypto = require('crypto');
+const log = require('loglevel');
+log.setLevel('info');
+const automl = require('@google-cloud/automl');
+const PubSub = require(`@google-cloud/pubsub`);
+const octokit = require('@octokit/rest')();
+const client = new automl.v1beta1.PredictionServiceClient();
+
+const PROJECT_ID = settings.PROJECT_ID;
+const COMPUTE_REGION = settings.COMPUTE_REGION;
+const MODEL_ID = settings.MODEL_ID;
+const TOPIC_NAME = settings.TOPIC_NAME;
+const SECRET_TOKEN = settings.SECRET_TOKEN;
+const SCORE_THRESHOLD = 70;
+
+/**
+ * Verifies request has come from Github and publishes
+ * message to specified Pub/Sub topic.
+ *
+ * @param {object} req
+ * @param {object} res
+ */
+async function handleNewIssue(req, res) {
+  try {
+    if (req.body.action !== 'opened') {
+      res.status(400).send('Wrong action.');
+      return;
+    }
+
+    await validateRequest(req);
+    const messageId = await publishMessage(req);
+    res.status(200).send(messageId);
+  } catch (err) {
+    log.error(err.stack);
+    res.status(403).send({error: err.message});
+  }
+}
+
+function validateRequest(req) {
+  return Promise.resolve().then(() => {
+    const digest = crypto
+      .createHmac('sha1', SECRET_TOKEN)
+      .update(JSON.stringify(req.body))
+      .digest('hex');
+    if (req.headers['x-hub-signature'] !== `sha1=${digest}`) {
+      const error = new Error('Unauthorized');
+      error.statusCode = 403;
+      throw error;
+    }
+  });
+}
+
+async function publishMessage(req) {
+  try {
+    const text = `${req.body.issue.title} ${req.body.issue.body}`;
+    const data = JSON.stringify({
+      owner: req.body.repository.owner.login,
+      repo: req.body.repository.name,
+      number: req.body.issue.number,
+      text: text,
+    });
+    const dataBuffer = Buffer.from(data);
+
+    const pubsubClient = new PubSub({
+      PROJECT_ID: PROJECT_ID,
+    });
+
+    const response = await pubsubClient
+      .topic(TOPIC_NAME)
+      .publisher()
+      .publish(dataBuffer);
+    return response;
+  } catch (err) {
+    log.error('ERROR:', err);
+  }
+}
+
+async function triage(event) {
+  octokit.authenticate({
+    type: 'oauth',
+    token: settings.SECRET_TOKEN,
+  });
+
+  const pubSubMessage = event.data;
+  let issueData = Buffer.from(pubSubMessage.data, 'base64').toString();
+  issueData = JSON.parse(issueData);
+  const owner = issueData.owner;
+  const repo = issueData.repo;
+  const number = issueData.number;
+
+  issueData.labeled = false;
+  let results = await predict(issueData.text);
+
+  if (results) {
+    const labels = ['bug'];
+    const response = await octokit.issues.addLabels({
+      owner: owner,
+      repo: repo,
+      number: number,
+      labels: labels,
+    });
+
+    if (response.status === 200) {
+      issueData.labeled = true;
+    }
+  }
+
+  return issueData;
+}
+
+async function predict(text) {
+  const modelFullId = client.modelPath(PROJECT_ID, COMPUTE_REGION, MODEL_ID);
+
+  const payload = {
+    textSnippet: {
+      content: text,
+      mimeType: `text/plain`,
+    },
+  };
+
+  try {
+    const response = await client.predict({
+      name: modelFullId,
+      payload: payload,
+      params: {},
+    });
+
+    if (response[0].payload[1].classification.score > SCORE_THRESHOLD) {
+      return true;
+    }
+
+    return false;
+  } catch (err) {
+    log.error(err);
+  }
+}
+
+module.exports = {
+  handleNewIssue,
+  triage,
+};

functions/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "labelcat-functions",
+  "engines": {
+    "node": ">= 8.x"
+  },
+  "dependencies": {
+    "@google-cloud/automl": "^0.1.2",
+    "@google-cloud/pubsub": "^0.20.1",
+    "@octokit/rest": "^15.15.1",
+    "loglevel": "^1.6.1",
+    "nconf": "^0.10.0"
+  }
+}

functions/settings.tmpl.json

@@ -0,0 +1,7 @@
+{
+  "secretToken": "YOUR GITHUB CLIENT ID HERE",
+  "projectID": "YOUR GCP PROJECT ID HERE",
+  "computeRegion": "YOUR GCP PROJECT COMPUTE REGION HERE",
+  "modelId": "YOUR AUTOML NL MODEL ID HERE",
+  "topicName": "YOUR PUB/SUB TOPIC NAME HERE"
+}

package.json

@@ -26,34 +26,40 @@
     "node": ">= 8.x"
   },
   "scripts": {
-    "cover": "nyc --reporter=lcov mocha test/*.js && nyc report",
-    "lint": "eslint src/ system-test/ test/ bin/",
-    "prettier": "prettier --write bin/*.js src/*.js test/*.js system-test/*.js",
+    "cover": "nyc --reporter=lcov mocha test/*.js system-test/*.js && nyc report",
+    "lint": "eslint src/ system-test/ test/ bin/ functions/",
+    "prettier": "prettier --write bin/*.js src/*.js test/*.js system-test/*.js functions/*.js",
     "system-test": "mocha system-test/*.js --timeout 600000",
     "test-no-cover": "mocha test/*.js",
     "test": "npm run cover"
   },
   "dependencies": {
     "@google-cloud/automl": "^0.1.2",
-    "@octokit/rest": "^15.12.1",
+    "@octokit/rest": "^15.15.1",
     "csv-write-stream": "^2.0.0",
     "json2csv": "^4.2.1",
     "loglevel": "^1.6.1",
+    "nconf": "^0.10.0",
     "papaparse": "^4.6.1",
     "yargs": "^12.0.2"
   },
   "devDependencies": {
+    "body-parser": "^1.18.3",
     "codecov": "^3.0.2",
+    "crypto": "^1.0.1",
     "eslint": "^5.0.0",
     "eslint-config-prettier": "^3.0.0",
     "eslint-plugin-node": "^7.0.0",
     "eslint-plugin-prettier": "^2.6.0",
+    "express": "^4.16.4",
     "intelli-espower-loader": "^1.0.1",
     "mocha": "^5.2.0",
     "nyc": "^13.0.0",
     "power-assert": "^1.6.0",
     "prettier": "^1.13.5",
     "proxyquire": "^2.1.0",
-    "sinon": "^6.3.4"
+    "sinon": "^6.3.4",
+    "supertest": "^3.3.0",
+    "uuid": "^3.3.2"
   }
 }

src/util.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const fs = require('fs');
-const settings = require('../settings.json'); // eslint-disable-line node/no-missing-require
+const settings = require('../functions/settings.json'); // eslint-disable-line node/no-missing-require
 const octokit = require('@octokit/rest')();
 const log = require('loglevel');
 const Papa = require('papaparse');
@@ -18,8 +18,7 @@ const automl = require(`@google-cloud/automl`);
 async function retrieveIssues(data, label, alternatives) {
   octokit.authenticate({
     type: 'oauth',
-    key: settings.githubClientID,
-    secret: settings.githubClientSecret,
+    token: settings.SECRET_TOKEN,
   });
 
   log.info('RETRIEVING ISSUES...');
@@ -106,7 +105,9 @@ function cleanLabels(issue, opts) {
  */
 function getIssueInfo(issue) {
   try {
-    const text = issue.title + ' ' + issue.body;
+    const raw = issue.title + ' ' + issue.body;
+    // remove punctuation that will interfere with csv
+    const text = raw.replace(/[^\w\s]/gi, '');
     const labels = issue.labels.map(labelObject => labelObject.name);
 
     return {text, labels};
@@ -134,20 +135,20 @@ function makeCSV(issues, file) {
 
 /**
  * Create a Google AutoML Natural Language dataset
- * @param {string} projectId
- * @param {string} computeRegion
+ * @param {string} PROJECT_ID
+ * @param {string} COMPUTE_REGION
  * @param {string} datasetName
  * @param {string} multiLabel
  */
 async function createDataset(
-  projectId,
-  computeRegion,
+  PROJECT_ID,
+  COMPUTE_REGION,
   datasetName,
   multiLabel
 ) {
   const automl = require(`@google-cloud/automl`);
   const client = new automl.v1beta1.AutoMlClient();
-  const projectLocation = client.locationPath(projectId, computeRegion);
+  const projectLocation = client.locationPath(PROJECT_ID, COMPUTE_REGION);
 
   // Classification type is assigned based on multiClass value.
   let classificationType = `MULTICLASS`;
@@ -191,16 +192,20 @@ async function createDataset(
 
 /**
  * Import data into Google AutoML NL dataset
- * @param {string} projectId
- * @param {string} computeRegion
+ * @param {string} PROJECT_ID
+ * @param {string} COMPUTE_REGION
  * @param {string} datasetId
  * @param {string} path
  */
-async function importData(projectId, computeRegion, datasetId, path) {
+async function importData(PROJECT_ID, COMPUTE_REGION, datasetId, path) {
   const client = new automl.v1beta1.AutoMlClient();
 
   // Get the full path of the dataset.
-  const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId);
+  const datasetFullId = client.datasetPath(
+    PROJECT_ID,
+    COMPUTE_REGION,
+    datasetId
+  );
 
   // Get the Google Cloud Storage URIs.
   const inputUris = path.split(`,`);
@@ -222,9 +227,9 @@ async function importData(projectId, computeRegion, datasetId, path) {
   }
 }
 
-async function listDatasets(projectId, computeRegion) {
+async function listDatasets(PROJECT_ID, COMPUTE_REGION) {
   const client = new automl.v1beta1.AutoMlClient();
-  const projectLocation = client.locationPath(projectId, computeRegion);
+  const projectLocation = client.locationPath(PROJECT_ID, COMPUTE_REGION);
 
   try {
     const responses = await client.listDatasets({parent: projectLocation});
@@ -247,11 +252,11 @@ async function listDatasets(projectId, computeRegion) {
   }
 }
 
-async function createModel(projectId, computeRegion, datasetId, modelName) {
+async function createModel(PROJECT_ID, COMPUTE_REGION, datasetId, modelName) {
   const client = new automl.v1beta1.AutoMlClient();
 
   // A resource that represents Google Cloud Platform location.
-  const projectLocation = client.locationPath(projectId, computeRegion);
+  const projectLocation = client.locationPath(PROJECT_ID, COMPUTE_REGION);
 
   // Set model name and model metadata for the dataset.
   const myModel = {