From c8a20ab9aea0c10999c414097d820fe43ed6341b Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Mon, 8 Oct 2018 11:24:10 -0700 Subject: [PATCH 01/23] added cloud functions --- .eslintignore | 1 + README.md | 8 ++ bin/labelcat.js | 2 +- defaultsettings.json | 7 +- functions/.gcloudignore | 16 +++ functions/defaultsettings.json | 7 ++ functions/index.js | 150 +++++++++++++++++++++++++++ functions/package.json | 14 +++ package.json | 17 +-- src/util.js | 8 +- system-test/functions_system_test.js | 136 ++++++++++++++++++++++++ system-test/issuePayload.js | 27 +++++ test/triage_test.js | 128 +++++++++++++++++++++++ 13 files changed, 507 insertions(+), 14 deletions(-) create mode 100644 functions/.gcloudignore create mode 100644 functions/defaultsettings.json create mode 100644 functions/index.js create mode 100644 functions/package.json create mode 100644 system-test/functions_system_test.js create mode 100644 system-test/issuePayload.js create mode 100644 test/triage_test.js diff --git a/.eslintignore b/.eslintignore index 8d87b1d2..944c0d70 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1 +1,2 @@ node_modules/* +functions/node_modules/* diff --git a/README.md b/README.md index e5ff229b..b4c8aba4 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,14 @@ Examples: labelcat createModel 123ABCD456789 firstModel +## Deploy Cloud Functions +1. `cd Functions` +1. `npm install` +1. `cp defaultsettings.json settings.json` +1. Modify `settings.json` as necessary. +1. `gcloud alpha functions deploy handleNewIssue --trigger-http --runtime nodejs8` +1. `gcloud functions deploy triage --runtime nodejs8 --trigger-resource YOUR-PUB/SUB-TOPIC-NAME --trigger-event google.pubsub.topic.publish +` ## Contributing See [CONTRIBUTING][3]. diff --git a/bin/labelcat.js b/bin/labelcat.js index 998836f9..af3acb91 100755 --- a/bin/labelcat.js +++ b/bin/labelcat.js @@ -1,7 +1,7 @@ #!/usr/bin/env node const util = require('../src/util.js'); -const settings = require('../settings.json'); // eslint-disable-line node/no-missing-require +const settings = require('../functions/settings.json'); // eslint-disable-line node/no-missing-require require(`yargs`) .demand(1) diff --git a/defaultsettings.json b/defaultsettings.json index 8aacb45b..ddfa2ebe 100644 --- a/defaultsettings.json +++ b/defaultsettings.json @@ -1,6 +1,7 @@ { - "githubClientID": "YOUR GITHUB CLIENT ID HERE", - "githubClientSecret": "YOUR GITHUB CLIENT SECRET HERE", + "secretToken": "YOUR GITHUB CLIENT ID HERE", "projectID": "YOUR GCP PROJECT ID HERE", - "computeRegion": "YOUR GCP PROJECT COMPUTE REGION HERE" + "computeRegion": "YOUR GCP PROJECT COMPUTE REGION HERE", + "modelId": "YOUR AUTOML NL MODEL ID HERE", + "topicName": "YOUR PUB/SUB TOPIC NAME HERE" } diff --git a/functions/.gcloudignore b/functions/.gcloudignore new file mode 100644 index 00000000..ccc4eb24 --- /dev/null +++ b/functions/.gcloudignore @@ -0,0 +1,16 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +node_modules diff --git a/functions/defaultsettings.json b/functions/defaultsettings.json new file mode 100644 index 00000000..ddfa2ebe --- /dev/null +++ b/functions/defaultsettings.json @@ -0,0 +1,7 @@ +{ + "secretToken": "YOUR GITHUB CLIENT ID HERE", + "projectID": "YOUR GCP PROJECT ID HERE", + "computeRegion": "YOUR GCP PROJECT COMPUTE REGION HERE", + "modelId": "YOUR AUTOML NL MODEL ID HERE", + "topicName": "YOUR PUB/SUB TOPIC NAME HERE" +} diff --git a/functions/index.js b/functions/index.js new file mode 100644 index 00000000..356f99f5 --- /dev/null +++ b/functions/index.js @@ -0,0 +1,150 @@ +const settings = require('./settings.json'); // eslint-disable-line node/no-missing-require +const crypto = require('crypto'); +const log = require('loglevel'); +log.setLevel('info'); +const automl = require('@google-cloud/automl'); +const PubSub = require(`@google-cloud/pubsub`); +const octokit = require('@octokit/rest')(); + +/** + * TODO(developer): Set the following... + */ +// Your Google Cloud Platform project ID & compute region +const projectId = settings.projectId; +const computeRegion = settings.computeRegion; +// Your Google Cloud AutoML NL model ID +const modelId = settings.modelId; +// Your Google Cloud Pub/Sub topic +const topicName = settings.topicName; +/** + * Verifies request has come from Github and publishes + * message to specified Pub/Sub topic. + * + * @param {object} req + * @param {object} res + */ +async function handleNewIssue(req, res) { + try { + if (req.body.action !== 'opened') { + res.status(400).send('Wrong action.'); + return; + } + + await validateRequest(req); + const messageId = await publishMessage(req); + res.status(200).send(messageId); + } catch (err) { + log.error(err.stack); + res.status(403).send({error: err.message}); + } +} + +function validateRequest(req) { + return Promise.resolve().then(() => { + const digest = crypto + .createHmac('sha1', settings.secretToken) + .update(JSON.stringify(req.body)) + .digest('hex'); + if (req.headers['x-hub-signature'] !== `sha1=${digest}`) { + const error = new Error('Unauthorized'); + error.statusCode = 403; + throw error; + } + }); +} + +async function publishMessage(req) { + try { + const text = req.body.issue.title + ' ' + req.body.issue.body; + const data = JSON.stringify({ + owner: req.body.repository.owner.login, + repo: req.body.repository.name, + number: req.body.issue.number, + text: text, + }); + const dataBuffer = Buffer.from(data); + + const pubsubClient = new PubSub({ + projectId: projectId, + }); + + const response = await pubsubClient + .topic(topicName) + .publisher() + .publish(dataBuffer); + return response; + } catch (err) { + log.error('ERROR:', err); + } +} + +async function triage(event, res) { + octokit.authenticate({ + type: 'oauth', + token: settings.secretToken, + }); + + const pubSubMessage = event.data; + let issueData = Buffer.from(pubSubMessage.data, 'base64').toString(); + issueData = JSON.parse(issueData); + const owner = issueData.owner; + const repo = issueData.repo; + const number = issueData.number; + + try { + issueData.labeled = false; + let results = await predict(issueData.text); + + if (results) { + const labels = ['bug']; + const response = await octokit.issues.addLabels({ + owner: owner, + repo: repo, + number: number, + labels: labels, + }); + + if (response.status === 200) { + issueData.labeled = true; + } + } + + return issueData; + } catch (err) { + res.status(403).send({error: err.message}); + } +} + +async function predict(text) { + const client = new automl.v1beta1.PredictionServiceClient(); + + const modelFullId = client.modelPath(projectId, computeRegion, modelId); + + const payload = { + textSnippet: { + content: text, + mimeType: `text/plain`, + }, + }; + + try { + const response = await client.predict({ + name: modelFullId, + payload: payload, + params: {}, + }); + + if (response[0].payload[1].classification.score > 89) { + return true; + } + + return false; + } catch (err) { + log.error(err); + } +} + +module.exports = { + handleNewIssue, + triage, +}; diff --git a/functions/package.json b/functions/package.json new file mode 100644 index 00000000..f880f007 --- /dev/null +++ b/functions/package.json @@ -0,0 +1,14 @@ +{ + "name": "labelcat-functions", + "engines": { + "node": ">= 8.x" + }, + "dependencies": { + "@google-cloud/automl": "^0.1.2", + "@google-cloud/datastore": "^2.0.0", + "@google-cloud/pubsub": "^0.20.1", + "@google-cloud/storage": "^2.1.0", + "@octokit/rest": "^15.15.1", + "loglevel": "^1.6.1" + } +} diff --git a/package.json b/package.json index 0141165f..e56f783b 100644 --- a/package.json +++ b/package.json @@ -26,34 +26,39 @@ "node": ">= 8.x" }, "scripts": { - "cover": "nyc --reporter=lcov mocha test/*.js && nyc report", - "lint": "eslint src/ system-test/ test/ bin/", - "prettier": "prettier --write bin/*.js src/*.js test/*.js system-test/*.js", + "cover": "nyc --reporter=lcov mocha test/*.js system-test/*.js && nyc report", + "lint": "eslint src/ system-test/ test/ bin/ functions/", + "prettier": "prettier --write bin/*.js src/*.js test/*.js system-test/*.js functions/*.js", "system-test": "mocha system-test/*.js --timeout 600000", "test-no-cover": "mocha test/*.js", "test": "npm run cover" }, "dependencies": { "@google-cloud/automl": "^0.1.2", - "@octokit/rest": "^15.12.1", "csv-write-stream": "^2.0.0", "json2csv": "^4.2.1", "loglevel": "^1.6.1", "papaparse": "^4.6.1", - "yargs": "^12.0.2" + "yargs": "^12.0.2", + "@octokit/rest": "^15.15.1" }, "devDependencies": { + "body-parser": "^1.18.3", "codecov": "^3.0.2", + "crypto": "^1.0.1", "eslint": "^5.0.0", "eslint-config-prettier": "^3.0.0", "eslint-plugin-node": "^7.0.0", "eslint-plugin-prettier": "^2.6.0", + "express": "^4.16.4", "intelli-espower-loader": "^1.0.1", "mocha": "^5.2.0", "nyc": "^13.0.0", "power-assert": "^1.6.0", "prettier": "^1.13.5", "proxyquire": "^2.1.0", - "sinon": "^6.3.4" + "sinon": "^6.3.4", + "supertest": "^3.3.0", + "uuid": "^3.3.2" } } diff --git a/src/util.js b/src/util.js index 031177c4..ddb950c2 100755 --- a/src/util.js +++ b/src/util.js @@ -1,7 +1,7 @@ 'use strict'; const fs = require('fs'); -const settings = require('../settings.json'); // eslint-disable-line node/no-missing-require +const settings = require('../functions/settings.json'); // eslint-disable-line node/no-missing-require const octokit = require('@octokit/rest')(); const log = require('loglevel'); const Papa = require('papaparse'); @@ -18,8 +18,7 @@ const automl = require(`@google-cloud/automl`); async function retrieveIssues(data, label, alternatives) { octokit.authenticate({ type: 'oauth', - key: settings.githubClientID, - secret: settings.githubClientSecret, + token: settings.secretToken, }); log.info('RETRIEVING ISSUES...'); @@ -106,7 +105,8 @@ function cleanLabels(issue, opts) { */ function getIssueInfo(issue) { try { - const text = issue.title + ' ' + issue.body; + const raw = issue.title + ' ' + issue.body; + const text = raw.replace(/[^\w\s]/gi, ''); const labels = issue.labels.map(labelObject => labelObject.name); return {text, labels}; diff --git a/system-test/functions_system_test.js b/system-test/functions_system_test.js new file mode 100644 index 00000000..0905fa2f --- /dev/null +++ b/system-test/functions_system_test.js @@ -0,0 +1,136 @@ +const express = require('express'); +const issueEvent = require('./issuePayload.js'); +const mocha = require('mocha'); +const describe = mocha.describe; +const it = mocha.it; +const beforeEach = mocha.beforeEach; +const proxyquire = require('proxyquire'); +const sinon = require('sinon'); +const assert = require('assert'); +const supertest = require(`supertest`); +const bodyParser = require('body-parser'); +const crypto = require('crypto'); + +const mockSettings = { + secretToken: 'foo', +}; + +function setup() { + let event = issueEvent; + return { + getHeader: () => { + const digest = crypto + .createHmac('sha1', mockSettings.secretToken) + .update(JSON.stringify(event.body)) + .digest('hex'); + return `sha1=${digest}`; + }, + }; +} + +describe('handleNewIssue()', function() { + let app, codeUnderTest, functs; + + const publishMock = sinon.stub().returns('123'); + const publisherMock = sinon.stub().returns({publish: publishMock}); + const topicMock = sinon.stub().returns({publisher: publisherMock}); + const pubsubMock = sinon.stub().returns({topic: topicMock}); + + beforeEach(() => { + codeUnderTest = setup(); + app = express(); + const requestLimit = '1024mb'; + + const rawBodySaver = (req, res, buf) => { + req.rawBody = buf; + }; + + const defaultBodySavingOptions = { + limit: requestLimit, + verify: rawBodySaver, + }; + + const rawBodySavingOptions = { + limit: requestLimit, + verify: rawBodySaver, + type: '*/*', + }; + + // Use extended query string parsing for URL-encoded bodies. + const urlEncodedOptions = { + limit: requestLimit, + verify: rawBodySaver, + extended: true, + }; + + // Parse request body + app.use(bodyParser.json(defaultBodySavingOptions)); + app.use(bodyParser.text(defaultBodySavingOptions)); + app.use(bodyParser.urlencoded(urlEncodedOptions)); + + // MUST be last in the list of body parsers as subsequent parsers will be + // skipped when one is matched. + app.use(bodyParser.raw(rawBodySavingOptions)); + + functs = proxyquire('../functions/index.js', { + '@google-cloud/pubsub': pubsubMock, + './settings.json': mockSettings, + }); + + app.post(`/handleNewIssue`, functs.handleNewIssue); + // }); + }); + + it('should validate request', function(done) { + supertest(app) + .post(`/handleNewIssue`) + .send(issueEvent.body) + .set('x-hub-signature', 'foo') + .end(function(err, res) { + assert.strictEqual(403, res.statusCode); + sinon.assert.notCalled(publishMock); + done(); + }); + }); + + it('should publish message and return messageId', function(done) { + supertest(app) + .post(`/handleNewIssue`) + .send(issueEvent.body) + .set('x-hub-signature', codeUnderTest.getHeader()) + .end(function(err, res) { + assert.strictEqual(200, res.statusCode); + assert.strictEqual('123', res.text); + sinon.assert.calledOnce(publishMock); + done(); + }); + }); + + it('should return if action is not opened', function(done) { + let wrongAction = issueEvent; + wrongAction.body.action = 'edited'; + supertest(app) + .post(`/handleNewIssue`) + .send(wrongAction.body) + .set('x-hub-signature', codeUnderTest.getHeader()) + .end(function(err, res) { + assert.strictEqual(400, res.statusCode); + assert.ok(res.text === 'Wrong action.'); + done(); + }); + }); + + it('should not publish request with incorrect data', function(done) { + let wrongAction = issueEvent; + wrongAction.body.action = 'opened'; + wrongAction.body.issue.title = undefined; + supertest(app) + .post(`/handleNewIssue`) + .send({body: null}) + .set('x-hub-signature', codeUnderTest.getHeader()) + .end(function(err, res) { + assert.strictEqual(400, res.statusCode); + done(); + }); + }); +}); diff --git a/system-test/issuePayload.js b/system-test/issuePayload.js new file mode 100644 index 00000000..239e2abb --- /dev/null +++ b/system-test/issuePayload.js @@ -0,0 +1,27 @@ +module.exports = { + body: { + action: 'opened', + issue: { + number: 2, + title: 'LABELCAT-TEST', + labels: [ + { + id: 949737505, + node_id: 'MDU6TGFiZWw5NDk3Mzc1MDU=', + url: 'https://api.github.com/repos/Codertocat/Hello-World/labels/bug', + name: 'bug', + color: 'd73a4a', + default: true, + }, + ], + state: 'open', + body: "It looks like you accidently spelled 'commit' with two 't's.", + }, + repository: { + name: 'Hello-World', + owner: { + login: 'Codertocat', + }, + }, + }, +}; diff --git a/test/triage_test.js b/test/triage_test.js new file mode 100644 index 00000000..ebade33d --- /dev/null +++ b/test/triage_test.js @@ -0,0 +1,128 @@ +const mocha = require('mocha'); +const describe = mocha.describe; +const it = mocha.it; +const proxyquire = require('proxyquire'); +const sinon = require('sinon'); +const assert = require('assert'); + +let functions, autoMlMock, octoMock, dataBuffer, mockSettings; + +beforeEach(() => { + const data = JSON.stringify({ + owner: 'GoogleCloudPlatform', + repo: 'labelcat', + number: 22, + text: 'some issue information', + }); + + dataBuffer = Buffer.from(data); + + const reject = sinon.stub(); + reject.withArgs(true).throws(401); + + const mockAdd = sinon.stub().returns( + Promise.resolve({ + data: [ + { + id: 271022241, + node_id: 'MDU6TGFiZWwyNzEwMjIyNDE=', + url: + 'https://api.github.com/repos/GoogleCloudPlatform/LabelCat/labels/bug', + name: 'bug', + color: 'fc2929', + default: true, + }, + ], + status: 200, + }) + ); + + octoMock = { + authenticate: sinon.stub(), + issues: {addLabels: mockAdd}, + }; + + const model = sinon.spy(); + const predict = sinon.stub(); + + predict.onCall(0).returns([ + { + payload: [ + { + annotationSpecId: '', + displayName: '0', + classification: [Object], + detail: 'classification', + }, + { + annotationSpecId: '', + displayName: '1', + classification: {score: 90}, + detail: 'classification', + }, + ], + }, + ]); + + predict.onCall(1).returns([ + { + payload: [ + { + annotationSpecId: '', + displayName: '0', + classification: [Object], + detail: 'classification', + }, + { + annotationSpecId: '', + displayName: '1', + classification: {score: 80}, + detail: 'classification', + }, + ], + }, + ]); + + const mockClient = sinon.stub().returns({ + modelPath: model, + predict: predict, + }); + + autoMlMock = {v1beta1: {PredictionServiceClient: mockClient}}; + + mockSettings = { + secretToken: 'foo', + projectId: 'test-project', + computeRegion: 'uscentral', + topicName: 'testTopic', + modelId: 'test-model', + }; +}); + +describe('triage()', function() { + it('should run prediction and returns correct boolean', async () => { + functions = proxyquire('../functions/index.js', { + '@octokit/rest': () => octoMock, + '@google-cloud/automl': autoMlMock, + './settings.json': mockSettings, + }); + + let result = await functions.triage({data: {data: dataBuffer}}); + + assert(result.labeled === true); + assert(result.number === 22); + result = await functions.triage({data: {data: dataBuffer}}); + assert(result.labeled === false); + assert(result.number === 22); + }); + + it('should throw error if unauthorized gitHub user', async () => { + functions = proxyquire('../functions/index.js', { + './settings.json': mockSettings, + }); + + let result = await functions.triage({data: {data: dataBuffer}}); + assert(result.labeled === false); + assert(result.number === 22); + }); +}); From 8b65da100f952c6f30bdbc0944091cbcb66038f4 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Wed, 24 Oct 2018 13:27:06 -0700 Subject: [PATCH 02/23] mocked settings --- test/util_test.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/util_test.js b/test/util_test.js index 794eed21..d2deb469 100644 --- a/test/util_test.js +++ b/test/util_test.js @@ -75,8 +75,14 @@ describe('retrieveIssues', () => { hasNextPage: hasNext, getNextPage: getNext, }; + + const mockSettings = { + secretToken: 'foo', + }; + util = proxyquire('../src/util.js', { '@octokit/rest': () => octoMock, + '../functions/settings.json': mockSettings, }); }); From 705f9c33de5630b8d8ef862f858f0304fe67b9ad Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Wed, 24 Oct 2018 13:37:24 -0700 Subject: [PATCH 03/23] mocked settings --- test/util_test.js | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/util_test.js b/test/util_test.js index d2deb469..60477d57 100644 --- a/test/util_test.js +++ b/test/util_test.js @@ -183,9 +183,14 @@ describe('createDataset()', function() { createDataset: create, }); + const mockSettings = { + secretToken: 'foo', + }; + const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, + '../functions/settings.json': mockSettings, }); util.createDataset(projectId, computeRegion, datasetName, multiLabel); @@ -210,9 +215,14 @@ describe('createDataset()', function() { createDataset: create, }); + const mockSettings = { + secretToken: 'foo', + }; + const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, + '../functions/settings.json': mockSettings, }); util.createDataset(projectId, computeRegion, datasetName, multiLabel); @@ -235,8 +245,14 @@ describe('importData()', function() { }); const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + + const mockSettings = { + secretToken: 'foo', + }; + const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, + '../functions/settings.json': mockSettings, }); util.importData(projectId, computeRegion, datasetId, file); @@ -254,8 +270,14 @@ describe('importData()', function() { }); const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + + const mockSettings = { + secretToken: 'foo', + }; + const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, + '../functions/settings.json': mockSettings, }); util.importData(projectId, computeRegion, datasetId, file); @@ -287,8 +309,14 @@ describe('listDatasets()', function() { }); const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + + const mockSettings = { + secretToken: 'foo', + }; + const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, + '../functions/settings.json': mockSettings, }); await util.listDatasets(projectId, computeRegion); @@ -314,8 +342,14 @@ describe('listDatasets()', function() { }); const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + + const mockSettings = { + secretToken: 'foo', + }; + const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, + '../functions/settings.json': mockSettings, }); util.listDatasets(projectId, computeRegion); @@ -347,9 +381,14 @@ describe('createModel()', function() { createModel: create, }); + const mockSettings = { + secretToken: 'foo', + }; + const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, + '../functions/settings.json': mockSettings, }); await util.createModel(projectId, computeRegion, '123456ABC', 'testModel'); @@ -368,8 +407,14 @@ describe('createModel()', function() { }); const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + + const mockSettings = { + secretToken: 'foo', + }; + const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, + '../functions/settings.json': mockSettings, }); await util.createModel(projectId, computeRegion, '123456ABC', 'testModel'); From d43b5b680cb1d8e9067d9ef7978bddd56247d163 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Wed, 24 Oct 2018 13:52:52 -0700 Subject: [PATCH 04/23] mocked settings --- system-test/functions_system_test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/system-test/functions_system_test.js b/system-test/functions_system_test.js index 0905fa2f..15831966 100644 --- a/system-test/functions_system_test.js +++ b/system-test/functions_system_test.js @@ -78,7 +78,6 @@ describe('handleNewIssue()', function() { }); app.post(`/handleNewIssue`, functs.handleNewIssue); - // }); }); it('should validate request', function(done) { From 1ce050d46d2f0c362bc34d999f183cc44b13e2c7 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Wed, 24 Oct 2018 14:37:17 -0700 Subject: [PATCH 05/23] fixing util require --- system-test/functions_system_test.js | 2 +- test/triage_test.js | 2 +- test/util_test.js | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/system-test/functions_system_test.js b/system-test/functions_system_test.js index 15831966..18b4cecb 100644 --- a/system-test/functions_system_test.js +++ b/system-test/functions_system_test.js @@ -4,7 +4,7 @@ const mocha = require('mocha'); const describe = mocha.describe; const it = mocha.it; const beforeEach = mocha.beforeEach; -const proxyquire = require('proxyquire'); +const proxyquire = require('proxyquire').noCallThru(); const sinon = require('sinon'); const assert = require('assert'); const supertest = require(`supertest`); diff --git a/test/triage_test.js b/test/triage_test.js index ebade33d..ecbf222b 100644 --- a/test/triage_test.js +++ b/test/triage_test.js @@ -1,7 +1,7 @@ const mocha = require('mocha'); const describe = mocha.describe; const it = mocha.it; -const proxyquire = require('proxyquire'); +const proxyquire = require('proxyquire').noCallThru(); const sinon = require('sinon'); const assert = require('assert'); diff --git a/test/util_test.js b/test/util_test.js index 60477d57..324e80da 100644 --- a/test/util_test.js +++ b/test/util_test.js @@ -9,6 +9,8 @@ const sinon = require('sinon'); const assert = require('assert'); describe('makeCSV()', function() { + const util = require('../src/util.js'); + it('should create a csv of issues', function() { const issues = [ { @@ -119,8 +121,10 @@ describe('retrieveIssues', () => { }); describe('getIssueInfo()', function() { - let originalIssue, returnedIssue, labelCount; + let originalIssue, returnedIssue, labelCount, util; beforeEach(() => { + util = require('../src/util.js'); + originalIssue = { id: 1, node_id: 'MDU6SXNWUx', From 5c33e2f196d2d006e809fae15321bf65a8a0bb39 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Wed, 24 Oct 2018 14:39:58 -0700 Subject: [PATCH 06/23] removed util require --- test/util_test.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/util_test.js b/test/util_test.js index 324e80da..c7bb879d 100644 --- a/test/util_test.js +++ b/test/util_test.js @@ -1,4 +1,3 @@ -const util = require('../src/util.js'); const fs = require('fs'); const mocha = require('mocha'); const describe = mocha.describe; @@ -9,7 +8,7 @@ const sinon = require('sinon'); const assert = require('assert'); describe('makeCSV()', function() { - const util = require('../src/util.js'); + const util = proxyquire('../src/util.js', {}); it('should create a csv of issues', function() { const issues = [ @@ -123,7 +122,7 @@ describe('retrieveIssues', () => { describe('getIssueInfo()', function() { let originalIssue, returnedIssue, labelCount, util; beforeEach(() => { - util = require('../src/util.js'); + util = proxyquire('../src/util.js', {}); originalIssue = { id: 1, From 714f0cb3a700ea5c207cd868ee585e21f07b302a Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Wed, 24 Oct 2018 14:44:02 -0700 Subject: [PATCH 07/23] updated proxyquire --- test/util_test.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/util_test.js b/test/util_test.js index c7bb879d..9f9bf17a 100644 --- a/test/util_test.js +++ b/test/util_test.js @@ -8,7 +8,13 @@ const sinon = require('sinon'); const assert = require('assert'); describe('makeCSV()', function() { - const util = proxyquire('../src/util.js', {}); + const mockSettings = { + secretToken: 'foo', + }; + + const util = proxyquire('../src/util.js', { + '../functions/settings.json': mockSettings, + }); it('should create a csv of issues', function() { const issues = [ @@ -122,7 +128,13 @@ describe('retrieveIssues', () => { describe('getIssueInfo()', function() { let originalIssue, returnedIssue, labelCount, util; beforeEach(() => { - util = proxyquire('../src/util.js', {}); + const mockSettings = { + secretToken: 'foo', + }; + + util = proxyquire('../src/util.js', { + '../functions/settings.json': mockSettings, + }); originalIssue = { id: 1, From bb4065971c7f1241183ba8f6112755f4ee6024d8 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Wed, 24 Oct 2018 14:50:09 -0700 Subject: [PATCH 08/23] updated proxyquire. again. --- test/util_test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/util_test.js b/test/util_test.js index 9f9bf17a..a801fbe6 100644 --- a/test/util_test.js +++ b/test/util_test.js @@ -3,7 +3,7 @@ const mocha = require('mocha'); const describe = mocha.describe; const it = mocha.it; const beforeEach = mocha.beforeEach; -const proxyquire = require('proxyquire'); +const proxyquire = require('proxyquire').noCallThru(); const sinon = require('sinon'); const assert = require('assert'); From be743bdcc2c9ca8ab391ae582542f591e286d3e3 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Wed, 24 Oct 2018 14:56:13 -0700 Subject: [PATCH 09/23] removed unused from package.json, updated triage test --- functions/package.json | 2 -- test/triage_test.js | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/package.json b/functions/package.json index f880f007..a9709cf6 100644 --- a/functions/package.json +++ b/functions/package.json @@ -5,9 +5,7 @@ }, "dependencies": { "@google-cloud/automl": "^0.1.2", - "@google-cloud/datastore": "^2.0.0", "@google-cloud/pubsub": "^0.20.1", - "@google-cloud/storage": "^2.1.0", "@octokit/rest": "^15.15.1", "loglevel": "^1.6.1" } diff --git a/test/triage_test.js b/test/triage_test.js index ecbf222b..aa7daea6 100644 --- a/test/triage_test.js +++ b/test/triage_test.js @@ -104,6 +104,7 @@ describe('triage()', function() { functions = proxyquire('../functions/index.js', { '@octokit/rest': () => octoMock, '@google-cloud/automl': autoMlMock, + '@google-cloud/pubsub': sinon.stub(), './settings.json': mockSettings, }); @@ -119,6 +120,7 @@ describe('triage()', function() { it('should throw error if unauthorized gitHub user', async () => { functions = proxyquire('../functions/index.js', { './settings.json': mockSettings, + '@google-cloud/pubsub': sinon.stub(), }); let result = await functions.triage({data: {data: dataBuffer}}); From 2cfc3af6f158c4b383b8999a28917e5b89c198fb Mon Sep 17 00:00:00 2001 From: Jason Dobry Date: Wed, 24 Oct 2018 15:03:16 -0700 Subject: [PATCH 10/23] Update CI config. --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1795dddb..ea7a3068 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,9 +42,12 @@ jobs: command: |- mkdir -p /home/node/.npm-global ./.circleci/npm-install-retry.js + cd functions + ../.circleci/npm-install-retry.js + cd .. environment: NPM_CONFIG_PREFIX: /home/node/.npm-global - - run: cp defaultsettings.json settings.json + - run: cp functions/defaultsettings.json functions/settings.json - run: npm test - run: node_modules/.bin/codecov node10: From c042958a66163b47a175748760ce1d8beaf0b94e Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Fri, 26 Oct 2018 12:46:15 -0700 Subject: [PATCH 11/23] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b4c8aba4..fa6bed6d 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,9 @@ animal, that's why you need LabelCat. 1. `cd LabelCat` 1. `npm install` 1. `npm link .` -1. `cp defaultsettings.json settings.json` (`settings.json` is where you +1. `cd functions` + + `cp defaultsettings.json settings.json` (`settings.json` is where you customize the app) 1. Modify `settings.json` as necessary. From beda0256d312121ed95e2c4e115496a2d7796475 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Sat, 3 Nov 2018 16:04:33 -0700 Subject: [PATCH 12/23] DRYed up testing, fixed inconsistent variables, addressed PR comments --- README.md | 6 +- functions/index.js | 15 ++-- ...efaultsettings.json => settings.tmpl.json} | 0 src/util.js | 1 + system-test/functions_system_test.js | 39 ++++---- system-test/issuePayload.js | 2 +- test/triage_test.js | 65 ++++++-------- test/util_test.js | 88 ++++++++++--------- 8 files changed, 106 insertions(+), 110 deletions(-) rename functions/{defaultsettings.json => settings.tmpl.json} (100%) diff --git a/README.md b/README.md index b4c8aba4..8dca2745 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,9 @@ animal, that's why you need LabelCat. 1. `cd LabelCat` 1. `npm install` 1. `npm link .` -1. `cp defaultsettings.json settings.json` (`settings.json` is where you +1. `cd functions` + + `cp settings.tmpl.json settings.json` (`settings.json` is where you customize the app) 1. Modify `settings.json` as necessary. @@ -172,7 +174,7 @@ Examples: ## Deploy Cloud Functions 1. `cd Functions` 1. `npm install` -1. `cp defaultsettings.json settings.json` +1. `cp settings.tmpl.json settings.json` 1. Modify `settings.json` as necessary. 1. `gcloud alpha functions deploy handleNewIssue --trigger-http --runtime nodejs8` 1. `gcloud functions deploy triage --runtime nodejs8 --trigger-resource YOUR-PUB/SUB-TOPIC-NAME --trigger-event google.pubsub.topic.publish diff --git a/functions/index.js b/functions/index.js index 356f99f5..6d902ead 100644 --- a/functions/index.js +++ b/functions/index.js @@ -5,17 +5,14 @@ log.setLevel('info'); const automl = require('@google-cloud/automl'); const PubSub = require(`@google-cloud/pubsub`); const octokit = require('@octokit/rest')(); +const client = new automl.v1beta1.PredictionServiceClient(); -/** - * TODO(developer): Set the following... - */ -// Your Google Cloud Platform project ID & compute region const projectId = settings.projectId; const computeRegion = settings.computeRegion; -// Your Google Cloud AutoML NL model ID const modelId = settings.modelId; -// Your Google Cloud Pub/Sub topic const topicName = settings.topicName; +const SCORE_THRESHOLD = 70; + /** * Verifies request has come from Github and publishes * message to specified Pub/Sub topic. @@ -55,7 +52,7 @@ function validateRequest(req) { async function publishMessage(req) { try { - const text = req.body.issue.title + ' ' + req.body.issue.body; + const text = `${req.body.issue.title} ${req.body.issue.body}`; const data = JSON.stringify({ owner: req.body.repository.owner.login, repo: req.body.repository.name, @@ -116,8 +113,6 @@ async function triage(event, res) { } async function predict(text) { - const client = new automl.v1beta1.PredictionServiceClient(); - const modelFullId = client.modelPath(projectId, computeRegion, modelId); const payload = { @@ -134,7 +129,7 @@ async function predict(text) { params: {}, }); - if (response[0].payload[1].classification.score > 89) { + if (response[0].payload[1].classification.score > SCORE_THRESHOLD) { return true; } diff --git a/functions/defaultsettings.json b/functions/settings.tmpl.json similarity index 100% rename from functions/defaultsettings.json rename to functions/settings.tmpl.json diff --git a/src/util.js b/src/util.js index ddb950c2..6512544c 100755 --- a/src/util.js +++ b/src/util.js @@ -106,6 +106,7 @@ function cleanLabels(issue, opts) { function getIssueInfo(issue) { try { const raw = issue.title + ' ' + issue.body; + // remove punctuation that will interfere with csv const text = raw.replace(/[^\w\s]/gi, ''); const labels = issue.labels.map(labelObject => labelObject.name); diff --git a/system-test/functions_system_test.js b/system-test/functions_system_test.js index 18b4cecb..84bf2b57 100644 --- a/system-test/functions_system_test.js +++ b/system-test/functions_system_test.js @@ -11,7 +11,7 @@ const supertest = require(`supertest`); const bodyParser = require('body-parser'); const crypto = require('crypto'); -const mockSettings = { +const settingsMock = { secretToken: 'foo', }; @@ -20,7 +20,7 @@ function setup() { return { getHeader: () => { const digest = crypto - .createHmac('sha1', mockSettings.secretToken) + .createHmac('sha1', settingsMock.secretToken) .update(JSON.stringify(event.body)) .digest('hex'); return `sha1=${digest}`; @@ -41,45 +41,50 @@ describe('handleNewIssue()', function() { app = express(); const requestLimit = '1024mb'; - const rawBodySaver = (req, res, buf) => { + const rawBody = (req, res, buf) => { req.rawBody = buf; }; - const defaultBodySavingOptions = { + const defaultBodyOptions = { limit: requestLimit, - verify: rawBodySaver, + verify: rawBody, }; - const rawBodySavingOptions = { + const rawBodyOptions = { limit: requestLimit, - verify: rawBodySaver, + verify: rawBody, type: '*/*', }; // Use extended query string parsing for URL-encoded bodies. const urlEncodedOptions = { limit: requestLimit, - verify: rawBodySaver, + verify: rawBody, extended: true, }; // Parse request body - app.use(bodyParser.json(defaultBodySavingOptions)); - app.use(bodyParser.text(defaultBodySavingOptions)); + app.use(bodyParser.json(defaultBodyOptions)); + app.use(bodyParser.text(defaultBodyOptions)); app.use(bodyParser.urlencoded(urlEncodedOptions)); // MUST be last in the list of body parsers as subsequent parsers will be // skipped when one is matched. - app.use(bodyParser.raw(rawBodySavingOptions)); + app.use(bodyParser.raw(rawBodyOptions)); functs = proxyquire('../functions/index.js', { '@google-cloud/pubsub': pubsubMock, - './settings.json': mockSettings, + './settings.json': settingsMock, }); app.post(`/handleNewIssue`, functs.handleNewIssue); }); + afterEach(() => { + issueEvent.body.action = 'opened'; + issueEvent.body.issue.title = 'LABELCAT-TEST'; + }); + it('should validate request', function(done) { supertest(app) .post(`/handleNewIssue`) @@ -106,11 +111,10 @@ describe('handleNewIssue()', function() { }); it('should return if action is not opened', function(done) { - let wrongAction = issueEvent; - wrongAction.body.action = 'edited'; + issueEvent.body.action = 'edited'; supertest(app) .post(`/handleNewIssue`) - .send(wrongAction.body) + .send(issueEvent.body) .set('x-hub-signature', codeUnderTest.getHeader()) .end(function(err, res) { assert.strictEqual(400, res.statusCode); @@ -120,9 +124,8 @@ describe('handleNewIssue()', function() { }); it('should not publish request with incorrect data', function(done) { - let wrongAction = issueEvent; - wrongAction.body.action = 'opened'; - wrongAction.body.issue.title = undefined; + issueEvent.body.action = 'opened'; + issueEvent.body.issue.title = undefined; supertest(app) .post(`/handleNewIssue`) .send({body: null}) diff --git a/system-test/issuePayload.js b/system-test/issuePayload.js index 239e2abb..3f86b52f 100644 --- a/system-test/issuePayload.js +++ b/system-test/issuePayload.js @@ -7,7 +7,7 @@ module.exports = { labels: [ { id: 949737505, - node_id: 'MDU6TGFiZWw5NDk3Mzc1MDU=', + node_id: 'MDU6TGFk3Mzc1MDU=', url: 'https://api.github.com/repos/Codertocat/Hello-World/labels/bug', name: 'bug', color: 'd73a4a', diff --git a/test/triage_test.js b/test/triage_test.js index aa7daea6..bc5bd81d 100644 --- a/test/triage_test.js +++ b/test/triage_test.js @@ -5,13 +5,26 @@ const proxyquire = require('proxyquire').noCallThru(); const sinon = require('sinon'); const assert = require('assert'); -let functions, autoMlMock, octoMock, dataBuffer, mockSettings; +// update this value to match SCORE_THRESHOLD in functions/index.js +const SCORE_THRESHOLD = 70; +const ISSUE_NUMBER = 22; + +let functions, autoMlMock, octoMock, dataBuffer, settingsMock; + +const makePayload = (classification, displayName) => { + return { + annotationSpecId: '', + displayName: displayName, + classification: classification, + detail: 'classification', + }; +}; beforeEach(() => { const data = JSON.stringify({ owner: 'GoogleCloudPlatform', repo: 'labelcat', - number: 22, + number: ISSUE_NUMBER, text: 'some issue information', }); @@ -25,7 +38,7 @@ beforeEach(() => { data: [ { id: 271022241, - node_id: 'MDU6TGFiZWwyNzEwMjIyNDE=', + node_id: 'MDwMjIyNDE=', url: 'https://api.github.com/repos/GoogleCloudPlatform/LabelCat/labels/bug', name: 'bug', @@ -47,50 +60,27 @@ beforeEach(() => { predict.onCall(0).returns([ { - payload: [ - { - annotationSpecId: '', - displayName: '0', - classification: [Object], - detail: 'classification', - }, - { - annotationSpecId: '', - displayName: '1', - classification: {score: 90}, - detail: 'classification', - }, - ], + payload: [makePayload([Object], 0), makePayload({score: 90}, 1)], }, ]); predict.onCall(1).returns([ { payload: [ - { - annotationSpecId: '', - displayName: '0', - classification: [Object], - detail: 'classification', - }, - { - annotationSpecId: '', - displayName: '1', - classification: {score: 80}, - detail: 'classification', - }, + makePayload([Object], 0), + makePayload({score: SCORE_THRESHOLD}, 1), ], }, ]); - const mockClient = sinon.stub().returns({ + const clientMock = sinon.stub().returns({ modelPath: model, predict: predict, }); - autoMlMock = {v1beta1: {PredictionServiceClient: mockClient}}; + autoMlMock = {v1beta1: {PredictionServiceClient: clientMock}}; - mockSettings = { + settingsMock = { secretToken: 'foo', projectId: 'test-project', computeRegion: 'uscentral', @@ -105,26 +95,25 @@ describe('triage()', function() { '@octokit/rest': () => octoMock, '@google-cloud/automl': autoMlMock, '@google-cloud/pubsub': sinon.stub(), - './settings.json': mockSettings, + './settings.json': settingsMock, }); let result = await functions.triage({data: {data: dataBuffer}}); - assert(result.labeled === true); - assert(result.number === 22); + assert(result.number === ISSUE_NUMBER); result = await functions.triage({data: {data: dataBuffer}}); assert(result.labeled === false); - assert(result.number === 22); + assert(result.number === ISSUE_NUMBER); }); it('should throw error if unauthorized gitHub user', async () => { functions = proxyquire('../functions/index.js', { - './settings.json': mockSettings, + './settings.json': settingsMock, '@google-cloud/pubsub': sinon.stub(), }); let result = await functions.triage({data: {data: dataBuffer}}); assert(result.labeled === false); - assert(result.number === 22); + assert(result.number === ISSUE_NUMBER); }); }); diff --git a/test/util_test.js b/test/util_test.js index a801fbe6..ee307cc4 100644 --- a/test/util_test.js +++ b/test/util_test.js @@ -8,12 +8,12 @@ const sinon = require('sinon'); const assert = require('assert'); describe('makeCSV()', function() { - const mockSettings = { + const settingsMock = { secretToken: 'foo', }; const util = proxyquire('../src/util.js', { - '../functions/settings.json': mockSettings, + '../functions/settings.json': settingsMock, }); it('should create a csv of issues', function() { @@ -74,22 +74,22 @@ describe('retrieveIssues', () => { hasNext.returns(true); hasNext.onCall(1).returns(false); - const mockGet = sinon.stub().returns(Promise.resolve(issueData)); + const getMock = sinon.stub().returns(Promise.resolve(issueData)); octoMock = { authenticate: sinon.stub(), - issues: {getForRepo: mockGet}, + issues: {getForRepo: getMock}, hasNextPage: hasNext, getNextPage: getNext, }; - const mockSettings = { + const settingsMock = { secretToken: 'foo', }; util = proxyquire('../src/util.js', { '@octokit/rest': () => octoMock, - '../functions/settings.json': mockSettings, + '../functions/settings.json': settingsMock, }); }); @@ -104,6 +104,7 @@ describe('retrieveIssues', () => { assert(result[1].text === 'another issue more details'); assert(result[1].label === 1); }); + it('should throw an error', async () => { let label = 'type: bug'; @@ -127,13 +128,14 @@ describe('retrieveIssues', () => { describe('getIssueInfo()', function() { let originalIssue, returnedIssue, labelCount, util; + beforeEach(() => { - const mockSettings = { + const settingsMock = { secretToken: 'foo', }; util = proxyquire('../src/util.js', { - '../functions/settings.json': mockSettings, + '../functions/settings.json': settingsMock, }); originalIssue = { @@ -193,25 +195,26 @@ describe('createDataset()', function() { }, ]); - const mockClient = sinon.stub().returns({ + const clientMock = sinon.stub().returns({ locationPath: location, createDataset: create, }); - const mockSettings = { + const settingsMock = { secretToken: 'foo', }; - const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, - '../functions/settings.json': mockSettings, + '../functions/settings.json': settingsMock, }); util.createDataset(projectId, computeRegion, datasetName, multiLabel); sinon.assert.calledOnce(location); assert(location.calledWith(projectId, computeRegion)); }); + it('should throw an error', function() { const location = sinon.spy(); const create = sinon.stub().returns([ @@ -225,19 +228,19 @@ describe('createDataset()', function() { }, ]); - const mockClient = sinon.stub().returns({ + const clientMock = sinon.stub().returns({ locationPath: location, createDataset: create, }); - const mockSettings = { + const settingsMock = { secretToken: 'foo', }; - const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, - '../functions/settings.json': mockSettings, + '../functions/settings.json': settingsMock, }); util.createDataset(projectId, computeRegion, datasetName, multiLabel); @@ -254,20 +257,20 @@ describe('importData()', function() { const path = sinon.spy(); const imports = sinon.stub().returns(); - const mockClient = sinon.stub().returns({ + const clientMock = sinon.stub().returns({ datasetPath: path, importData: imports, }); - const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - const mockSettings = { + const settingsMock = { secretToken: 'foo', }; const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, - '../functions/settings.json': mockSettings, + '../functions/settings.json': settingsMock, }); util.importData(projectId, computeRegion, datasetId, file); @@ -275,24 +278,25 @@ describe('importData()', function() { assert(path.calledWith(projectId, computeRegion, datasetId)); sinon.assert.calledOnce(imports); }); + it('should throw an error', function() { const path = sinon.spy(); const imports = sinon.stub().throws(); - const mockClient = sinon.stub().returns({ + const clientMock = sinon.stub().returns({ datasetPath: path, importData: imports, }); - const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - const mockSettings = { + const settingsMock = { secretToken: 'foo', }; const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, - '../functions/settings.json': mockSettings, + '../functions/settings.json': settingsMock, }); util.importData(projectId, computeRegion, datasetId, file); @@ -318,20 +322,20 @@ describe('listDatasets()', function() { ], ]); - const mockClient = sinon.stub().returns({ + const clientMock = sinon.stub().returns({ locationPath: location, listDatasets: list, }); - const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - const mockSettings = { + const settingsMock = { secretToken: 'foo', }; const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, - '../functions/settings.json': mockSettings, + '../functions/settings.json': settingsMock, }); await util.listDatasets(projectId, computeRegion); @@ -340,6 +344,7 @@ describe('listDatasets()', function() { sinon.assert.calledOnce(location); assert(location.calledWith(projectId, computeRegion)); }); + it('should throw an error', function() { const location = sinon.spy(); const list = sinon.stub().returns([ @@ -351,26 +356,26 @@ describe('listDatasets()', function() { ], ]); - const mockClient = sinon.stub().returns({ + const clientMock = sinon.stub().returns({ locationPath: location, listDatasets: list, }); - const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - const mockSettings = { + const settingsMock = { secretToken: 'foo', }; const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, - '../functions/settings.json': mockSettings, + '../functions/settings.json': settingsMock, }); util.listDatasets(projectId, computeRegion); }); }); -// createModel(projectId, computeRegion, datasetId, modelName); + describe('createModel()', function() { const projectId = 'test-project'; const computeRegion = 'us-central1'; @@ -391,19 +396,19 @@ describe('createModel()', function() { }, ]); - const mockClient = sinon.stub().returns({ + const clientMock = sinon.stub().returns({ locationPath: location, createModel: create, }); - const mockSettings = { + const settingsMock = { secretToken: 'foo', }; - const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, - '../functions/settings.json': mockSettings, + '../functions/settings.json': settingsMock, }); await util.createModel(projectId, computeRegion, '123456ABC', 'testModel'); @@ -412,24 +417,25 @@ describe('createModel()', function() { sinon.assert.calledOnce(location); assert(location.calledWith(projectId, computeRegion)); }); + it('should throw an error', async function() { const location = sinon.spy(); const create = sinon.stub().returns([]); - const mockClient = sinon.stub().returns({ + const clientMock = sinon.stub().returns({ locationPath: location, createModel: create, }); - const autoMlMock = {v1beta1: {AutoMlClient: mockClient}}; + const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - const mockSettings = { + const settingsMock = { secretToken: 'foo', }; const util = proxyquire('../src/util.js', { '@google-cloud/automl': autoMlMock, - '../functions/settings.json': mockSettings, + '../functions/settings.json': settingsMock, }); await util.createModel(projectId, computeRegion, '123456ABC', 'testModel'); From 45234d9b093bb1232dca9088b0f3f1d6283d48e9 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Sat, 3 Nov 2018 16:09:25 -0700 Subject: [PATCH 13/23] changed name for default settings file --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea7a3068..df12de0a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ jobs: cd .. environment: NPM_CONFIG_PREFIX: /home/node/.npm-global - - run: cp functions/defaultsettings.json functions/settings.json + - run: cp functions/settings.tmpl.json functions/settings.json - run: npm test - run: node_modules/.bin/codecov node10: From e949bcc14c5a7d868b328a32108fea2d0de36787 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Sat, 3 Nov 2018 16:15:27 -0700 Subject: [PATCH 14/23] reverted beforeEach in functions_system_test --- system-test/functions_system_test.js | 82 ++++++++++++++-------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/system-test/functions_system_test.js b/system-test/functions_system_test.js index 84bf2b57..4094b5f9 100644 --- a/system-test/functions_system_test.js +++ b/system-test/functions_system_test.js @@ -37,47 +37,47 @@ describe('handleNewIssue()', function() { const pubsubMock = sinon.stub().returns({topic: topicMock}); beforeEach(() => { - codeUnderTest = setup(); - app = express(); - const requestLimit = '1024mb'; - - const rawBody = (req, res, buf) => { - req.rawBody = buf; - }; - - const defaultBodyOptions = { - limit: requestLimit, - verify: rawBody, - }; - - const rawBodyOptions = { - limit: requestLimit, - verify: rawBody, - type: '*/*', - }; - - // Use extended query string parsing for URL-encoded bodies. - const urlEncodedOptions = { - limit: requestLimit, - verify: rawBody, - extended: true, - }; - - // Parse request body - app.use(bodyParser.json(defaultBodyOptions)); - app.use(bodyParser.text(defaultBodyOptions)); - app.use(bodyParser.urlencoded(urlEncodedOptions)); - - // MUST be last in the list of body parsers as subsequent parsers will be - // skipped when one is matched. - app.use(bodyParser.raw(rawBodyOptions)); - - functs = proxyquire('../functions/index.js', { - '@google-cloud/pubsub': pubsubMock, - './settings.json': settingsMock, - }); - - app.post(`/handleNewIssue`, functs.handleNewIssue); + codeUnderTest = setup(); + app = express(); + const requestLimit = '1024mb'; + + const rawBodySaver = (req, res, buf) => { + req.rawBody = buf; + }; + + const defaultBodySavingOptions = { + limit: requestLimit, + verify: rawBodySaver, + }; + + const rawBodySavingOptions = { + limit: requestLimit, + verify: rawBodySaver, + type: '*/*', + }; + + // Use extended query string parsing for URL-encoded bodies. + const urlEncodedOptions = { + limit: requestLimit, + verify: rawBodySaver, + extended: true, + }; + + // Parse request body + app.use(bodyParser.json(defaultBodySavingOptions)); + app.use(bodyParser.text(defaultBodySavingOptions)); + app.use(bodyParser.urlencoded(urlEncodedOptions)); + + // MUST be last in the list of body parsers as subsequent parsers will be + // skipped when one is matched. + app.use(bodyParser.raw(rawBodySavingOptions)); + + functs = proxyquire('../functions/index.js', { + '@google-cloud/pubsub': pubsubMock, + './settings.json': settingsMock, + }); + + app.post(`/handleNewIssue`, functs.handleNewIssue); }); afterEach(() => { From 2c22416e59e22074d19aea26385a4463f5aeb546 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Sat, 3 Nov 2018 16:28:09 -0700 Subject: [PATCH 15/23] debug --- system-test/functions_system_test.js | 82 ++++++++++++++-------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/system-test/functions_system_test.js b/system-test/functions_system_test.js index 4094b5f9..56a223f6 100644 --- a/system-test/functions_system_test.js +++ b/system-test/functions_system_test.js @@ -37,47 +37,47 @@ describe('handleNewIssue()', function() { const pubsubMock = sinon.stub().returns({topic: topicMock}); beforeEach(() => { - codeUnderTest = setup(); - app = express(); - const requestLimit = '1024mb'; - - const rawBodySaver = (req, res, buf) => { - req.rawBody = buf; - }; - - const defaultBodySavingOptions = { - limit: requestLimit, - verify: rawBodySaver, - }; - - const rawBodySavingOptions = { - limit: requestLimit, - verify: rawBodySaver, - type: '*/*', - }; - - // Use extended query string parsing for URL-encoded bodies. - const urlEncodedOptions = { - limit: requestLimit, - verify: rawBodySaver, - extended: true, - }; - - // Parse request body - app.use(bodyParser.json(defaultBodySavingOptions)); - app.use(bodyParser.text(defaultBodySavingOptions)); - app.use(bodyParser.urlencoded(urlEncodedOptions)); - - // MUST be last in the list of body parsers as subsequent parsers will be - // skipped when one is matched. - app.use(bodyParser.raw(rawBodySavingOptions)); - - functs = proxyquire('../functions/index.js', { - '@google-cloud/pubsub': pubsubMock, - './settings.json': settingsMock, - }); - - app.post(`/handleNewIssue`, functs.handleNewIssue); + codeUnderTest = setup(); + app = express(); + const requestLimit = '1024mb'; + + const rawBodySaver = (req, res, buf) => { + req.rawBody = buf; + }; + + const defaultBodySavingOptions = { + limit: requestLimit, + verify: rawBodySaver, + }; + + const rawBodySavingOptions = { + limit: requestLimit, + verify: rawBodySaver, + type: '*/*', + }; + + // Use extended query string parsing for URL-encoded bodies. + const urlEncodedOptions = { + limit: requestLimit, + verify: rawBodySaver, + extended: true, + }; + + // Parse request body + app.use(bodyParser.json(defaultBodySavingOptions)); + app.use(bodyParser.text(defaultBodySavingOptions)); + app.use(bodyParser.urlencoded(urlEncodedOptions)); + + // MUST be last in the list of body parsers as subsequent parsers will be + // skipped when one is matched. + app.use(bodyParser.raw(rawBodySavingOptions)); + + functs = proxyquire('../functions/index.js', { + '@google-cloud/pubsub': pubsubMock, + './settings.json': settingsMock, + }); + + app.post(`/handleNewIssue`, functs.handleNewIssue); }); afterEach(() => { From 9104847df4e68fe62490e599ceada3f220f596ba Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Tue, 6 Nov 2018 11:13:39 -0800 Subject: [PATCH 16/23] updated constants and other variable names for consistency --- defaultsettings.json | 7 ------- functions/index.js | 18 +++++++++--------- functions/package.json | 3 ++- package.json | 5 +++-- src/util.js | 28 ++++++++++++++-------------- system-test/functions_system_test.js | 4 ++-- test/triage_test.js | 10 +++++----- 7 files changed, 35 insertions(+), 40 deletions(-) delete mode 100644 defaultsettings.json diff --git a/defaultsettings.json b/defaultsettings.json deleted file mode 100644 index ddfa2ebe..00000000 --- a/defaultsettings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "secretToken": "YOUR GITHUB CLIENT ID HERE", - "projectID": "YOUR GCP PROJECT ID HERE", - "computeRegion": "YOUR GCP PROJECT COMPUTE REGION HERE", - "modelId": "YOUR AUTOML NL MODEL ID HERE", - "topicName": "YOUR PUB/SUB TOPIC NAME HERE" -} diff --git a/functions/index.js b/functions/index.js index 6d902ead..0b2bf9a8 100644 --- a/functions/index.js +++ b/functions/index.js @@ -7,10 +7,10 @@ const PubSub = require(`@google-cloud/pubsub`); const octokit = require('@octokit/rest')(); const client = new automl.v1beta1.PredictionServiceClient(); -const projectId = settings.projectId; -const computeRegion = settings.computeRegion; -const modelId = settings.modelId; -const topicName = settings.topicName; +const PROJECT_ID = settings.PROJECT_ID; +const COMPUTE_REGION = settings.COMPUTE_REGION; +const MODEL_ID = settings.MODEL_ID; +const TOPIC_NAME = settings.TOPIC_NAME; const SCORE_THRESHOLD = 70; /** @@ -39,7 +39,7 @@ async function handleNewIssue(req, res) { function validateRequest(req) { return Promise.resolve().then(() => { const digest = crypto - .createHmac('sha1', settings.secretToken) + .createHmac('sha1', settings.SECRET_TOKEN) .update(JSON.stringify(req.body)) .digest('hex'); if (req.headers['x-hub-signature'] !== `sha1=${digest}`) { @@ -62,11 +62,11 @@ async function publishMessage(req) { const dataBuffer = Buffer.from(data); const pubsubClient = new PubSub({ - projectId: projectId, + PROJECT_ID: PROJECT_ID, }); const response = await pubsubClient - .topic(topicName) + .topic(TOPIC_NAME) .publisher() .publish(dataBuffer); return response; @@ -78,7 +78,7 @@ async function publishMessage(req) { async function triage(event, res) { octokit.authenticate({ type: 'oauth', - token: settings.secretToken, + token: settings.SECRET_TOKEN, }); const pubSubMessage = event.data; @@ -113,7 +113,7 @@ async function triage(event, res) { } async function predict(text) { - const modelFullId = client.modelPath(projectId, computeRegion, modelId); + const modelFullId = client.modelPath(PROJECT_ID, COMPUTE_REGION, MODEL_ID); const payload = { textSnippet: { diff --git a/functions/package.json b/functions/package.json index a9709cf6..99e559ca 100644 --- a/functions/package.json +++ b/functions/package.json @@ -7,6 +7,7 @@ "@google-cloud/automl": "^0.1.2", "@google-cloud/pubsub": "^0.20.1", "@octokit/rest": "^15.15.1", - "loglevel": "^1.6.1" + "loglevel": "^1.6.1", + "nconf": "^0.10.0" } } diff --git a/package.json b/package.json index e56f783b..4fcdfd6c 100644 --- a/package.json +++ b/package.json @@ -35,12 +35,13 @@ }, "dependencies": { "@google-cloud/automl": "^0.1.2", + "@octokit/rest": "^15.15.1", "csv-write-stream": "^2.0.0", "json2csv": "^4.2.1", "loglevel": "^1.6.1", + "nconf": "^0.10.0", "papaparse": "^4.6.1", - "yargs": "^12.0.2", - "@octokit/rest": "^15.15.1" + "yargs": "^12.0.2" }, "devDependencies": { "body-parser": "^1.18.3", diff --git a/src/util.js b/src/util.js index 6512544c..37b1b234 100755 --- a/src/util.js +++ b/src/util.js @@ -18,7 +18,7 @@ const automl = require(`@google-cloud/automl`); async function retrieveIssues(data, label, alternatives) { octokit.authenticate({ type: 'oauth', - token: settings.secretToken, + token: settings.SECRET_TOKEN, }); log.info('RETRIEVING ISSUES...'); @@ -135,20 +135,20 @@ function makeCSV(issues, file) { /** * Create a Google AutoML Natural Language dataset - * @param {string} projectId - * @param {string} computeRegion + * @param {string} PROJECT_ID + * @param {string} COMPUTE_REGION * @param {string} datasetName * @param {string} multiLabel */ async function createDataset( - projectId, - computeRegion, + PROJECT_ID, + COMPUTE_REGION, datasetName, multiLabel ) { const automl = require(`@google-cloud/automl`); const client = new automl.v1beta1.AutoMlClient(); - const projectLocation = client.locationPath(projectId, computeRegion); + const projectLocation = client.locationPath(PROJECT_ID, COMPUTE_REGION); // Classification type is assigned based on multiClass value. let classificationType = `MULTICLASS`; @@ -192,16 +192,16 @@ async function createDataset( /** * Import data into Google AutoML NL dataset - * @param {string} projectId - * @param {string} computeRegion + * @param {string} PROJECT_ID + * @param {string} COMPUTE_REGION * @param {string} datasetId * @param {string} path */ -async function importData(projectId, computeRegion, datasetId, path) { +async function importData(PROJECT_ID, COMPUTE_REGION, datasetId, path) { const client = new automl.v1beta1.AutoMlClient(); // Get the full path of the dataset. - const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId); + const datasetFullId = client.datasetPath(PROJECT_ID, COMPUTE_REGION, datasetId); // Get the Google Cloud Storage URIs. const inputUris = path.split(`,`); @@ -223,9 +223,9 @@ async function importData(projectId, computeRegion, datasetId, path) { } } -async function listDatasets(projectId, computeRegion) { +async function listDatasets(PROJECT_ID, COMPUTE_REGION) { const client = new automl.v1beta1.AutoMlClient(); - const projectLocation = client.locationPath(projectId, computeRegion); + const projectLocation = client.locationPath(PROJECT_ID, COMPUTE_REGION); try { const responses = await client.listDatasets({parent: projectLocation}); @@ -248,11 +248,11 @@ async function listDatasets(projectId, computeRegion) { } } -async function createModel(projectId, computeRegion, datasetId, modelName) { +async function createModel(PROJECT_ID, COMPUTE_REGION, datasetId, modelName) { const client = new automl.v1beta1.AutoMlClient(); // A resource that represents Google Cloud Platform location. - const projectLocation = client.locationPath(projectId, computeRegion); + const projectLocation = client.locationPath(PROJECT_ID, COMPUTE_REGION); // Set model name and model metadata for the dataset. const myModel = { diff --git a/system-test/functions_system_test.js b/system-test/functions_system_test.js index 56a223f6..0c30d528 100644 --- a/system-test/functions_system_test.js +++ b/system-test/functions_system_test.js @@ -12,7 +12,7 @@ const bodyParser = require('body-parser'); const crypto = require('crypto'); const settingsMock = { - secretToken: 'foo', + SECRET_TOKEN: 'foo', }; function setup() { @@ -20,7 +20,7 @@ function setup() { return { getHeader: () => { const digest = crypto - .createHmac('sha1', settingsMock.secretToken) + .createHmac('sha1', settingsMock.SECRET_TOKEN) .update(JSON.stringify(event.body)) .digest('hex'); return `sha1=${digest}`; diff --git a/test/triage_test.js b/test/triage_test.js index bc5bd81d..68e37667 100644 --- a/test/triage_test.js +++ b/test/triage_test.js @@ -81,11 +81,11 @@ beforeEach(() => { autoMlMock = {v1beta1: {PredictionServiceClient: clientMock}}; settingsMock = { - secretToken: 'foo', - projectId: 'test-project', - computeRegion: 'uscentral', - topicName: 'testTopic', - modelId: 'test-model', + SECRET_TOKEN: 'foo', + PROJECT_ID: 'test-project', + COMPUTE_REGION: 'uscentral', + TOPIC_NAME: 'testTopic', + MODEL_ID: 'test-model', }; }); From b5352e3eafd25247d64ab9bee4a9a89ecef49750 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Tue, 6 Nov 2018 12:07:21 -0800 Subject: [PATCH 17/23] corrected variables, reverted some refctoring --- functions/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functions/index.js b/functions/index.js index 0b2bf9a8..8b052d79 100644 --- a/functions/index.js +++ b/functions/index.js @@ -11,6 +11,7 @@ const PROJECT_ID = settings.PROJECT_ID; const COMPUTE_REGION = settings.COMPUTE_REGION; const MODEL_ID = settings.MODEL_ID; const TOPIC_NAME = settings.TOPIC_NAME; +const SECRET_TOKEN = settings.SECRET_TOKEN const SCORE_THRESHOLD = 70; /** @@ -39,7 +40,7 @@ async function handleNewIssue(req, res) { function validateRequest(req) { return Promise.resolve().then(() => { const digest = crypto - .createHmac('sha1', settings.SECRET_TOKEN) + .createHmac('sha1', SECRET_TOKEN) .update(JSON.stringify(req.body)) .digest('hex'); if (req.headers['x-hub-signature'] !== `sha1=${digest}`) { From 60f1526ec05577470da604003f4c7d369748d75f Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Tue, 6 Nov 2018 12:44:44 -0800 Subject: [PATCH 18/23] linted --- functions/index.js | 2 +- src/util.js | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/functions/index.js b/functions/index.js index 8b052d79..7eb1ffdb 100644 --- a/functions/index.js +++ b/functions/index.js @@ -11,7 +11,7 @@ const PROJECT_ID = settings.PROJECT_ID; const COMPUTE_REGION = settings.COMPUTE_REGION; const MODEL_ID = settings.MODEL_ID; const TOPIC_NAME = settings.TOPIC_NAME; -const SECRET_TOKEN = settings.SECRET_TOKEN +const SECRET_TOKEN = settings.SECRET_TOKEN; const SCORE_THRESHOLD = 70; /** diff --git a/src/util.js b/src/util.js index 37b1b234..7353a1d6 100755 --- a/src/util.js +++ b/src/util.js @@ -201,7 +201,11 @@ async function importData(PROJECT_ID, COMPUTE_REGION, datasetId, path) { const client = new automl.v1beta1.AutoMlClient(); // Get the full path of the dataset. - const datasetFullId = client.datasetPath(PROJECT_ID, COMPUTE_REGION, datasetId); + const datasetFullId = client.datasetPath( + PROJECT_ID, + COMPUTE_REGION, + datasetId + ); // Get the Google Cloud Storage URIs. const inputUris = path.split(`,`); From 19b9f21d9d3a15503f3651e5b71b04c37b2da52f Mon Sep 17 00:00:00 2001 From: Jason Dobry Date: Tue, 13 Nov 2018 23:34:01 -0800 Subject: [PATCH 19/23] Debug test --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4fcdfd6c..e99b455e 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "prettier": "prettier --write bin/*.js src/*.js test/*.js system-test/*.js functions/*.js", "system-test": "mocha system-test/*.js --timeout 600000", "test-no-cover": "mocha test/*.js", - "test": "npm run cover" + "test": "mocha system-test/functions_system_test.js --timeout 600000" }, "dependencies": { "@google-cloud/automl": "^0.1.2", From 331a53a74d4323e855a06dafb3c5b9039ab20f80 Mon Sep 17 00:00:00 2001 From: Jason Dobry Date: Wed, 14 Nov 2018 09:50:36 -0800 Subject: [PATCH 20/23] Fix tests. --- functions/index.js | 38 +++++++++++++--------------- package.json | 2 +- system-test/functions_system_test.js | 7 +++++ test/triage_test.js | 10 +++++--- 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/functions/index.js b/functions/index.js index 7eb1ffdb..db14271c 100644 --- a/functions/index.js +++ b/functions/index.js @@ -76,7 +76,7 @@ async function publishMessage(req) { } } -async function triage(event, res) { +async function triage(event, context) { octokit.authenticate({ type: 'oauth', token: settings.SECRET_TOKEN, @@ -89,28 +89,24 @@ async function triage(event, res) { const repo = issueData.repo; const number = issueData.number; - try { - issueData.labeled = false; - let results = await predict(issueData.text); - - if (results) { - const labels = ['bug']; - const response = await octokit.issues.addLabels({ - owner: owner, - repo: repo, - number: number, - labels: labels, - }); - - if (response.status === 200) { - issueData.labeled = true; - } - } + issueData.labeled = false; + let results = await predict(issueData.text); - return issueData; - } catch (err) { - res.status(403).send({error: err.message}); + if (results) { + const labels = ['bug']; + const response = await octokit.issues.addLabels({ + owner: owner, + repo: repo, + number: number, + labels: labels, + }); + + if (response.status === 200) { + issueData.labeled = true; + } } + + return issueData; } async function predict(text) { diff --git a/package.json b/package.json index e99b455e..4fcdfd6c 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "prettier": "prettier --write bin/*.js src/*.js test/*.js system-test/*.js functions/*.js", "system-test": "mocha system-test/*.js --timeout 600000", "test-no-cover": "mocha test/*.js", - "test": "mocha system-test/functions_system_test.js --timeout 600000" + "test": "npm run cover" }, "dependencies": { "@google-cloud/automl": "^0.1.2", diff --git a/system-test/functions_system_test.js b/system-test/functions_system_test.js index 0c30d528..8826d129 100644 --- a/system-test/functions_system_test.js +++ b/system-test/functions_system_test.js @@ -35,6 +35,12 @@ describe('handleNewIssue()', function() { const publisherMock = sinon.stub().returns({publish: publishMock}); const topicMock = sinon.stub().returns({publisher: publisherMock}); const pubsubMock = sinon.stub().returns({topic: topicMock}); + const automlClientMock = {}; + const automlMock = { + v1beta1: { + PredictionServiceClient: sinon.stub().returns(automlClientMock) + } + }; beforeEach(() => { codeUnderTest = setup(); @@ -73,6 +79,7 @@ describe('handleNewIssue()', function() { app.use(bodyParser.raw(rawBodySavingOptions)); functs = proxyquire('../functions/index.js', { + '@google-cloud/automl': automlMock, '@google-cloud/pubsub': pubsubMock, './settings.json': settingsMock, }); diff --git a/test/triage_test.js b/test/triage_test.js index 68e37667..248ba345 100644 --- a/test/triage_test.js +++ b/test/triage_test.js @@ -109,11 +109,15 @@ describe('triage()', function() { it('should throw error if unauthorized gitHub user', async () => { functions = proxyquire('../functions/index.js', { './settings.json': settingsMock, + '@google-cloud/automl': autoMlMock, '@google-cloud/pubsub': sinon.stub(), }); - let result = await functions.triage({data: {data: dataBuffer}}); - assert(result.labeled === false); - assert(result.number === ISSUE_NUMBER); + try { + await functions.triage({data: {data: dataBuffer}}); + assert.fail('triage should have failed'); + } catch (err) { + assert(JSON.parse(err.message).message === 'Bad credentials'); + } }); }); From 8f3b8dd6be1b1dbc118f1f2f9d5cd26ced51d44b Mon Sep 17 00:00:00 2001 From: Jason Dobry Date: Wed, 14 Nov 2018 10:30:24 -0800 Subject: [PATCH 21/23] Fix lint. --- functions/index.js | 2 +- system-test/functions_system_test.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/index.js b/functions/index.js index db14271c..1c2e9daa 100644 --- a/functions/index.js +++ b/functions/index.js @@ -76,7 +76,7 @@ async function publishMessage(req) { } } -async function triage(event, context) { +async function triage(event) { octokit.authenticate({ type: 'oauth', token: settings.SECRET_TOKEN, diff --git a/system-test/functions_system_test.js b/system-test/functions_system_test.js index 8826d129..345aa122 100644 --- a/system-test/functions_system_test.js +++ b/system-test/functions_system_test.js @@ -38,8 +38,8 @@ describe('handleNewIssue()', function() { const automlClientMock = {}; const automlMock = { v1beta1: { - PredictionServiceClient: sinon.stub().returns(automlClientMock) - } + PredictionServiceClient: sinon.stub().returns(automlClientMock), + }, }; beforeEach(() => { From 05d2b90412c330e08fde947848d1bf3f1bed1432 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Mon, 26 Nov 2018 00:46:54 -0800 Subject: [PATCH 22/23] cleaned up --- bin/labelcat.js | 2 +- functions/index.js | 14 ++++++++---- functions/package.json | 3 +-- package.json | 1 - src/util.js | 50 ++++++++++++++++++++++++++++++------------ 5 files changed, 48 insertions(+), 22 deletions(-) diff --git a/bin/labelcat.js b/bin/labelcat.js index af3acb91..42404194 100755 --- a/bin/labelcat.js +++ b/bin/labelcat.js @@ -76,7 +76,7 @@ require(`yargs`) ) .command( `listDatasets`, - `Train an AutoML NL model using existing dataset.`, + `Lists all AutoML NL datasets for current Google Cloud Platform project.`, {}, () => { const projectId = settings.projectId; diff --git a/functions/index.js b/functions/index.js index 7eb1ffdb..5407511f 100644 --- a/functions/index.js +++ b/functions/index.js @@ -1,11 +1,11 @@ const settings = require('./settings.json'); // eslint-disable-line node/no-missing-require const crypto = require('crypto'); -const log = require('loglevel'); -log.setLevel('info'); const automl = require('@google-cloud/automl'); const PubSub = require(`@google-cloud/pubsub`); const octokit = require('@octokit/rest')(); const client = new automl.v1beta1.PredictionServiceClient(); +const log = require('loglevel'); +log.setLevel('info'); const PROJECT_ID = settings.PROJECT_ID; const COMPUTE_REGION = settings.COMPUTE_REGION; @@ -76,13 +76,19 @@ async function publishMessage(req) { } } -async function triage(event, res) { +/** + * receives message from handleNewIssue cloud function, + * runs label prediction using assigned Google AutoML Natural Language model + * @param {object} req + * @param {object} res + */ +async function triage(req, res) { octokit.authenticate({ type: 'oauth', token: settings.SECRET_TOKEN, }); - const pubSubMessage = event.data; + const pubSubMessage = req.data; let issueData = Buffer.from(pubSubMessage.data, 'base64').toString(); issueData = JSON.parse(issueData); const owner = issueData.owner; diff --git a/functions/package.json b/functions/package.json index 99e559ca..a9709cf6 100644 --- a/functions/package.json +++ b/functions/package.json @@ -7,7 +7,6 @@ "@google-cloud/automl": "^0.1.2", "@google-cloud/pubsub": "^0.20.1", "@octokit/rest": "^15.15.1", - "loglevel": "^1.6.1", - "nconf": "^0.10.0" + "loglevel": "^1.6.1" } } diff --git a/package.json b/package.json index 4fcdfd6c..acbbf53f 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "csv-write-stream": "^2.0.0", "json2csv": "^4.2.1", "loglevel": "^1.6.1", - "nconf": "^0.10.0", "papaparse": "^4.6.1", "yargs": "^12.0.2" }, diff --git a/src/util.js b/src/util.js index 7353a1d6..8496dee3 100755 --- a/src/util.js +++ b/src/util.js @@ -3,17 +3,19 @@ const fs = require('fs'); const settings = require('../functions/settings.json'); // eslint-disable-line node/no-missing-require const octokit = require('@octokit/rest')(); -const log = require('loglevel'); const Papa = require('papaparse'); +const log = require('loglevel'); log.setLevel('info'); const automl = require(`@google-cloud/automl`); /** - * Take a filepath to a json object of issues - * and a filename to save the resulting issue data, + * Take a filepath to a json object of issues, + * the issue label to train the model on, + * and alternative names for the label, * then makes api call to GitHub to retrieve current data * @param {string} data - * @param {string} file + * @param {string} label + * @param {array} alternatives */ async function retrieveIssues(data, label, alternatives) { octokit.authenticate({ @@ -48,11 +50,11 @@ async function retrieveIssues(data, label, alternatives) { }); } - let opts = [label]; + let labelList = [label]; if (alternatives) { - opts = opts.concat(alternatives); + labelList = labelList.concat(alternatives); } - issueResults = issueResults.map(issue => cleanLabels(issue, opts)); + issueResults = issueResults.map(issue => cleanLabels(issue, labelList)); log.info(`ISSUES RETRIEVED: ${issueResults.length}`); return issueResults; @@ -64,6 +66,13 @@ async function retrieveIssues(data, label, alternatives) { } } +/** + * handles pagination for GitHub API call + * + * @param {object} method + * @param {string} repo + * @param {string} owner + */ async function paginate(method, repo, owner) { let response = await method({ owner: owner, @@ -84,12 +93,12 @@ async function paginate(method, repo, owner) { /** * determines whether label is present on issue * - * @param {array} issues - * @param {string} label + * @param {object} issue + * @param {array} labelList */ -function cleanLabels(issue, opts) { +function cleanLabels(issue, labelList) { let info; - if (issue.labels.some(r => opts.includes(r))) { + if (issue.labels.some(label => labelList.includes(label))) { info = {text: issue.text, label: 1}; } else { info = {text: issue.text, label: 0}; @@ -106,6 +115,7 @@ function cleanLabels(issue, opts) { function getIssueInfo(issue) { try { const raw = issue.title + ' ' + issue.body; + // remove punctuation that will interfere with csv const text = raw.replace(/[^\w\s]/gi, ''); const labels = issue.labels.map(labelObject => labelObject.name); @@ -150,13 +160,13 @@ async function createDataset( const client = new automl.v1beta1.AutoMlClient(); const projectLocation = client.locationPath(PROJECT_ID, COMPUTE_REGION); - // Classification type is assigned based on multiClass value. + // Classification type is assigned based on multiClass value let classificationType = `MULTICLASS`; if (multiLabel) { classificationType = `MULTILABEL`; } - // Set dataset name and metadata. + // Set dataset name and metadata const myDataset = { displayName: datasetName, textClassificationDatasetMetadata: { @@ -191,7 +201,7 @@ async function createDataset( } /** - * Import data into Google AutoML NL dataset + * Import data into Google AutoML Natural Language dataset * @param {string} PROJECT_ID * @param {string} COMPUTE_REGION * @param {string} datasetId @@ -227,6 +237,11 @@ async function importData(PROJECT_ID, COMPUTE_REGION, datasetId, path) { } } +/** + * List AutoML Natural Language datasets for current GCP project + * @param {string} PROJECT_ID + * @param {string} COMPUTE_REGION + */ async function listDatasets(PROJECT_ID, COMPUTE_REGION) { const client = new automl.v1beta1.AutoMlClient(); const projectLocation = client.locationPath(PROJECT_ID, COMPUTE_REGION); @@ -252,6 +267,13 @@ async function listDatasets(PROJECT_ID, COMPUTE_REGION) { } } +/** + * Create Google AutoML Natural Language model + * @param {string} PROJECT_ID + * @param {string} COMPUTE_REGION + * @param {string} datasetId + * @param {string} modelName + */ async function createModel(PROJECT_ID, COMPUTE_REGION, datasetId, modelName) { const client = new automl.v1beta1.AutoMlClient(); From 67cde4897cc59242b619a7ae2e927c05521bfa35 Mon Sep 17 00:00:00 2001 From: Steffany Brown Date: Wed, 5 Dec 2018 14:42:45 -0800 Subject: [PATCH 23/23] added getMocks function to tests --- functions/index.js | 12 +- functions/settings.tmpl.json | 2 +- src/util.js | 40 ++-- test/util_test.js | 346 +++++++++++++++-------------------- 4 files changed, 170 insertions(+), 230 deletions(-) diff --git a/functions/index.js b/functions/index.js index ad70b40b..cd01dda7 100644 --- a/functions/index.js +++ b/functions/index.js @@ -91,20 +91,16 @@ async function triage(req) { const pubSubMessage = req.data; let issueData = Buffer.from(pubSubMessage.data, 'base64').toString(); issueData = JSON.parse(issueData); - const owner = issueData.owner; - const repo = issueData.repo; - const number = issueData.number; issueData.labeled = false; let results = await predict(issueData.text); if (results) { - const labels = ['bug']; const response = await octokit.issues.addLabels({ - owner: owner, - repo: repo, - number: number, - labels: labels, + owner: issueData.owner, + repo: issueData.repo, + number: issueData.number, + labels: ['bug'], }); if (response.status === 200) { diff --git a/functions/settings.tmpl.json b/functions/settings.tmpl.json index ddfa2ebe..fb04feeb 100644 --- a/functions/settings.tmpl.json +++ b/functions/settings.tmpl.json @@ -1,6 +1,6 @@ { "secretToken": "YOUR GITHUB CLIENT ID HERE", - "projectID": "YOUR GCP PROJECT ID HERE", + "projectId": "YOUR GCP PROJECT ID HERE", "computeRegion": "YOUR GCP PROJECT COMPUTE REGION HERE", "modelId": "YOUR AUTOML NL MODEL ID HERE", "topicName": "YOUR PUB/SUB TOPIC NAME HERE" diff --git a/src/util.js b/src/util.js index 8496dee3..39d79141 100755 --- a/src/util.js +++ b/src/util.js @@ -114,7 +114,7 @@ function cleanLabels(issue, labelList) { */ function getIssueInfo(issue) { try { - const raw = issue.title + ' ' + issue.body; + const raw = `${issue.title} ${issue.body}`; // remove punctuation that will interfere with csv const text = raw.replace(/[^\w\s]/gi, ''); @@ -145,20 +145,20 @@ function makeCSV(issues, file) { /** * Create a Google AutoML Natural Language dataset - * @param {string} PROJECT_ID - * @param {string} COMPUTE_REGION + * @param {string} projectId + * @param {string} computeRegion * @param {string} datasetName * @param {string} multiLabel */ async function createDataset( - PROJECT_ID, - COMPUTE_REGION, + projectId, + computeRegion, datasetName, multiLabel ) { const automl = require(`@google-cloud/automl`); const client = new automl.v1beta1.AutoMlClient(); - const projectLocation = client.locationPath(PROJECT_ID, COMPUTE_REGION); + const projectLocation = client.locationPath(projectId, computeRegion); // Classification type is assigned based on multiClass value let classificationType = `MULTICLASS`; @@ -202,20 +202,16 @@ async function createDataset( /** * Import data into Google AutoML Natural Language dataset - * @param {string} PROJECT_ID - * @param {string} COMPUTE_REGION + * @param {string} projectId + * @param {string} computeRegion * @param {string} datasetId * @param {string} path */ -async function importData(PROJECT_ID, COMPUTE_REGION, datasetId, path) { +async function importData(projectId, computeRegion, datasetId, path) { const client = new automl.v1beta1.AutoMlClient(); // Get the full path of the dataset. - const datasetFullId = client.datasetPath( - PROJECT_ID, - COMPUTE_REGION, - datasetId - ); + const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId); // Get the Google Cloud Storage URIs. const inputUris = path.split(`,`); @@ -239,12 +235,12 @@ async function importData(PROJECT_ID, COMPUTE_REGION, datasetId, path) { /** * List AutoML Natural Language datasets for current GCP project - * @param {string} PROJECT_ID - * @param {string} COMPUTE_REGION + * @param {string} projectId + * @param {string} computeRegion */ -async function listDatasets(PROJECT_ID, COMPUTE_REGION) { +async function listDatasets(projectId, computeRegion) { const client = new automl.v1beta1.AutoMlClient(); - const projectLocation = client.locationPath(PROJECT_ID, COMPUTE_REGION); + const projectLocation = client.locationPath(projectId, computeRegion); try { const responses = await client.listDatasets({parent: projectLocation}); @@ -269,16 +265,16 @@ async function listDatasets(PROJECT_ID, COMPUTE_REGION) { /** * Create Google AutoML Natural Language model - * @param {string} PROJECT_ID - * @param {string} COMPUTE_REGION + * @param {string} projectId + * @param {string} computeRegion * @param {string} datasetId * @param {string} modelName */ -async function createModel(PROJECT_ID, COMPUTE_REGION, datasetId, modelName) { +async function createModel(projectId, computeRegion, datasetId, modelName) { const client = new automl.v1beta1.AutoMlClient(); // A resource that represents Google Cloud Platform location. - const projectLocation = client.locationPath(PROJECT_ID, COMPUTE_REGION); + const projectLocation = client.locationPath(projectId, computeRegion); // Set model name and model metadata for the dataset. const myModel = { diff --git a/test/util_test.js b/test/util_test.js index ee307cc4..3f1b78c9 100644 --- a/test/util_test.js +++ b/test/util_test.js @@ -7,6 +7,60 @@ const proxyquire = require('proxyquire').noCallThru(); const sinon = require('sinon'); const assert = require('assert'); +function getMocks() { + const path = sinon.stub(); + const imports = sinon.stub().returns(); + const location = sinon.spy(); + const create = sinon.stub(); + const list = sinon.stub(); + const getNext = sinon.stub(); + const hasNext = sinon.stub(); + const getRepoMock = sinon.stub(); + + const settingsMock = { + secretToken: 'foo', + }; + + const octoMock = { + authenticate: sinon.stub(), + issues: {getForRepo: getRepoMock}, + hasNextPage: hasNext, + getNextPage: getNext, + }; + + const clientMock = sinon.stub().returns({ + datasetPath: path, + importData: imports, + locationPath: location, + createDataset: create, + listDatasets: list, + createModel: create, + }); + + const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; + + return { + util: proxyquire('../src/util.js', { + '@google-cloud/automl': autoMlMock, + '../functions/settings.json': settingsMock, + '@octokit/rest': () => octoMock, + }), + mocks: { + list: list, + path: path, + create: create, + imports: imports, + getNext: getNext, + hasNext: hasNext, + location: location, + clientMock: clientMock, + autoMlMock: autoMlMock, + getRepoMock: getRepoMock, + settingsMock: settingsMock, + }, + }; +} + describe('makeCSV()', function() { const settingsMock = { secretToken: 'foo', @@ -46,7 +100,7 @@ describe('makeCSV()', function() { }); describe('retrieveIssues', () => { - let util, octoMock; + let mockData; beforeEach(() => { const issueData = { @@ -69,34 +123,21 @@ describe('retrieveIssues', () => { ], }; - const getNext = sinon.stub().returns(issueData); - const hasNext = sinon.stub(); - hasNext.returns(true); - hasNext.onCall(1).returns(false); - - const getMock = sinon.stub().returns(Promise.resolve(issueData)); - - octoMock = { - authenticate: sinon.stub(), - issues: {getForRepo: getMock}, - hasNextPage: hasNext, - getNextPage: getNext, - }; - - const settingsMock = { - secretToken: 'foo', - }; - - util = proxyquire('../src/util.js', { - '@octokit/rest': () => octoMock, - '../functions/settings.json': settingsMock, - }); + mockData = getMocks(); + mockData.mocks.getNext.returns(issueData); + mockData.mocks.hasNext.returns(true); + mockData.mocks.hasNext.onCall(1).returns(false); + mockData.mocks.getRepoMock.returns(Promise.resolve(issueData)); }); it('should pass new issue object to makeCSV', async () => { const label = 'type: bug'; const alt = ['bug']; - const result = await util.retrieveIssues('test/test_repos.txt', label, alt); + const result = await mockData.util.retrieveIssues( + 'test/test_repos.txt', + label, + alt + ); assert(result.length === 6); assert(result[0].text === 'issue details'); @@ -115,28 +156,27 @@ describe('retrieveIssues', () => { }, }); - octoMock.issues.getForRepo.returns(expectedResponse); + mockData.mocks.getRepoMock.returns(expectedResponse); + mockData.mocks.hasNextPage = sinon.spy(); + mockData.mocks.getNextPage = sinon.spy(); - const result = await util.retrieveIssues('test/test_repos.txt', label); + const result = await mockData.util.retrieveIssues( + 'test/test_repos.txt', + label + ); assert(result === undefined); - sinon.assert.calledOnce(octoMock.issues.getForRepo); - sinon.assert.notCalled(octoMock.hasNextPage); - sinon.assert.notCalled(octoMock.getNextPage); + sinon.assert.calledOnce(mockData.mocks.getRepoMock); + sinon.assert.notCalled(mockData.mocks.hasNextPage); + sinon.assert.notCalled(mockData.mocks.getNextPage); }); }); describe('getIssueInfo()', function() { - let originalIssue, returnedIssue, labelCount, util; + let originalIssue, returnedIssue, labelCount, mockData; beforeEach(() => { - const settingsMock = { - secretToken: 'foo', - }; - - util = proxyquire('../src/util.js', { - '../functions/settings.json': settingsMock, - }); + mockData = getMocks(); originalIssue = { id: 1, @@ -156,7 +196,8 @@ describe('getIssueInfo()', function() { }); it('should return issue object with text & labels keys', async function() { - const result = await util.getIssueInfo(originalIssue); + const result = await mockData.util.getIssueInfo(originalIssue); + assert.strictEqual( Object.keys(result).length, Object.keys(returnedIssue).length @@ -172,20 +213,24 @@ describe('getIssueInfo()', function() { body: 'issue body', }; - let result = await util.getIssueInfo(badIssue, labelCount); + const result = await mockData.util.getIssueInfo(badIssue, labelCount); assert(result === undefined); }); }); describe('createDataset()', function() { + let mockData; const projectId = 'test-project'; const computeRegion = 'us-central1'; const datasetName = 'testSet'; const multiLabel = 'false'; + beforeEach(() => { + mockData = getMocks(); + }); + it('should create a Google AutoML Natural Language dataset', function() { - const location = sinon.spy(); - const create = sinon.stub().returns([ + mockData.mocks.create.returns([ { name: 'dataset/location/378646', displayName: 'testSet', @@ -195,29 +240,19 @@ describe('createDataset()', function() { }, ]); - const clientMock = sinon.stub().returns({ - locationPath: location, - createDataset: create, - }); - - const settingsMock = { - secretToken: 'foo', - }; - - const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - const util = proxyquire('../src/util.js', { - '@google-cloud/automl': autoMlMock, - '../functions/settings.json': settingsMock, - }); + mockData.util.createDataset( + projectId, + computeRegion, + datasetName, + multiLabel + ); - util.createDataset(projectId, computeRegion, datasetName, multiLabel); - sinon.assert.calledOnce(location); - assert(location.calledWith(projectId, computeRegion)); + sinon.assert.calledOnce(mockData.mocks.location); + assert(mockData.mocks.location.calledWith(projectId, computeRegion)); }); it('should throw an error', function() { - const location = sinon.spy(); - const create = sinon.stub().returns([ + mockData.mocks.create.returns([ { err: 'error', name: 'dataset/location/378646', @@ -228,88 +263,52 @@ describe('createDataset()', function() { }, ]); - const clientMock = sinon.stub().returns({ - locationPath: location, - createDataset: create, - }); - - const settingsMock = { - secretToken: 'foo', - }; - - const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - const util = proxyquire('../src/util.js', { - '@google-cloud/automl': autoMlMock, - '../functions/settings.json': settingsMock, - }); - - util.createDataset(projectId, computeRegion, datasetName, multiLabel); + mockData.util.createDataset( + projectId, + computeRegion, + datasetName, + multiLabel + ); }); }); describe('importData()', function() { + let mockData; + const projectId = 'test-project'; const computeRegion = 'us-central1'; const datasetId = '123TEST4567'; const file = 'gs://testbucket-lcm/testIssues.csv'; - it('should import data into AutoML NL dataset', function() { - const path = sinon.spy(); - const imports = sinon.stub().returns(); - - const clientMock = sinon.stub().returns({ - datasetPath: path, - importData: imports, - }); - - const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - - const settingsMock = { - secretToken: 'foo', - }; + beforeEach(() => { + mockData = getMocks(); + }); - const util = proxyquire('../src/util.js', { - '@google-cloud/automl': autoMlMock, - '../functions/settings.json': settingsMock, - }); + it('should import data into AutoML NL dataset', function() { + mockData.util.importData(projectId, computeRegion, datasetId, file); - util.importData(projectId, computeRegion, datasetId, file); - sinon.assert.calledOnce(path); - assert(path.calledWith(projectId, computeRegion, datasetId)); - sinon.assert.calledOnce(imports); + sinon.assert.calledOnce(mockData.mocks.path); + assert(mockData.mocks.path.calledWith(projectId, computeRegion, datasetId)); + sinon.assert.calledOnce(mockData.mocks.imports); }); it('should throw an error', function() { - const path = sinon.spy(); - const imports = sinon.stub().throws(); - - const clientMock = sinon.stub().returns({ - datasetPath: path, - importData: imports, - }); - - const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - - const settingsMock = { - secretToken: 'foo', - }; - - const util = proxyquire('../src/util.js', { - '@google-cloud/automl': autoMlMock, - '../functions/settings.json': settingsMock, - }); - - util.importData(projectId, computeRegion, datasetId, file); + mockData.mocks.imports.throws(); + mockData.util.importData(projectId, computeRegion, datasetId, file); }); }); describe('listDatasets()', function() { + let mockData; const projectId = 'test-project'; const computeRegion = 'us-central1'; + beforeEach(() => { + mockData = getMocks(); + }); + it('should return a list of datasets', async function() { - const location = sinon.spy(); - const list = sinon.stub().returns([ + mockData.mocks.list.returns([ [ { name: 'projects/12345/locations/us-central1/datasets/12345', @@ -322,32 +321,15 @@ describe('listDatasets()', function() { ], ]); - const clientMock = sinon.stub().returns({ - locationPath: location, - listDatasets: list, - }); - - const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - - const settingsMock = { - secretToken: 'foo', - }; - - const util = proxyquire('../src/util.js', { - '@google-cloud/automl': autoMlMock, - '../functions/settings.json': settingsMock, - }); + await mockData.util.listDatasets(projectId, computeRegion); - await util.listDatasets(projectId, computeRegion); - - sinon.assert.calledOnce(list); - sinon.assert.calledOnce(location); - assert(location.calledWith(projectId, computeRegion)); + sinon.assert.calledOnce(mockData.mocks.list); + sinon.assert.calledOnce(mockData.mocks.location); + assert(mockData.mocks.location.calledWith(projectId, computeRegion)); }); it('should throw an error', function() { - const location = sinon.spy(); - const list = sinon.stub().returns([ + mockData.mocks.list.returns([ [ { name: 'projects/12345/locations/us-central1/datasets/12345', @@ -356,33 +338,21 @@ describe('listDatasets()', function() { ], ]); - const clientMock = sinon.stub().returns({ - locationPath: location, - listDatasets: list, - }); - - const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - - const settingsMock = { - secretToken: 'foo', - }; - - const util = proxyquire('../src/util.js', { - '@google-cloud/automl': autoMlMock, - '../functions/settings.json': settingsMock, - }); - - util.listDatasets(projectId, computeRegion); + mockData.util.listDatasets(projectId, computeRegion); }); }); describe('createModel()', function() { + let mockData; const projectId = 'test-project'; const computeRegion = 'us-central1'; + beforeEach(() => { + mockData = getMocks(); + }); + it('should call AutoML NL API to train model', async function() { - const location = sinon.spy(); - const create = sinon.stub().returns([ + mockData.mocks.create.returns([ {Operation: 'data'}, { name: @@ -396,48 +366,26 @@ describe('createModel()', function() { }, ]); - const clientMock = sinon.stub().returns({ - locationPath: location, - createModel: create, - }); - - const settingsMock = { - secretToken: 'foo', - }; - - const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; - const util = proxyquire('../src/util.js', { - '@google-cloud/automl': autoMlMock, - '../functions/settings.json': settingsMock, - }); - - await util.createModel(projectId, computeRegion, '123456ABC', 'testModel'); + await mockData.util.createModel( + projectId, + computeRegion, + '123456ABC', + 'testModel' + ); - sinon.assert.calledOnce(create); - sinon.assert.calledOnce(location); - assert(location.calledWith(projectId, computeRegion)); + sinon.assert.calledOnce(mockData.mocks.create); + sinon.assert.calledOnce(mockData.mocks.location); + assert(mockData.mocks.location.calledWith(projectId, computeRegion)); }); it('should throw an error', async function() { - const location = sinon.spy(); - const create = sinon.stub().returns([]); - - const clientMock = sinon.stub().returns({ - locationPath: location, - createModel: create, - }); - - const autoMlMock = {v1beta1: {AutoMlClient: clientMock}}; + mockData.mocks.create.returns([]); - const settingsMock = { - secretToken: 'foo', - }; - - const util = proxyquire('../src/util.js', { - '@google-cloud/automl': autoMlMock, - '../functions/settings.json': settingsMock, - }); - - await util.createModel(projectId, computeRegion, '123456ABC', 'testModel'); + await mockData.util.createModel( + projectId, + computeRegion, + '123456ABC', + 'testModel' + ); }); });