added subscription for new blocks and sends data to a url creating a webhook; added websocket to create subscriptions and for future use for custom websocket server

Author: ar-to

README.md

@@ -21,10 +21,13 @@
 - [Error Debugging](#error-debugging)
 - [Sample Contracts](#sample-contracts)
 - [Version Control]($version-control)
+- [Connections](#Connections)
 - [Endpoint Notes](#endpoint-notes)
   - [Get Block Info](#get-block-info)
   - [Get Transaction Info](#get-transaction-info)
   - [Send Transaction](#send-transaction)
+- [Webhook](#webhook)
+- [Websockets](#websockets)
 - [List of All API Endpoints](#endpoints)
 
 ## Quick Start
@@ -338,12 +341,54 @@ Question about storing artifact files (e.g. Token.json) after truffle compiles c
 - [stackexchange](https://ethereum.stackexchange.com/questions/19486/storing-a-truffle-contract-interface-in-version-control) - recommendation to store outside
 - [artifact-updates](https://github.com/trufflesuite/artifact-updates) - community repo for updating some issues with doing this.
 
+## Connections
+
+Full example of `connections.json`
+
+```
+{
+  "networks": {
+    "connectApi": "ropsten",
+    "ganache":{
+      "url": "http://127.0.0.1:7545",
+      "websocketUrl": "'ws://localhost:7545",
+      "networkId": "5777",
+      "networkName": "ganache",
+      "networkType": "testrpc",
+      "token": {
+        "ownerAddress": "0x451E62137891156215d9D58BeDdc6dE3f30218e7",
+        "tokenContractAddress": "0x4c59b696552863429d25917b52263532af6e6078",
+        "migrateContractAddress": "0xcf068555df7eab0a9bad97829aa1a187bbffbdba"
+      }
+    },
+    "ropsten":{
+      "url": "http://10.10.0.163:8545",
+      "websocketUrl": "ws://10.10.0.163:8546",
+      "blockWebhookUrl": "http://4f5a64ba.ngrok.io/api/eth/webhook",
+      "syncingWebhookUrl": "http://4f5a64ba.ngrok.io/api/eth/webhook",
+      "testWebhookUrl": "http://4f5a64ba.ngrok.io/api/eth/webhook",
+      "networkId": 3,
+      "networkName": "ropsten",
+      "networkType": "testnet",
+      "token": {
+        "ownerAddress": "0x83634a8eaadc34b860b4553e0daf1fac1cb43b1e",
+        "tokenContractAddress": "0x3e672122bfd3d6548ee1cc4f1fa111174e8465fb",
+        "migrateContractAddress": "0xa8ebf36b0a34acf98395bc5163103efc37621052"
+      }
+    },
+    "mainnet":{}
+  }
+}
+```
+
 ## Bugs to Fix
 
 transferOwnership & kill functions both catch an 'invalid address'. The response it 200 with `false` boolean indicating request failed. 
 
 `api/eth/tx-from-block/hashStringOrNumber` endpoint seems to fail when connected to Ropsten Testnet but not with ganache local node
 
+`` subscription does not seem to work and or do
+
 
 ## Endpoint Notes
 
@@ -506,6 +551,85 @@ See [web3 api](http://web3js.readthedocs.io/en/1.0/web3-eth-accounts.html#signtr
 }
 ```
 
+## Webhook
+
+Webhooks allow data to be sent as POST request to an external url specified in the `config/connections.json` iside the `blockWebhookUrl` paramater. 
+
+**To use this webhook you need to have websockets connected see [websockets](#websockets) for more information**
+
+```
+"ropsten":{
+  ...
+  "blockWebhookUrl": "http://4f5a64ba.ngrok.io/api/eth/webhook",
+  "testWebhookUrl": "http://4f5a64ba.ngrok.io/api/eth/webhook",
+  ...
+},
+```
+
+The `testWebhookUrl` parameter is used for testing but following these instructions:
+
+Run this api server and then run these
+
+```
+brew cask install ngrok
+ngrok http 3000
+```
+grab the ngrok url and add it the `testWebhookUrl` parameter e.g.`http://4f5a64ba.ngrok.io/api/eth/webhook`.
+
+Then open the debug url for ngrok e.g. `http://127.0.0.1:4040`
+
+Send a post request to `http://localhost:3000/api/eth/post-to-webhook` and watch two request show, one for the request and the second that send another POST request to the `/api/eth/webhook` endpoint
+
+## Websockets
+
+A websocket connection to the node is used to subscribe to events (e.g. new blocks, transactions, see [web3 api](http://web3js.readthedocs.io/en/1.0/web3-eth-subscribe.html)) and required to use the webhook feature which sends these subscriptions to a specified url.
+
+**Setup**
+
+You need to enable a websocket connection inside the node and add that url to the `connections.json`
+
+```
+```
+"ropsten":{
+  ...
+  "websocketUrl": "ws://10.10.0.163:8546",
+  ...
+},
+```
+
+or `.env` files which will overwrite the connections.json parameter
+
+```
+WEBSOCKET_URL=ws://127.0.0.1:8546
+```
+
+Websockets are an experimental feature, and the code can be found inside `server/app/helpers/web3-websocket.js`
+There is one `subscriptionType` that is corrently tested and working:
+
+```
+  blockSubscription
+```
+
+Another is to get status on the syncing of the node with the most current block. **Needs more testing so use with caution**
+ 
+```
+  syncingSubscription
+```
+
+**Create subscription**
+To create a subscription to block headers so every new block will be emitted and POSTed to a webhook url.
+
+```
+http://localhost:3000/api/eth/subscribe-block
+```
+
+**Remove Subscription**
+
+```
+http://localhost:3000/api/eth/close-subscriptions/blockSubscription
+http://localhost:3000/api/eth/close-subscriptions/syncingSubscription
+```
+
 ## Endpoints
 
 ```

package-lock.json

@@ -25,7 +25,8 @@
     "truffle-contract": "^3.0.5",
     "truffle-hdwallet-provider": "0.0.5",
     "truffle-wallet-provider": "0.0.5",
-    "web3": "^1.0.0-beta.34"
+    "web3": "^1.0.0-beta.34",
+    "ws": "^5.2.0"
   },
   "devDependencies": {
     "nodemon": "^1.17.4"

package.json

@@ -2,10 +2,37 @@ const ethereum = require('../helpers/ethereum.js');
 const Web3 = require('../helpers/web3.js');
 const web3 = Web3.web3;
 
+const Web3Websocket = require('../helpers/web3-websocket.js');
+const web3Socket = Web3Websocket.web3Socket;
+const WebSocket = Web3Websocket.WebSocket;
+const axios = require('axios')
+
+const Webhook = require('../helpers/webhook.js');
+const testWebhookUrl = Webhook.testWebhookUrl;
+
 module.exports = {
   getTest: function (req, res, next) {
     res.send("Ethereum API")
   },
+  testWebhookPost: function (req, res, next) {
+    let obj = {};
+    axios.post(testWebhookUrl, req.body)
+      .then(function (response) {
+        obj.message = 'sent to webhook url';
+        obj.data = response.data;
+        res.send(obj)
+      })
+      .catch(function (error) {
+        obj.error = 'There was an error sending post request to webhook url';
+        obj.config = error.config;
+        res.status(404).send(obj).end()
+      });
+  },
+  testWebhook: function (req, res, next) {
+    console.log('posted into webhook: ', req.body)
+    // res.send('req.body')
+    res.send(req.body)
+  },
   isSyncing: function (req, res, next) {
     ethereum.isSyncing().then((value) => {
       return (value.error != null ? res.status(404).send(value).end() : res.send(value));
@@ -98,6 +125,42 @@ module.exports = {
         });
       })
   },
+  subscribeSyncing: function (req, res, next) {
+    let obj = {};
+    ethereum.subscribeSyncing()
+      .then((value) => {
+        // console.log('value block', value)
+        obj.subscriptionOpened = value;
+        res.send(obj)
+      }, (rej) => {
+        obj.subscriptionNotOpened = rej;
+        res.status(404).send(obj).end();
+      })
+  },
+  subscribeBlock: function (req, res, next) {
+    let obj = {};
+    ethereum.subscribeBlock()
+      .then((value) => {
+        // console.log('value block', value)
+        obj.subscriptionOpened = value;
+        res.send(obj)
+      }, (rej) => {
+        obj.subscriptionNotOpened = rej;
+        res.status(404).send(obj).end();
+      })
+  },
+  closeSubscriptions: function (req, res, next) {
+    let obj = {};
+    obj.subscriptionType = req.params.subscriptionType;
+
+    ethereum.closeSubscription(obj.subscriptionType).then((value) => {
+      obj.closed = value;
+      res.send(obj);
+    }, (rej) => {
+      obj.notClosed = rej;
+      res.send(obj);
+    })
+  }
 }
 
 async function validateBody(res, body) {

server/app/controllers/api-ethereum.js

@@ -2,6 +2,12 @@ const Web3 = require('../helpers/web3.js');
 const web3 = Web3.web3;
 const BigNumber = Web3.BigNumber;
 const axios = require('axios');
+const Web3Websocket = require('../helpers/web3-websocket.js');
+const web3Socket = Web3Websocket.web3Socket;
+
+const Webhook = require('../helpers/webhook.js');
+const blockWebhookUrl = Webhook.blockWebhookUrl;
+const syncingWebhookUrl = Webhook.syncingWebhookUrl;
 
 const gasAPI = "https://ethgasstation.info/json/ethgasAPI.json";
 var e; //await function
@@ -262,6 +268,84 @@ Ethereum = {
       });
     return e;
   },
+  /**
+   * Close a subscription by type
+   * @param {String} subscriptionType is the Ethereum.parameter used to store the instance of the subscription e.g. syncingSubscription
+   * @return {Promise}
+   */
+  closeSubscription: function (subscriptionType) {
+    return new Promise((resolve, reject) => {
+      if (Ethereum[subscriptionType] != null) {
+        return Ethereum[subscriptionType].unsubscribe(function (error, success) {
+          if (success) {
+            console.log(`Successfully unsubscribed! for ${subscriptionType}`, success);
+            resolve(`Successfully unsubscribed! for ${subscriptionType}`)
+          } else {
+            console.log('unsubscribed error:', error);
+            reject(`Could not unsubscribed. There was an error unsubscribing for ${subscriptionType}`)
+          }
+        })
+      } else {
+        reject(`no open subscriptions for ${subscriptionType}`)
+      }
+    })
+  },
+  /**
+   * Parameters used to store subscription instances
+   * which can then be used elsewhere or to unsubscribe
+   */
+  syncingSubscription: null,
+  blockSubscription: null,
+  /**
+   * Start a subscription to syncing and send data to webhook url
+   * @return {Promise}
+   */
+  subscribeSyncing: function () {
+    let obj = {};
+    return new Promise((resolve, reject) => {
+      return Ethereum.syncingSubscription = web3Socket.eth.subscribe('syncing', function (error, sync) {
+        if (error) reject(`There was an error creating a new webhook subscription to syncing: ${error}`)
+        if (!error) resolve(`Successfully created a webhook subscription to syncing`)
+      })
+        .on('data', function (sync) {
+          console.log('on syncing subscription: ', sync);
+          /**
+           * Send post request to url to initiate webhook
+           */
+          axios.post(syncingWebhookUrl, sync)
+            .catch(function (error) {
+              obj.error = 'There was an error sending post request to webhook url';
+              obj.config = error.config;
+              reject(obj);
+            });
+        });
+    })
+  },
+  /**
+   * Start a subscription to new blocks and send data to webhook url
+   * @return {Promise}
+   */
+  subscribeBlock: function () {
+    let obj = {};
+    return new Promise((resolve, reject) => {
+      return Ethereum.blockSubscription = web3Socket.eth.subscribe('newBlockHeaders', function (error, sync) {
+        if (error) reject(`There was an error creating a new webhook subscription to newBlockHeaders: ${error}`)
+        if (!error) resolve(`Successfully created a webhook subscription to newBlockHeaders`)
+      })
+        .on('data', function (sync) {
+          // console.log('on block subscription: ', sync);
+          /**
+           * Send post request to url to initiate webhook
+           */
+          axios.post(blockWebhookUrl, sync)
+            .catch(function (error) {
+              obj.error = 'There was an error sending post request to webhook url';
+              obj.config = error.config;
+              reject(obj);
+            });
+        })
+    });
+  },
 }
 
 module.exports = Ethereum;