🚀 v0.8: submit POST params in body. Encode special chars. Streaming test. Fix #36.

Author: dandv

package.json

@@ -1,6 +1,6 @@
 {
   "name": "twitter-lite",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "A tiny, full-featured client / server library for the Twitter API",
   "source": [
     "twitter.js",

stream.test.js

@@ -1,5 +1,129 @@
+require("dotenv").config();
 const Stream = require("./stream");
+const Twitter = require("./twitter");
+
+const {
+  TWITTER_CONSUMER_KEY,
+  TWITTER_CONSUMER_SECRET,
+  ACCESS_TOKEN,
+  ACCESS_TOKEN_SECRET
+} = process.env;
+
+function newClient(subdomain = "api") {
+  return new Twitter({
+    subdomain,
+    consumer_key: TWITTER_CONSUMER_KEY,
+    consumer_secret: TWITTER_CONSUMER_SECRET,
+    access_token_key: ACCESS_TOKEN,
+    access_token_secret: ACCESS_TOKEN_SECRET
+  });
+}
 
 it("should default export to be a function", () => {
   expect(new Stream()).toBeInstanceOf(Stream);
 });
+
+describe("functionality", () => {
+  let client;
+  beforeAll(() => (client = newClient()));
+
+  it("should filter realtime tweets from up to 5000 users", done => {
+    // https://developer.twitter.com/en/docs/tweets/filter-realtime/api-reference/post-statuses-filter
+    const stream = client.stream("statuses/filter", {
+      follow: [
+        // First pass a ton of accounts that don't tweet often (@dandv), to stress-test the POST body
+        ...Array(4900).fill("15008676"),
+        // Then add prolific users from https://socialblade.com/twitter/top/100/tweets that are
+        // still active. Get with $('div.table-cell a').map(function () { return this.href })
+        // then use users/lookup to convert to IDs.
+        "63299591",
+        "115639376",
+        "4823945834",
+        "2669983818",
+        "6529402",
+        "362413805",
+        "450395397",
+        "15007299",
+        "132355708",
+        "561669474",
+        "2213312341",
+        "2050001283",
+        "89142182",
+        "2316574981",
+        "133684052",
+        "255409050",
+        "15518000",
+        "124172948",
+        "225647847",
+        "3012764258",
+        "382430644",
+        "42832810",
+        "2233720891",
+        "290395312",
+        "50706690",
+        "1388673048",
+        "414306138",
+        "155409802",
+        "21976463",
+        "1179710990",
+        "130426181",
+        "171299971",
+        "32453798",
+        "22279680",
+        "22274998",
+        "59804598",
+        "3048544857",
+        "17872077",
+        "85741735",
+        "3032932864",
+        "120421476",
+        "473656787",
+        "876302191",
+        "717628618906570752",
+        "15518784",
+        "152641509",
+        "5950272",
+        "416383737",
+        "2569759392",
+        "165796189",
+        "1680484418",
+        "108192135",
+        "3007312628",
+        "32771325",
+        "764410142679035904",
+        "19272300",
+        "829411574",
+        "68956490",
+        "2836271637",
+        "392599269",
+        "1145130336",
+        "52236744",
+        "243133079",
+        "104120518",
+        "51684249",
+        "18057450",
+        "1027850761",
+        "1868107663",
+        "213165296",
+        "15503908",
+        "1346933186",
+        "2857426909",
+        "2814731582",
+        "453780255",
+        "3027662932",
+        "23719043",
+        "486288760",
+        "121190725",
+        "2942062137",
+        "19286574",
+        "21033096",
+        "271986064"
+      ]
+    });
+
+    stream.on("data", tweet => {
+      // Within seconds, one of those prolific accounts will tweet something
+      done();
+    });
+  });
+});

twitter.js

@@ -31,11 +31,31 @@ const defaults = {
   bearer_token: null
 };
 
+// Twitter expects POST body parameters to be URL-encoded: https://developer.twitter.com/en/docs/basics/authentication/guides/creating-a-signature
+// However, some endpoints expect a JSON payload - https://developer.twitter.com/en/docs/direct-messages/sending-and-receiving/api-reference/new-event
+// It appears that JSON payloads don't need to be included in the signature,
+// because sending DMs works without signing the POST body
+const JSON_ENDPOINTS = [
+  "direct_messages/events/new",
+  "direct_messages/welcome_messages/new",
+  "direct_messages/welcome_messages/rules/new"
+];
+
 const baseHeaders = {
   "Content-Type": "application/json",
   Accept: "application/json"
 };
 
+function percentEncode(string) {
+  // From OAuth.prototype.percentEncode
+  return string
+    .replace(/!/g, "%21")
+    .replace(/\*/g, "%2A")
+    .replace(/'/g, "%27")
+    .replace(/\(/g, "%28")
+    .replace(/\)/g, "%29");
+}
+
 class Twitter {
   constructor(options) {
     const config = Object.assign({}, defaults, options);
@@ -64,7 +84,7 @@ class Twitter {
   static _handleResponse(response) {
     const headers = response.headers.raw(); // https://github.com/bitinn/node-fetch/issues/495
     return response.json().then(res => {
-      res._headers = headers;
+      res._headers = headers; // TODO: this creates an array-like object when it adds _headers to an array response
       return res;
     });
   }
@@ -151,7 +171,9 @@ class Twitter {
       url: `${this.url}/${resource}.json`,
       method
     };
-    if (parameters) requestData.url += "?" + querystring.stringify(parameters);
+    if (parameters)
+      if (method === "POST") requestData.data = parameters;
+      else requestData.url += "?" + querystring.stringify(parameters);
 
     let headers = {};
     if (this.authType === "User") {
@@ -178,8 +200,8 @@ class Twitter {
 
     return Fetch(requestData.url, { headers })
       .then(Twitter._handleResponse)
-      .then(
-        results => ("errors" in results ? Promise.reject(results) : results)
+      .then(results =>
+        "errors" in results ? Promise.reject(results) : results
       );
   }
 
@@ -190,14 +212,22 @@ class Twitter {
       parameters
     );
 
+    const postHeaders = Object.assign({}, baseHeaders, headers);
+    if (JSON_ENDPOINTS.includes(resource)) {
+      body = JSON.stringify(body);
+    } else {
+      body = querystring.stringify(parameters);
+      postHeaders["Content-Type"] = "application/x-www-form-urlencoded";
+    }
+
     return Fetch(requestData.url, {
       method: "POST",
-      headers: Object.assign({}, baseHeaders, headers),
-      body: JSON.stringify(body)
+      headers: postHeaders,
+      body: percentEncode(body)
     })
       .then(Twitter._handleResponse)
-      .then(
-        results => ("errors" in results ? Promise.reject(results) : results)
+      .then(results =>
+        "errors" in results ? Promise.reject(results) : results
       );
   }
 
@@ -207,17 +237,26 @@ class Twitter {
 
     const stream = new Stream();
 
+    // POST the request, in order to accommodate long parameter lists, e.g.
+    // up to 5000 ids for statuses/filter - https://developer.twitter.com/en/docs/tweets/filter-realtime/api-reference/post-statuses-filter
     const requestData = {
       url: `${getUrl("stream")}/${resource}.json`,
-      method: "GET"
+      method: "POST"
     };
-    if (parameters) requestData.url += "?" + querystring.stringify(parameters);
+    if (parameters) requestData.data = parameters;
 
     const headers = this.client.toHeader(
       this.client.authorize(requestData, this.token)
     );
 
-    const request = Fetch(requestData.url, { headers });
+    const request = Fetch(requestData.url, {
+      method: "POST",
+      headers: {
+        ...headers,
+        "Content-Type": "application/x-www-form-urlencoded"
+      },
+      body: percentEncode(querystring.stringify(parameters))
+    });
 
     request
       .then(response => {