deduplicated get and post functions

Author: max-ostapenko

lib/helper.js

@@ -9,7 +9,8 @@ var xml2js = require('xml2js'),
   url = require('url'),
   os = require('os'),
   csv = require('csv'),
-  entities = require('entities');
+  entities = require('entities'),
+  qs = require('querystring');
 
 var parser = new xml2js.Parser({ explicitArray: false, mergeAttrs: true });
 
@@ -155,10 +156,10 @@ function scriptToString(data) {
 }
 
 // Build the RESTful API url call only
-function dryRun(config, path, params) {
+function dryRun(config, path, form) {
   path = url.parse(path, true);
 
-  if (params && params.custom) {
+  if (config.method == "POST") {
     return {
       url: url.format({
         protocol: config.protocol,
@@ -167,9 +168,8 @@ function dryRun(config, path, params) {
           config.port : undefined),
         pathname: path.pathname,
       }),
-      form: params
+      form: qs.stringify(form)
     };
-
   } else {
     return {
       url: url.format({
@@ -178,7 +178,7 @@ function dryRun(config, path, params) {
         port: (config.port !== 80 && config.port !== 443 ?
           config.port : undefined),
         pathname: path.pathname,
-        query: params
+        query: path.query
       })
     };
   }

lib/webpagetest.js

@@ -58,15 +58,10 @@ var filenames = {
   cached: "_Cached",
 };
 
-// GET helper function
-function get(config, pathname, query, proxy, agent, callback, encoding) {
+// GET/POST helper function
+function get(config, pathname, data, proxy, agent, callback, encoding) {
   var protocol, options;
 
-  pathname = url.format({
-    pathname: pathname,
-    query: query,
-  });
-
   if (proxy) {
     var proxyUrl = url.parse(proxy);
     var pathForProxy = config.protocol + "//";
@@ -85,6 +80,8 @@ function get(config, pathname, query, proxy, agent, callback, encoding) {
       headers: {
         Host: config.hostname,
       },
+      method: config.method
+
     };
   } else {
     protocol = config.protocol === "https:" ? https : http;
@@ -93,117 +90,13 @@ function get(config, pathname, query, proxy, agent, callback, encoding) {
       host: config.hostname,
       auth: config.auth,
       port: config.port,
+      method: config.method,
       headers: {},
     };
   }
 
-  if (encoding !== "binary") {
-    options.headers["X-WPT-API-KEY"] = this.config.key;
-    options.headers["accept-encoding"] = "gzip,deflate";
-    options.headers["User-Agent"] = "WebpagetestNodeWrapper/v0.6.0";
-  }
-
-  if (agent) {
-    options.agent = agent;
-  }
-
-  return protocol
-    .get(options, function getResponse(res) {
-      var data,
-        length,
-        statusCode = res.statusCode;
-
-      if (statusCode !== 200) {
-        callback(
-          new helper.WPTAPIError(statusCode, http.STATUS_CODES[statusCode])
-        );
-      } else {
-        data = [];
-        length = 0;
-
-        encoding = res.headers["content-encoding"] || encoding || "uft8";
-
-        res.on("data", function onData(chunk) {
-          data.push(chunk);
-          length += chunk.length;
-        });
-
-        res.on("end", function onEnd() {
-          var i,
-            len,
-            pos,
-            buffer = new Buffer.alloc(length),
-            type = (res.headers["content-type"] || "").split(";")[0];
-
-          for (i = 0, len = data.length, pos = 0; i < len; i += 1) {
-            data[i].copy(buffer, pos);
-            pos += data[i].length;
-          }
-
-          if (encoding === "gzip" || encoding === "deflate") {
-            // compressed response (gzip,deflate)
-            zlib.unzip(buffer, function unzip(err, buffer) {
-              if (err) {
-                callback(err);
-              } else {
-                callback(undefined, buffer.toString(), {
-                  type: type,
-                  encoding: encoding,
-                });
-              }
-            });
-          } else {
-            // uncompressed response
-            callback(undefined, buffer, {
-              type: type,
-              encoding: encoding,
-            });
-          }
-        });
-      }
-    })
-    .on("error", function onError(err) {
-      callback(err);
-    });
-}
-
-// execute runTest using POST request
-function post(config, pathname, query, proxy, agent, callback, encoding) {
-  var protocol, options;
-
-  if (proxy) {
-    var proxyUrl = url.parse(proxy);
-    var pathForProxy = config.protocol + "//";
-
-    if (config.auth) {
-      pathForProxy += config.auth + "@";
-    }
-
-    pathForProxy += config.hostname + ":" + config.port + pathname;
-    protocol = proxyUrl.protocol === "https:" ? https : http;
-
-    options = {
-      host: proxyUrl.hostname,
-      port: proxyUrl.port,
-      path: pathForProxy,
-      method: "POST",
-      headers: {
-        Host: config.hostname,
-      },
-
-    };
-  } else {
-    protocol = config.protocol === "https:" ? https : http;
-    options = {
-      path: pathname,
-      host: config.hostname,
-      auth: config.auth,
-      port: config.port,
-      method: "POST",
-      headers: {
-        'Content-Type': 'application/x-www-form-urlencoded',
-      },
-    };
+  if (options.method == "POST") {
+    options.headers['Content-Type'] = 'application/x-www-form-urlencoded';
   }
 
   if (encoding !== "binary") {
@@ -216,9 +109,7 @@ function post(config, pathname, query, proxy, agent, callback, encoding) {
     options.agent = agent;
   }
 
-  postData = qs.stringify(query)
-
-  return protocol
+  var request = protocol
     .request(options, function getResponse(res) {
       var data,
         length,
@@ -276,7 +167,13 @@ function post(config, pathname, query, proxy, agent, callback, encoding) {
     .on("error", function onError(err) {
       callback(err);
     })
-    .end(postData);
+
+  if (options.method == "POST") {
+    return request.end(qs.stringify(data));
+  } else {
+    return request.end();
+  }
+
 }
 
 // execute callback properly normalizing optional args
@@ -306,14 +203,27 @@ function api(pathname, callback, query, options) {
 
   pathname = url.resolve(config.pathname, pathname);
 
+  config.method = url.format({
+    pathname: pathname,
+    query: query,
+  }).toString().length > 6 * 1024 ? "POST" : "GET";
+
+  if (config.method == "GET") {
+    pathname = url.format({
+      pathname: pathname,
+      query: query,
+    });
+    query = undefined;
+  }
+
   if (options.dryRun) {
     // dry run: return the API url (string) only
     if (typeof callback === "function") {
       callback.apply(callback, [undefined, helper.dryRun(config, pathname, query)]);
     }
   } else {
     // make the real API call
-    (options.custom !== undefined ? post : get).call(
+    get.call(
       this,
       config,
       pathname,

test/edge-cases-test.js

@@ -88,11 +88,12 @@ describe('Edge Cases of', function() {
     it('gets a test with custom metrics then returns API url and payload with custom metrics data present', function (done) {
       wpt.runTest('http://foobar.com', {
         dryRun: true,
-        custom: '[example]\nreturn 1;'
+        mobile: 1,
+        custom: '[example]\n\\\\' + 'X'.repeat(6 * 1024) + '\nreturn 1;'
       }, function (err, data) {
         if (err) return done(err);
         assert.equal(data.url, wptServer + 'runtest.php');
-        assert.equal(data.form.custom, '[example]\nreturn 1;');
+        assert.equal(data.form.length, 6233);
         done();
       });
     });