Merge pull request #1 from llaumgui/proxy-patch

Add support of HTTP_PROXY and http_proxy

Author: lgraubner

lib/W3CValidator.js

@@ -20,6 +20,7 @@ var pkg = require("../package.json");
 program.version(pkg.version)
         .usage("[options] <url>")
         .option("-l, --log", "log errors in a text file")
+        .option("-m, --max <n>", "max URLs to crawl")
         .option("-q, --query", "consider query string")
         .option("-v, --verbose", "show error details")
         .parse(process.argv);
@@ -35,6 +36,7 @@ var W3CValidator = function(url, java_home) {
     this.invalid = 0;
     this.JAVA_HOME = java_home;
     this.logData = "";
+    this.httpProxy = false;
 
     this.uri = new URL(url);
     this.crawler = new Crawler(this.uri.host);
@@ -54,6 +56,22 @@ var W3CValidator = function(url, java_home) {
         this.uri.set("protocol", "http:");
     }
 
+    // Get proxy
+    if (["127.0.0.1", "localhost"].indexOf(this.crawler.host) === -1) {
+        if (process.env.HTTP_PROXY !== undefined) {
+            this.httpProxy = URL(process.env.HTTP_PROXY);
+        } else if (process.env.http_proxy !== undefined) {
+            this.httpProxy = URL(process.env.http_proxy);
+        }
+    }
+
+    // Set Crawler proxy
+    if (this.httpProxy !== false) {
+        this.crawler.useProxy = true;
+        this.crawler.proxyHostname = this.httpProxy.hostname;
+        this.crawler.proxyPort = this.httpProxy.port;
+    }
+
     this.crawler.initialProtocol = this.uri.protocol.replace(":", "");
     this.crawler.userAgent = "Node/W3CValidator";
 
@@ -75,12 +93,20 @@ var W3CValidator = function(url, java_home) {
 
 W3CValidator.prototype.checkURL = function() {
     var url = this.chunk.pop();
+    var javaOpts = ""
+
+    // Set JAVA Proxy
+    if (this.httpProxy !== false) {
+        javaOpts += `-Dhttp.proxyHost=${this.httpProxy.hostname} -Dhttp.proxyPort=${this.httpProxy.port}`
+    }
 
     var spinner = new Spinner(`  ${url} %s`);
     spinner.start();
 
+
+
     var vnuPath = path.join(__dirname, "../vnu/vnu.jar").replace(/\s/g, "\\ ");
-    var child = exec(`java -jar ${vnuPath} --format json ${url}`, { env: { JAVA_HOME: this.JAVA_HOME }}, (error, stdout, stderr) => {
+    var child = exec(`java ${javaOpts} -jar ${vnuPath} --format json ${url}`, { env: { JAVA_HOME: this.JAVA_HOME }}, (error, stdout, stderr) => {
         var errors = JSON.parse(stderr);
 
         spinner.stop(true);
@@ -135,6 +161,10 @@ W3CValidator.prototype.create = function() {
 
     this.crawler.on("fetchcomplete", (item) => {
         this.chunk.push(item.url);
+        if (program.max && this.chunk.length >= program.max) {
+            this.crawler.emit("complete");
+            this.crawler.stop();
+        }
     });
 
     this.crawler.on("complete", () => {