Merge pull request #5 from Fr0stM0urne/gzip_decode_fix

Fix for python3 http client when accept-encoding is gzip

Author: pimterry

.gitignore

@@ -1,3 +1,4 @@
 *.log
 node_modules
 coverage*
+.vscode

src/targets/c/libcurl.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const CodeBuilder = require('../../helpers/code-builder')
+const helpers = require('../../helpers/headers')
 
 module.exports = function (source, options) {
   const code = new CodeBuilder()
@@ -26,16 +27,21 @@ module.exports = function (source, options) {
   }
 
   // construct cookies
-  if (source.allHeaders.cookie) {
+  if (helpers.hasHeader(source.allHeaders, 'cookie')) {
     code.blank()
-      .push('curl_easy_setopt(hnd, CURLOPT_COOKIE, "%s");', source.allHeaders.cookie)
+      .push('curl_easy_setopt(hnd, CURLOPT_COOKIE, "%s");', helpers.getHeader(source.allHeaders, 'cookie'))
   }
 
   if (source.postData.text) {
     code.blank()
       .push('curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, %s);', JSON.stringify(source.postData.text))
   }
 
+  if (helpers.hasHeader(source.allHeaders, 'accept-encoding')) {
+    code.blank()
+      .push('curl_easy_setopt(hnd, CURLOPT_ACCEPT_ENCODING, "");')
+  }
+
   code.blank()
     .push('CURLcode ret = curl_easy_perform(hnd);')
 

src/targets/go/native.js

@@ -11,6 +11,7 @@
 'use strict'
 
 const CodeBuilder = require('../../helpers/code-builder')
+const helpers = require('../../helpers/headers')
 
 module.exports = function (source, options) {
   // Let's Go!
@@ -110,6 +111,11 @@ module.exports = function (source, options) {
   errorCheck()
 
   // Add headers
+
+  // Go automatically adds this and handles decompression, as long as we don't try to
+  // manually add it ourselves:
+  delete source.allHeaders[helpers.getHeaderName(source.allHeaders, 'accept-encoding')]
+
   if (Object.keys(source.allHeaders).length) {
     Object.keys(source.allHeaders).forEach(function (key) {
       code.push(indent, 'req.Header.Add("%s", "%qd")', key, source.allHeaders[key])

src/targets/python/python3.js

@@ -11,16 +11,25 @@
 'use strict'
 
 const CodeBuilder = require('../../helpers/code-builder')
+const helpers = require('../../helpers/headers')
 
 module.exports = function (source, options) {
   const code = new CodeBuilder()
+
   // Start Request
   code.push('import http.client')
 
   if (options.insecureSkipVerify) {
     code.push('import ssl')
   }
 
+  const mayBeGzipped = helpers.hasHeader(source.allHeaders, 'accept-encoding') &&
+    helpers.getHeader(source.allHeaders, 'accept-encoding').includes('gzip')
+
+  if (mayBeGzipped) {
+    code.push('import gzip')
+  }
+
   code.blank()
 
   // Check which protocol to be used for the client connection
@@ -90,7 +99,16 @@ module.exports = function (source, options) {
     .push('res = conn.getresponse()')
     .push('data = res.read()')
     .blank()
-    .push('print(data.decode("utf-8"))')
+
+  // Decode response
+  if (mayBeGzipped) {
+    code.push("if res.headers['content-encoding'] == 'gzip':")
+    code.push('    print(gzip.decompress(data).decode("utf-8"))')
+    code.push('else:')
+    code.push('    print(data.decode("utf-8"))')
+  } else {
+    code.push('print(data.decode("utf-8"))')
+  }
 
   return code.join()
 }

test/fixtures/output/c/libcurl/compression.c

@@ -0,0 +1,12 @@
+CURL *hnd = curl_easy_init();
+
+curl_easy_setopt(hnd, CURLOPT_CUSTOMREQUEST, "GET");
+curl_easy_setopt(hnd, CURLOPT_URL, "http://mockbin.com/har");
+
+struct curl_slist *headers = NULL;
+headers = curl_slist_append(headers, "accept-encoding: deflate, gzip, br");
+curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, headers);
+
+curl_easy_setopt(hnd, CURLOPT_ACCEPT_ENCODING, "");
+
+CURLcode ret = curl_easy_perform(hnd);

test/fixtures/output/clojure/clj_http/compression.clj

@@ -0,0 +1,3 @@
+(require '[clj-http.client :as client])
+
+(client/get "http://mockbin.com/har" {:headers {:accept-encoding "deflate, gzip, br"}})

test/fixtures/output/csharp/httpclient/compression.cs

@@ -0,0 +1,16 @@
+var clientHandler = new HttpClientHandler
+{
+    AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
+};
+var client = new HttpClient(clientHandler);
+var request = new HttpRequestMessage
+{
+    Method = HttpMethod.Get,
+    RequestUri = new Uri("http://mockbin.com/har"),
+};
+using (var response = await client.SendAsync(request))
+{
+    response.EnsureSuccessStatusCode();
+    var body = await response.Content.ReadAsStringAsync();
+    Console.WriteLine(body);
+}

test/fixtures/output/csharp/restsharp/compression.cs

@@ -0,0 +1,4 @@
+var client = new RestClient("http://mockbin.com/har");
+var request = new RestRequest(Method.GET);
+request.AddHeader("accept-encoding", "deflate, gzip, br");
+IRestResponse response = client.Execute(request);

test/fixtures/output/go/native/compression.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"io"
+)
+
+func main() {
+
+	url := "http://mockbin.com/har"
+
+	req, _ := http.NewRequest("GET", url, nil)
+
+	res, _ := http.DefaultClient.Do(req)
+
+	defer res.Body.Close()
+	body, _ := io.ReadAll(res.Body)
+
+	fmt.Println(res)
+	fmt.Println(string(body))
+
+}

test/fixtures/output/http/1.1/compression

@@ -0,0 +1,3 @@
+GET /har HTTP/1.1
+Accept-Encoding: deflate, gzip, br
+Host: mockbin.com