Remove Accept-Encoding header on redirect

Prior to this commit, Puppet would copy all request headers in an HTTP
redirect, including Accept-Encoding. In some cases when HTTP compression
was enabled, the response would fail to get decompressed, then would
fail to get parsed and trigger a vague error.

This commit strips the Accept-Encoding headers on redirect, allowing
Ruby's built-in Net::HTTP to both compress and decompress the traffic.

Author: mhashizume

lib/puppet/http/redirector.rb

@@ -56,6 +56,10 @@ def redirect_to(request, response, redirects)
         next if header.casecmp('Authorization').zero? && request.uri.host.casecmp(location.host) != 0
         next if header.casecmp('Cookie').zero? && request.uri.host.casecmp(location.host) != 0
       end
+      # Allow Net::HTTP to set its own Accept-Encoding header to avoid errors with HTTP compression.
+      # See https://github.com/puppetlabs/puppet/issues/9143
+      next if header.casecmp('Accept-Encoding').zero?
+
       new_request[header] = value
     end
 

spec/unit/http/client_spec.rb

@@ -820,6 +820,17 @@ def redirect_to(status: 302, url:)
       response = client.get(https)
       expect(response).to be_success
     end
+
+    it "does not preserve accept-encoding header when redirecting" do
+      headers = { 'Accept-Encoding' => 'unwanted-encoding'}
+
+      stub_request(:get, start_url).with(headers: headers).to_return(redirect_to(url: other_host))
+      stub_request(:get, other_host).to_return(status: 200)
+
+      client.get(start_url, headers: headers)
+      expect(a_request(:get, other_host).
+      with{ |req| req.headers['Accept-Encoding'] != 'unwanted-encoding' }).to have_been_made
+    end
   end
 
   context "when response indicates an overloaded server" do