Fix gzip decoding of HTTP sources.

The go http library will normally implicitly set an Accept-Encoding of
gzip to requests and then transparently decompress the response body.
However, if the Accept-Encoding header is explicitly set by the caller
it will no longer transparently decompress the body. This was causing
HTTP LLB sources to have unexpectedly compressed contents.

The fix here just unsets the Accept-Encoding header after the HEAD
request. Another possible fix would be to do our own gzip decompression
after reading the body if it was gzipped, but this approach seemed
slightly simpler.

Signed-off-by: Erik Sipsma <[email protected]>

Author: sipsma

client/client_test.go

@@ -3,6 +3,7 @@ package client
 import (
 	"archive/tar"
 	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/rand"
 	"crypto/rsa"
@@ -2196,6 +2197,49 @@ func testBuildHTTPSource(t *testing.T, sb integration.Sandbox) {
 	require.Equal(t, http.MethodHead, allReqs[1].Method)
 	require.Equal(t, "gzip", allReqs[1].Header.Get("Accept-Encoding"))
 
+	require.NoError(t, os.RemoveAll(filepath.Join(tmpdir, "foo")))
+
+	// update the content at the url to be gzipped now, the final output
+	// should remain the same
+	modTime = time.Now().Add(-23 * time.Hour)
+	var buf bytes.Buffer
+	gw := gzip.NewWriter(&buf)
+	_, err = gw.Write(resp.Content)
+	require.NoError(t, err)
+	require.NoError(t, gw.Close())
+	gzipBytes := buf.Bytes()
+	respGzip := httpserver.Response{
+		Etag:            identity.NewID(),
+		Content:         gzipBytes,
+		LastModified:    &modTime,
+		ContentEncoding: "gzip",
+	}
+	server.SetRoute("/foo", respGzip)
+
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: tmpdir,
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, server.Stats("/foo").AllRequests, 4)
+	require.Equal(t, server.Stats("/foo").CachedRequests, 1)
+
+	dt, err = os.ReadFile(filepath.Join(tmpdir, "foo"))
+	require.NoError(t, err)
+	require.Equal(t, resp.Content, dt)
+
+	allReqs = server.Stats("/foo").Requests
+	require.Equal(t, 4, len(allReqs))
+	require.Equal(t, http.MethodHead, allReqs[2].Method)
+	require.Equal(t, "gzip", allReqs[2].Header.Get("Accept-Encoding"))
+	require.Equal(t, http.MethodGet, allReqs[3].Method)
+	require.Equal(t, "gzip", allReqs[3].Header.Get("Accept-Encoding"))
+
 	// test extra options
 	st = llb.HTTP(server.URL+"/foo", llb.Filename("bar"), llb.Chmod(0741), llb.Chown(1000, 1000))
 
@@ -2212,7 +2256,7 @@ func testBuildHTTPSource(t *testing.T, sb integration.Sandbox) {
 	}, nil)
 	require.NoError(t, err)
 
-	require.Equal(t, server.Stats("/foo").AllRequests, 3)
+	require.Equal(t, server.Stats("/foo").AllRequests, 5)
 	require.Equal(t, server.Stats("/foo").CachedRequests, 1)
 
 	dt, err = os.ReadFile(filepath.Join(tmpdir, "bar"))

source/http/httpsource.go

@@ -205,6 +205,10 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, g session.Group, inde
 			resp.Body.Close()
 		}
 		req.Method = "GET"
+		// Unset explicit Accept-Encoding for GET, otherwise the go http library will not
+		// transparently decompress the response body when it is gzipped. It will still add
+		// this header implicitly when the request is made though.
+		req.Header.Del("Accept-Encoding")
 	}
 
 	resp, err := client.Do(req)

util/testutil/httpserver/server.go

@@ -51,6 +51,10 @@ func (s *TestServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Last-Modified", resp.LastModified.Format(time.RFC850))
 	}
 
+	if resp.ContentEncoding != "" {
+		w.Header().Set("Content-Encoding", resp.ContentEncoding)
+	}
+
 	if resp.Etag != "" {
 		w.Header().Set("ETag", resp.Etag)
 		if match := r.Header.Get("If-None-Match"); match == resp.Etag {
@@ -75,9 +79,10 @@ func (s *TestServer) Stats(name string) (st Stat) {
 }
 
 type Response struct {
-	Content      []byte
-	Etag         string
-	LastModified *time.Time
+	Content         []byte
+	Etag            string
+	LastModified    *time.Time
+	ContentEncoding string
 }
 
 type Stat struct {