Fetch: handle compressed git+http

andrewrk · andrewrk · commit af2ac24333a7 · 2025-08-07T19:55:40.000-07:00
diff --git a/lib/std/http.zig b/lib/std/http.zig
@@ -292,6 +292,14 @@ pub const ContentEncoding = enum {
         });
         return map.get(s);
     }
+
+    pub fn minBufferCapacity(ce: ContentEncoding) usize {
+        return switch (ce) {
+            .zstd => std.compress.zstd.default_window_len,
+            .gzip, .deflate => std.compress.flate.max_window_len,
+            .compress, .identity => 0,
+        };
+    }
 };
 
 pub const Connection = enum {
@@ -464,8 +472,8 @@ pub const Reader = struct {
         transfer_encoding: TransferEncoding,
         content_length: ?u64,
         content_encoding: ContentEncoding,
-        decompressor: *Decompressor,
-        decompression_buffer: []u8,
+        decompress: *Decompress,
+        decompress_buffer: []u8,
     ) *std.Io.Reader {
         if (transfer_encoding == .none and content_length == null) {
             assert(reader.state == .received_head);
@@ -475,22 +483,22 @@ pub const Reader = struct {
                     return reader.in;
                 },
                 .deflate => {
-                    decompressor.* = .{ .flate = .init(reader.in, .zlib, decompression_buffer) };
-                    return &decompressor.flate.reader;
+                    decompress.* = .{ .flate = .init(reader.in, .zlib, decompress_buffer) };
+                    return &decompress.flate.reader;
                 },
                 .gzip => {
-                    decompressor.* = .{ .flate = .init(reader.in, .gzip, decompression_buffer) };
-                    return &decompressor.flate.reader;
+                    decompress.* = .{ .flate = .init(reader.in, .gzip, decompress_buffer) };
+                    return &decompress.flate.reader;
                 },
                 .zstd => {
-                    decompressor.* = .{ .zstd = .init(reader.in, decompression_buffer, .{ .verify_checksum = false }) };
-                    return &decompressor.zstd.reader;
+                    decompress.* = .{ .zstd = .init(reader.in, decompress_buffer, .{ .verify_checksum = false }) };
+                    return &decompress.zstd.reader;
                 },
                 .compress => unreachable,
             }
         }
         const transfer_reader = bodyReader(reader, transfer_buffer, transfer_encoding, content_length);
-        return decompressor.init(transfer_reader, decompression_buffer, content_encoding);
+        return decompress.init(transfer_reader, decompress_buffer, content_encoding);
     }
 
     fn contentLengthStream(
@@ -692,33 +700,33 @@ pub const Reader = struct {
     }
 };
 
-pub const Decompressor = union(enum) {
+pub const Decompress = union(enum) {
     flate: std.compress.flate.Decompress,
     zstd: std.compress.zstd.Decompress,
     none: *std.Io.Reader,
 
     pub fn init(
-        decompressor: *Decompressor,
+        decompress: *Decompress,
         transfer_reader: *std.Io.Reader,
         buffer: []u8,
         content_encoding: ContentEncoding,
     ) *std.Io.Reader {
         switch (content_encoding) {
             .identity => {
-                decompressor.* = .{ .none = transfer_reader };
+                decompress.* = .{ .none = transfer_reader };
                 return transfer_reader;
             },
             .deflate => {
-                decompressor.* = .{ .flate = .init(transfer_reader, .zlib, buffer) };
-                return &decompressor.flate.reader;
+                decompress.* = .{ .flate = .init(transfer_reader, .zlib, buffer) };
+                return &decompress.flate.reader;
             },
             .gzip => {
-                decompressor.* = .{ .flate = .init(transfer_reader, .gzip, buffer) };
-                return &decompressor.flate.reader;
+                decompress.* = .{ .flate = .init(transfer_reader, .gzip, buffer) };
+                return &decompress.flate.reader;
             },
             .zstd => {
-                decompressor.* = .{ .zstd = .init(transfer_reader, buffer, .{ .verify_checksum = false }) };
-                return &decompressor.zstd.reader;
+                decompress.* = .{ .zstd = .init(transfer_reader, buffer, .{ .verify_checksum = false }) };
+                return &decompress.zstd.reader;
             },
             .compress => unreachable,
         }
diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig
@@ -724,8 +724,8 @@ pub const Response = struct {
     pub fn readerDecompressing(
         response: *Response,
         transfer_buffer: []u8,
-        decompressor: *http.Decompressor,
-        decompression_buffer: []u8,
+        decompress: *http.Decompress,
+        decompress_buffer: []u8,
     ) *Reader {
         response.head.invalidateStrings();
         const head = &response.head;
@@ -734,8 +734,8 @@ pub const Response = struct {
             head.transfer_encoding,
             head.content_length,
             head.content_encoding,
-            decompressor,
-            decompression_buffer,
+            decompress,
+            decompress_buffer,
         );
     }
 
@@ -1797,8 +1797,8 @@ pub fn fetch(client: *Client, options: FetchOptions) FetchError!FetchResult {
     defer if (options.decompress_buffer == null) client.allocator.free(decompress_buffer);
 
     var transfer_buffer: [64]u8 = undefined;
-    var decompressor: http.Decompressor = undefined;
-    const reader = response.readerDecompressing(&transfer_buffer, &decompressor, decompress_buffer);
+    var decompress: http.Decompress = undefined;
+    const reader = response.readerDecompressing(&transfer_buffer, &decompress, decompress_buffer);
 
     _ = reader.streamRemaining(response_writer) catch |err| switch (err) {
         error.ReadFailed => return response.bodyErr().?,
diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig
@@ -883,7 +883,9 @@ const Resource = union(enum) {
     const HttpRequest = struct {
         request: std.http.Client.Request,
         response: std.http.Client.Response,
-        buffer: []u8,
+        transfer_buffer: []u8,
+        decompress: std.http.Decompress,
+        decompress_buffer: []u8,
     };
 
     fn deinit(resource: *Resource) void {
@@ -892,7 +894,6 @@ const Resource = union(enum) {
             .http_request => |*http_request| http_request.request.deinit(),
             .git => |*git_resource| {
                 git_resource.fetch_stream.deinit();
-                git_resource.session.deinit();
             },
             .dir => |*dir| dir.close(),
         }
@@ -902,7 +903,11 @@ const Resource = union(enum) {
     fn reader(resource: *Resource) *std.Io.Reader {
         return switch (resource.*) {
             .file => |*file_reader| return &file_reader.interface,
-            .http_request => |*http_request| return http_request.response.reader(http_request.buffer),
+            .http_request => |*http_request| return http_request.response.readerDecompressing(
+                http_request.transfer_buffer,
+                &http_request.decompress,
+                http_request.decompress_buffer,
+            ),
             .git => |*g| return &g.fetch_stream.reader,
             .dir => unreachable,
         };
@@ -971,7 +976,6 @@ const FileType = enum {
 const init_resource_buffer_size = git.Packet.max_data_length;
 
 fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u8) RunError!void {
-    const gpa = f.arena.child_allocator;
     const arena = f.arena.allocator();
     const eb = &f.error_bundle;
 
@@ -995,7 +999,9 @@ fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u
             .request = http_client.request(.GET, uri, .{}) catch |err|
                 return f.fail(f.location_tok, try eb.printString("unable to connect to server: {t}", .{err})),
             .response = undefined,
-            .buffer = reader_buffer,
+            .transfer_buffer = reader_buffer,
+            .decompress_buffer = &.{},
+            .decompress = undefined,
         } };
         const request = &resource.http_request.request;
         errdefer request.deinit();
@@ -1019,6 +1025,7 @@ fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u
             .{ response.head.status, response.head.status.phrase() orelse "" },
         ));
 
+        resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
         return;
     }
 
@@ -1027,13 +1034,12 @@ fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u
     {
         var transport_uri = uri;
         transport_uri.scheme = uri.scheme["git+".len..];
-        var session = git.Session.init(gpa, http_client, transport_uri, reader_buffer) catch |err| {
-            return f.fail(f.location_tok, try eb.printString(
-                "unable to discover remote git server capabilities: {s}",
-                .{@errorName(err)},
-            ));
+        var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch |err| {
+            return f.fail(
+                f.location_tok,
+                try eb.printString("unable to discover remote git server capabilities: {t}", .{err}),
+            );
         };
-        errdefer session.deinit();
 
         const want_oid = want_oid: {
             const want_ref =
diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig
