Skip to content

add direct http proxy support #635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 15, 2025
Merged

add direct http proxy support #635

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/app.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ pub const App = struct {
run_mode: RunMode,
gc_hints: bool = false,
tls_verify_host: bool = true,
http_proxy: ?std.Uri = null,
};

pub fn init(allocator: Allocator, config: Config) !*App {
Expand All @@ -54,6 +55,7 @@ pub const App = struct {
.app_dir_path = app_dir_path,
.notification = notification,
.http_client = try HttpClient.init(allocator, 5, .{
.http_proxy = config.http_proxy,
.tls_verify_host = config.tls_verify_host,
}),
.config = config,
Expand Down
2 changes: 1 addition & 1 deletion src/browser/browser.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ pub const Page = struct {
var response = try request.sendSync(.{});

// would be different than self.url in the case of a redirect
self.url = try URL.fromURI(arena, request.uri);
self.url = try URL.fromURI(arena, request.request_uri);

const header = response.header;
try session.cookie_jar.populateFromResponse(&self.url.uri, &header);
Expand Down
2 changes: 1 addition & 1 deletion src/browser/xhr/xhr.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,7 +497,7 @@ pub const XMLHttpRequest = struct {
self.state = .loading;
self.dispatchEvt("readystatechange");

try self.cookie_jar.populateFromResponse(self.request.?.uri, &header);
try self.cookie_jar.populateFromResponse(self.request.?.request_uri, &header);
}

if (progress.data) |data| {
Expand Down
116 changes: 73 additions & 43 deletions src/http/client.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,15 @@ const MAX_HEADER_LINE_LEN = 4096;
pub const Client = struct {
allocator: Allocator,
state_pool: StatePool,
http_proxy: ?Uri,
root_ca: tls.config.CertBundle,
tls_verify_host: bool = true,
idle_connections: IdleConnections,
connection_pool: std.heap.MemoryPool(Connection),

const Opts = struct {
tls_verify_host: bool = true,
http_proxy: ?std.Uri = null,
max_idle_connection: usize = 10,
};

Expand All @@ -70,6 +72,7 @@ pub const Client = struct {
.root_ca = root_ca,
.allocator = allocator,
.state_pool = state_pool,
.http_proxy = opts.http_proxy,
.idle_connections = idle_connections,
.tls_verify_host = opts.tls_verify_host,
.connection_pool = std.heap.MemoryPool(Connection).init(allocator),
Expand Down Expand Up @@ -150,10 +153,14 @@ pub const Request = struct {
method: Method,

// The URI we're requested
uri: *const Uri,
request_uri: *const Uri,

// The URI that we're connecting to. Can be different than request_uri when
// proxying is enabled
connect_uri: *const Uri,

// If we're redirecting, this is where we're redirecting to. The only reason
// we really have this is so that we can set self.uri = &self.redirect_url.?
// we really have this is so that we can set self.request_uri = &self.redirect_url.?
redirect_uri: ?Uri = null,

// Optional body
Expand All @@ -174,9 +181,12 @@ pub const Request = struct {
// for other requests
_keepalive: bool,

_port: u16,
// extracted from request_uri
_request_host: []const u8,

_host: []const u8,
// extracted from connect_uri
_connect_port: u16,
_connect_host: []const u8,

// whether or not the socket comes from the connection pool. If it does,
// and we get an error sending the header, we might retry on a new connection
Expand Down Expand Up @@ -222,16 +232,18 @@ pub const Request = struct {
};

fn init(client: *Client, state: *State, method: Method, uri: *const Uri) !Request {
const secure, const host, const port = try decomposeURL(uri);
const decomposed = try decomposeURL(client, uri);
return .{
.uri = uri,
.request_uri = uri,
.connect_uri = decomposed.connect_uri,
.body = null,
.headers = .{},
.method = method,
.arena = state.arena.allocator(),
._secure = secure,
._host = host,
._port = port,
._secure = decomposed.secure,
._connect_host = decomposed.connect_host,
._connect_port = decomposed.connect_port,
._request_host = decomposed.request_host,
._state = state,
._client = client,
._connection = null,
Expand All @@ -249,26 +261,44 @@ pub const Request = struct {
self._client.state_pool.release(self._state);
}

fn decomposeURL(uri: *const Uri) !struct { bool, []const u8, u16 } {
const DecomposedURL = struct {
secure: bool,
connect_port: u16,
connect_host: []const u8,
connect_uri: *const std.Uri,
request_host: []const u8,
};
fn decomposeURL(client: *const Client, uri: *const Uri) !DecomposedURL {
if (uri.host == null) {
return error.UriMissingHost;
}
const request_host = uri.host.?.percent_encoded;

var secure: bool = undefined;
var connect_uri = uri;
var connect_host = request_host;
if (client.http_proxy) |*proxy| {
connect_uri = proxy;
connect_host = proxy.host.?.percent_encoded;
}

const scheme = uri.scheme;
var secure: bool = undefined;
const scheme = connect_uri.scheme;
if (std.ascii.eqlIgnoreCase(scheme, "https")) {
secure = true;
} else if (std.ascii.eqlIgnoreCase(scheme, "http")) {
secure = false;
} else {
return error.UnsupportedUriScheme;
}
const connect_port: u16 = connect_uri.port orelse if (secure) 443 else 80;

const host = uri.host.?.percent_encoded;
const port: u16 = uri.port orelse if (secure) 443 else 80;

return .{ secure, host, port };
return .{
.secure = secure,
.connect_port = connect_port,
.connect_host = connect_host,
.connect_uri = connect_uri,
.request_host = request_host,
};
}

// Called in deinit, but also called when we're redirecting to another page
Expand All @@ -293,11 +323,11 @@ pub const Request = struct {
errdefer client.connection_pool.destroy(connection);

connection.* = .{
.socket = socket,
.tls = null,
.port = self._port,
.socket = socket,
.blocking = blocking,
.host = try client.allocator.dupe(u8, self._host),
.port = self._connect_port,
.host = try client.allocator.dupe(u8, self._connect_host),
};

return connection;
Expand Down Expand Up @@ -374,12 +404,10 @@ pub const Request = struct {
return err;
};

errdefer self.destroyConnection(connection);

if (self._secure) {
connection.tls = .{
.blocking = try tls.client(std.net.Stream{ .handle = socket }, .{
.host = connection.host,
.host = self._connect_host,
.root_ca = self._client.root_ca,
.insecure_skip_verify = self._tls_verify_host == false,
// .key_log_callback = tls.config.key_log.callback,
Expand All @@ -391,11 +419,9 @@ pub const Request = struct {
self._connection_from_keepalive = false;
}

errdefer self.destroyConnection(self._connection.?);

var handler = SyncHandler{ .request = self };
return handler.send() catch |err| {
log.warn("HTTP error: {any} ({any} {any} {d})", .{ err, self.method, self.uri, self._redirect_count });
log.warn("HTTP error: {any} ({any} {any} {d})", .{ err, self.method, self.request_uri, self._redirect_count });
return err;
};
}
Expand Down Expand Up @@ -461,7 +487,7 @@ pub const Request = struct {
if (self._secure) {
connection.tls = .{
.nonblocking = try tls.nb.Client().init(self._client.allocator, .{
.host = connection.host,
.host = self._connect_host,
.root_ca = self._client.root_ca,
.insecure_skip_verify = self._tls_verify_host == false,
.key_log_callback = tls.config.key_log.callback,
Expand Down Expand Up @@ -501,7 +527,7 @@ pub const Request = struct {
}

if (!self._has_host_header) {
try self.headers.append(arena, .{ .name = "Host", .value = self._host });
try self.headers.append(arena, .{ .name = "Host", .value = self._request_host });
}

try self.headers.append(arena, .{ .name = "User-Agent", .value = "Lightpanda/1.0" });
Expand All @@ -512,7 +538,7 @@ pub const Request = struct {
self.releaseConnection();

// CANNOT reset the arena (╥﹏╥)
// We need it for self.uri (which we're about to use to resolve
// We need it for self.request_uri (which we're about to use to resolve
// redirect.location, and it might own some/all headers)

const redirect_count = self._redirect_count;
Expand All @@ -522,14 +548,16 @@ pub const Request = struct {

var buf = try self.arena.alloc(u8, 2048);

const previous_host = self._host;
self.redirect_uri = try self.uri.resolve_inplace(redirect.location, &buf);
const previous_request_host = self._request_host;
self.redirect_uri = try self.request_uri.resolve_inplace(redirect.location, &buf);

self.uri = &self.redirect_uri.?;
const secure, const host, const port = try decomposeURL(self.uri);
self._host = host;
self._port = port;
self._secure = secure;
self.request_uri = &self.redirect_uri.?;
const decomposed = try decomposeURL(self._client, self.request_uri);
self.connect_uri = decomposed.connect_uri;
self._request_host = decomposed.request_host;
self._connect_host = decomposed.connect_host;
self._connect_port = decomposed.connect_port;
self._secure = decomposed.secure;
self._keepalive = false;
self._redirect_count = redirect_count + 1;

Expand All @@ -538,7 +566,7 @@ pub const Request = struct {
// to a GET.
self.method = .GET;
}
log.info("redirecting to: {any} {any}", .{ self.method, self.uri });
log.info("redirecting to: {any} {any}", .{ self.method, self.request_uri });

if (self.body != null and self.method == .GET) {
// If we have a body and the method is a GET, then we must be following
Expand All @@ -553,10 +581,10 @@ pub const Request = struct {
}
}

if (std.mem.eql(u8, previous_host, host) == false) {
if (std.mem.eql(u8, previous_request_host, self._request_host) == false) {
for (self.headers.items) |*hdr| {
if (std.mem.eql(u8, hdr.name, "Host")) {
hdr.value = host;
hdr.value = self._request_host;
break;
}
}
Expand All @@ -577,11 +605,11 @@ pub const Request = struct {
return null;
}

return self._client.idle_connections.get(self._secure, self._host, self._port, blocking);
return self._client.idle_connections.get(self._secure, self._connect_host, self._connect_port, blocking);
}

fn createSocket(self: *Request, blocking: bool) !struct { posix.socket_t, std.net.Address } {
const addresses = try std.net.getAddressList(self.arena, self._host, self._port);
const addresses = try std.net.getAddressList(self.arena, self._connect_host, self._connect_port);
if (addresses.addrs.len == 0) {
return error.UnknownHostName;
}
Expand All @@ -600,13 +628,15 @@ pub const Request = struct {
}

fn buildHeader(self: *Request) ![]const u8 {
const proxied = self.connect_uri != self.request_uri;

const buf = self._state.header_buf;
var fbs = std.io.fixedBufferStream(buf);
var writer = fbs.writer();

try writer.writeAll(@tagName(self.method));
try writer.writeByte(' ');
try self.uri.writeToStream(.{ .path = true, .query = true }, writer);
try self.request_uri.writeToStream(.{ .scheme = proxied, .authority = proxied, .path = true, .query = true }, writer);
try writer.writeAll(" HTTP/1.1\r\n");
for (self.headers.items) |header| {
try writer.writeAll(header.name);
Expand Down Expand Up @@ -906,7 +936,7 @@ fn AsyncHandler(comptime H: type, comptime L: type) type {
}

fn handleError(self: *Self, comptime msg: []const u8, err: anyerror) void {
log.err(msg ++ ": {any} ({any} {any})", .{ err, self.request.method, self.request.uri });
log.err(msg ++ ": {any} ({any} {any})", .{ err, self.request.method, self.request.request_uri });
self.handler.onHttpResponse(err) catch {};
// just to be safe
self.request._keepalive = false;
Expand Down Expand Up @@ -1127,7 +1157,7 @@ const SyncHandler = struct {
// See CompressedReader for an explanation. This isn't great code. Sorry.
if (reader.response.get("content-encoding")) |ce| {
if (std.ascii.eqlIgnoreCase(ce, "gzip") == false) {
log.err("unsupported content encoding '{s}' for: {}", .{ ce, request.uri });
log.err("unsupported content encoding '{s}' for: {}", .{ ce, request.request_uri });
return error.UnsupportedContentEncoding;
}

Expand Down
Loading