Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/BaseAgent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { AsyncLocalStorage } from 'node:async_hooks';

import { Agent, Dispatcher } from 'undici';

import { FetchOpaque } from './FetchOpaqueInterceptor.js';
import type { FetchOpaque } from './FetchOpaqueInterceptor.js';

export interface BaseAgentOptions extends Agent.Options {
opaqueLocalStorage?: AsyncLocalStorage<FetchOpaque>;
Expand Down
2 changes: 1 addition & 1 deletion src/FormData.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import _FormData from 'form-data';
const NON_ASCII_RE = /[^\x00-\x7F]/i;

export class FormData extends _FormData {
_getContentDisposition(value: any, options: any) {
_getContentDisposition(value: any, options: any): string | undefined {
// support non-ascii filename
// https://github.com/form-data/form-data/pull/571
let filename;
Expand Down
6 changes: 4 additions & 2 deletions src/HttpAgent.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import dns from 'node:dns';
import { LookupFunction, isIP } from 'node:net';
import { isIP } from 'node:net';
import type { LookupFunction } from 'node:net';

import { Agent, Dispatcher, buildConnector } from 'undici';

import { BaseAgent, BaseAgentOptions } from './BaseAgent.js';
import { BaseAgent } from './BaseAgent.js';
import type { BaseAgentOptions } from './BaseAgent.js';

export type CheckAddressFunction = (ip: string, family: number | string, hostname: string) => boolean;

Expand Down
41 changes: 24 additions & 17 deletions src/HttpClient.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import diagnosticsChannel from 'node:diagnostics_channel';
import type { Channel } from 'node:diagnostics_channel';
import { EventEmitter } from 'node:events';
import { createReadStream } from 'node:fs';
import { STATUS_CODES } from 'node:http';
import { LookupFunction } from 'node:net';
import type { LookupFunction } from 'node:net';
import { basename } from 'node:path';
import { performance } from 'node:perf_hooks';
import querystring from 'node:querystring';
Expand All @@ -23,13 +24,14 @@ import { request as undiciRequest, Dispatcher, Agent, getGlobalDispatcher, Pool
import undiciSymbols from 'undici/lib/core/symbols.js';

import { initDiagnosticsChannel } from './diagnosticsChannel.js';
import { FetchOpaque } from './FetchOpaqueInterceptor.js';
import type { FetchOpaque } from './FetchOpaqueInterceptor.js';
import { FormData } from './FormData.js';
import { HttpAgent, CheckAddressFunction } from './HttpAgent.js';
import { HttpAgent } from './HttpAgent.js';
import type { CheckAddressFunction } from './HttpAgent.js';
import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
import { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request.js';
import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.js';
import type { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request.js';
import type { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.js';
import symbols from './symbols.js';
import { parseJSON, digestAuthHeader, globalId, performanceTime, isReadable, updateSocketInfo } from './utils.js';

Expand All @@ -38,7 +40,7 @@ type UndiciRequestOption = Exists<Parameters<typeof undiciRequest>[1]>;
type PropertyShouldBe<T, K extends keyof T, V> = Omit<T, K> & { [P in K]: V };
type IUndiciRequestOption = PropertyShouldBe<UndiciRequestOption, 'headers', IncomingHttpHeaders>;

export const PROTO_RE = /^https?:\/\//i;
export const PROTO_RE: RegExp = /^https?:\/\//i;

export interface UndiciTimingInfo {
startTime: number;
Expand All @@ -64,7 +66,7 @@ export interface UndiciTimingInfo {
// keep typo compatibility
export interface UnidiciTimingInfo extends UndiciTimingInfo {}

function noop() {
function noop(): void {
// noop
}

Expand Down Expand Up @@ -108,19 +110,19 @@ export type ClientOptions = {
};
};

export const VERSION = 'VERSION';
export const VERSION: string = 'VERSION';
// 'node-urllib/4.0.0 Node.js/18.19.0 (darwin; x64)'
export const HEADER_USER_AGENT = `node-urllib/${VERSION} Node.js/${process.version.substring(1)} (${process.platform}; ${process.arch})`;
export const HEADER_USER_AGENT: string = `node-urllib/${VERSION} Node.js/${process.version.substring(1)} (${process.platform}; ${process.arch})`;

function getFileName(stream: Readable) {
function getFileName(stream: Readable): string {
const filePath: string = (stream as any).path;
if (filePath) {
return basename(filePath);
}
return '';
}

function defaultIsRetry(response: HttpClientResponse) {
function defaultIsRetry(response: HttpClientResponse): boolean {
return response.status >= 500;
}

Expand All @@ -132,7 +134,12 @@ export type RequestContext = {
history: string[];
};

export const channels = {
export const channels: {
request: Channel;
response: Channel;
fetchRequest: Channel;
fetchResponse: Channel;
} = {
request: diagnosticsChannel.channel('urllib:request'),
response: diagnosticsChannel.channel('urllib:response'),
fetchRequest: diagnosticsChannel.channel('urllib:fetch:request'),
Expand Down Expand Up @@ -205,15 +212,15 @@ export class HttpClient extends EventEmitter {
initDiagnosticsChannel();
}

getDispatcher() {
getDispatcher(): Dispatcher {
return this.#dispatcher ?? getGlobalDispatcher();
}

setDispatcher(dispatcher: Dispatcher) {
setDispatcher(dispatcher: Dispatcher): void {
this.#dispatcher = dispatcher;
}

getDispatcherPoolStats() {
getDispatcherPoolStats(): Record<string, PoolStat> {
const agent = this.getDispatcher();
// origin => Pool Instance
const clients: Map<string, WeakRef<Pool>> | undefined = Reflect.get(agent, undiciSymbols.kClients);
Expand All @@ -239,12 +246,12 @@ export class HttpClient extends EventEmitter {
return poolStatsMap;
}

async request<T = any>(url: RequestURL, options?: RequestOptions) {
async request<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>> {
return await this.#requestInternal<T>(url, options);
}

// alias to request, keep compatible with urllib@2 HttpClient.curl
async curl<T = any>(url: RequestURL, options?: RequestOptions) {
async curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>> {
return await this.request<T>(url, options);
}

Expand Down
4 changes: 2 additions & 2 deletions src/diagnosticsChannel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Socket } from 'node:net';
import { performance } from 'node:perf_hooks';
import { debuglog } from 'node:util';

import { DiagnosticsChannel } from 'undici';
import type { DiagnosticsChannel } from 'undici';

// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
Expand Down Expand Up @@ -82,7 +82,7 @@ function getRequestOpaque(request: DiagnosticsChannel.Request, kHandler?: symbol
return handler?.opts?.opaque ?? handler?.opaque;
}

export function initDiagnosticsChannel() {
export function initDiagnosticsChannel(): void {
// make sure init global DiagnosticsChannel once
if (initedDiagnosticsChannel) return;
initedDiagnosticsChannel = true;
Expand Down
49 changes: 21 additions & 28 deletions src/fetch.ts
Original file line number Diff line number Diff line change
@@ -1,39 +1,32 @@
import { AsyncLocalStorage } from 'node:async_hooks';
import { debuglog } from 'node:util';

import {
fetch as UndiciFetch,
RequestInfo,
RequestInit,
Request,
Response,
Agent,
getGlobalDispatcher,
Pool,
Dispatcher,
} from 'undici';
import { fetch as UndiciFetch, Request, Response, Agent, getGlobalDispatcher, Pool, Dispatcher } from 'undici';
import type { RequestInfo, RequestInit } from 'undici';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import undiciSymbols from 'undici/lib/core/symbols.js';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import { getResponseState } from 'undici/lib/web/fetch/response.js';

import { BaseAgent, BaseAgentOptions } from './BaseAgent.js';
import { BaseAgent } from './BaseAgent.js';
import type { BaseAgentOptions } from './BaseAgent.js';
import { initDiagnosticsChannel } from './diagnosticsChannel.js';
import { FetchOpaque } from './FetchOpaqueInterceptor.js';
import { HttpAgent, HttpAgentOptions } from './HttpAgent.js';
import {
channels,
import type { FetchOpaque } from './FetchOpaqueInterceptor.js';
import { HttpAgent } from './HttpAgent.js';
import type { HttpAgentOptions } from './HttpAgent.js';
import { channels } from './HttpClient.js';
import type {
ClientOptions,
PoolStat,
RequestDiagnosticsMessage,
ResponseDiagnosticsMessage,
UndiciTimingInfo,
} from './HttpClient.js';
import { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
import { FetchMeta, HttpMethod, RequestMeta } from './Request.js';
import { RawResponseWithMeta, SocketInfo } from './Response.js';
import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
import type { FetchMeta, HttpMethod, RequestMeta } from './Request.js';
import type { RawResponseWithMeta, SocketInfo } from './Response.js';
import symbols from './symbols.js';
import { convertHeader, globalId, performanceTime, updateSocketInfo } from './utils.js';

Expand Down Expand Up @@ -63,7 +56,7 @@ export class FetchFactory {

static #instance = new FetchFactory();

setClientOptions(clientOptions: ClientOptions) {
setClientOptions(clientOptions: ClientOptions): void {
let dispatcherOption: BaseAgentOptions = {
opaqueLocalStorage: this.#opaqueLocalStorage,
};
Expand Down Expand Up @@ -96,15 +89,15 @@ export class FetchFactory {
initDiagnosticsChannel();
}

getDispatcher() {
getDispatcher(): Dispatcher {
return this.#dispatcher ?? getGlobalDispatcher();
}

setDispatcher(dispatcher: Agent) {
setDispatcher(dispatcher: Agent): void {
this.#dispatcher = dispatcher;
}

getDispatcherPoolStats() {
getDispatcherPoolStats(): Record<string, PoolStat> {
const agent = this.getDispatcher();
// origin => Pool Instance
const clients: Map<string, WeakRef<Pool>> | undefined = Reflect.get(agent, undiciSymbols.kClients);
Expand All @@ -130,11 +123,11 @@ export class FetchFactory {
return poolStatsMap;
}

static setClientOptions(clientOptions: ClientOptions) {
static setClientOptions(clientOptions: ClientOptions): void {
FetchFactory.#instance.setClientOptions(clientOptions);
}

static getDispatcherPoolStats() {
static getDispatcherPoolStats(): Record<string, PoolStat> {
return FetchFactory.#instance.getDispatcherPoolStats();
}

Expand Down Expand Up @@ -283,11 +276,11 @@ export class FetchFactory {
return res!;
}

static getDispatcher() {
static getDispatcher(): Dispatcher {
return FetchFactory.#instance.getDispatcher();
}

static setDispatcher(dispatcher: Agent) {
static setDispatcher(dispatcher: Agent): void {
FetchFactory.#instance.setDispatcher(dispatcher);
}

Expand All @@ -296,4 +289,4 @@ export class FetchFactory {
}
}

export const fetch = FetchFactory.fetch;
export const fetch: (input: RequestInfo, init?: UrllibRequestInit) => Promise<Response> = FetchFactory.fetch;
41 changes: 21 additions & 20 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ import { patchForNode16 } from './utils.js';
patchForNode16();

import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
import { RequestOptions, RequestURL } from './Request.js';
import type { RequestOptions, RequestURL } from './Request.js';
import type { HttpClientResponse } from './Response.js';

let httpClient: HttpClient;
let allowH2HttpClient: HttpClient;
Expand Down Expand Up @@ -63,7 +64,10 @@ interface UrllibRequestOptions extends RequestOptions {
allowH2?: boolean;
}

export async function request<T = any>(url: RequestURL, options?: UrllibRequestOptions) {
export async function request<T = any>(
url: RequestURL,
options?: UrllibRequestOptions,
): Promise<HttpClientResponse<T>> {
if (options?.socketPath) {
let domainSocketHttpclient = domainSocketHttpClients.get<HttpClient>(options.socketPath);
if (!domainSocketHttpclient) {
Expand All @@ -83,7 +87,7 @@ export async function request<T = any>(url: RequestURL, options?: UrllibRequestO
// import * as urllib from 'urllib';
// urllib.curl(url);
// ```
export async function curl<T = any>(url: RequestURL, options?: UrllibRequestOptions) {
export async function curl<T = any>(url: RequestURL, options?: UrllibRequestOptions): Promise<HttpClientResponse<T>> {
return await request<T>(url, options);
}

Expand All @@ -95,25 +99,16 @@ export {
setGlobalDispatcher,
getGlobalDispatcher,
Request,
RequestInfo,
RequestInit,
Response,
BodyInit,
ResponseInit,
Headers,
FormData,
} from 'undici';
export type { RequestInfo, RequestInit, BodyInit, ResponseInit } from 'undici';
// HttpClient2 is keep compatible with urllib@2 HttpClient2
export {
HttpClient,
HttpClient as HttpClient2,
HEADER_USER_AGENT as USER_AGENT,
RequestDiagnosticsMessage,
ResponseDiagnosticsMessage,
ClientOptions,
} from './HttpClient.js';
export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT } from './HttpClient.js';
export type { RequestDiagnosticsMessage, ResponseDiagnosticsMessage, ClientOptions } from './HttpClient.js';
// RequestOptions2 is keep compatible with urllib@2 RequestOptions2
export {
export type {
RequestOptions,
RequestOptions as RequestOptions2,
RequestURL,
Expand All @@ -122,16 +117,22 @@ export {
FixJSONCtlChars,
} from './Request.js';

export { CheckAddressFunction } from './HttpAgent.js';
export type { CheckAddressFunction } from './HttpAgent.js';

export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response.js';
export { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
export type { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response.js';
export type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
export * from './HttpClientError.js';
export { FetchFactory, fetch } from './fetch.js';
export { FormData as WebFormData } from './FormData.js';

export default {
const urllib: {
request: typeof request;
curl: typeof curl;
USER_AGENT: string;
} = {
request,
curl,
USER_AGENT: HEADER_USER_AGENT,
};

export default urllib;
Loading
Loading