Add support for private github repos via git:directory

packages/playground/blueprints/src/lib/v1/compile.ts

@@ -83,6 +83,10 @@ export interface CompileBlueprintV1Options {
 	 * A filesystem to use for the blueprint.
 	 */
 	streamBundledFile?: StreamBundledFile;
+	/**
+	 * Additional headers to pass to git operations.
+	 */
+	gitAdditionalHeaders?: Record<string, string>;
 	/**
 	 * Additional steps to add to the blueprint.
 	 */
@@ -142,6 +146,7 @@ function compileBlueprintJson(
 		onBlueprintValidated = () => {},
 		corsProxy,
 		streamBundledFile,
+		gitAdditionalHeaders,
 		additionalSteps,
 	}: CompileBlueprintV1Options = {}
 ): CompiledBlueprintV1 {
@@ -321,6 +326,7 @@ function compileBlueprintJson(
 			totalProgressWeight,
 			corsProxy,
 			streamBundledFile,
+			gitAdditionalHeaders,
 		})
 	);
 
@@ -514,6 +520,10 @@ interface CompileStepArgsOptions {
 	 * A filesystem to use for the "blueprint" resource type.
 	 */
 	streamBundledFile?: StreamBundledFile;
+	/**
+	 * Additional headers to pass to git operations.
+	 */
+	gitAdditionalHeaders?: Record<string, string>;
 }
 
 /**
@@ -532,6 +542,7 @@ function compileStep<S extends StepDefinition>(
 		totalProgressWeight,
 		corsProxy,
 		streamBundledFile,
+		gitAdditionalHeaders,
 	}: CompileStepArgsOptions
 ): { run: CompiledV1Step; step: S; resources: Array<Resource<any>> } {
 	const stepProgress = rootProgressTracker.stage(
@@ -546,6 +557,7 @@ function compileStep<S extends StepDefinition>(
 				semaphore,
 				corsProxy,
 				streamBundledFile,
+				gitAdditionalHeaders,
 			});
 		}
 		args[key] = value;

packages/playground/blueprints/src/lib/v1/resources.ts

@@ -157,12 +157,14 @@ export abstract class Resource<T extends File | Directory> {
 			progress,
 			corsProxy,
 			streamBundledFile,
+			gitAdditionalHeaders,
 		}: {
 			/** Optional semaphore to limit concurrent downloads */
 			semaphore?: Semaphore;
 			progress?: ProgressTracker;
 			corsProxy?: string;
 			streamBundledFile?: StreamBundledFile;
+			gitAdditionalHeaders?: Record<string, string>;
 		}
 	): Resource<File | Directory> {
 		let resource: Resource<File | Directory>;
@@ -185,6 +187,7 @@ export abstract class Resource<T extends File | Directory> {
 			case 'git:directory':
 				resource = new GitDirectoryResource(ref, progress, {
 					corsProxy,
+					additionalHeaders: gitAdditionalHeaders,
 				});
 				break;
 			case 'literal:directory':
@@ -556,12 +559,18 @@ export class UrlResource extends FetchResource {
  */
 export class GitDirectoryResource extends Resource<Directory> {
 	private reference: GitDirectoryReference;
-	private options?: { corsProxy?: string };
+	private options?: {
+		corsProxy?: string;
+		additionalHeaders?: Record<string, string>;
+	};
 
 	constructor(
 		reference: GitDirectoryReference,
 		_progress?: ProgressTracker,
-		options?: { corsProxy?: string }
+		options?: {
+			corsProxy?: string;
+			additionalHeaders?: Record<string, string>;
+		}
 	) {
 		super();
 		this.reference = reference;
@@ -574,11 +583,19 @@ export class GitDirectoryResource extends Resource<Directory> {
 			? `${this.options.corsProxy}${this.reference.url}`
 			: this.reference.url;
 
-		const commitHash = await resolveCommitHash(repoUrl, {
-			value: this.reference.ref,
-			type: this.reference.refType ?? 'infer',
-		});
-		const allFiles = await listGitFiles(repoUrl, commitHash);
+		const commitHash = await resolveCommitHash(
+			repoUrl,
+			{
+				value: this.reference.ref,
+				type: this.reference.refType ?? 'infer',
+			},
+			this.options?.additionalHeaders
+		);
+		const allFiles = await listGitFiles(
+			repoUrl,
+			commitHash,
+			this.options?.additionalHeaders
+		);
 
 		const requestedPath = (this.reference.path ?? '').replace(/^\/+/, '');
 		const filesToClone = listDescendantFiles(allFiles, requestedPath);
@@ -588,6 +605,7 @@ export class GitDirectoryResource extends Resource<Directory> {
 			filesToClone,
 			{
 				withObjects: this.reference['.git'],
+				additionalHeaders: this.options?.additionalHeaders,
 			}
 		);
 		let files = checkout.files;

packages/playground/client/src/blueprints-v1-handler.ts

@@ -21,6 +21,7 @@ export class BlueprintsV1Handler {
 			onBlueprintValidated,
 			onBlueprintStepCompleted,
 			corsProxy,
+			gitAdditionalHeaders,
 			mounts,
 			sapiName,
 			scope,
@@ -72,6 +73,7 @@ export class BlueprintsV1Handler {
 				onStepCompleted: onBlueprintStepCompleted,
 				onBlueprintValidated,
 				corsProxy,
+				gitAdditionalHeaders,
 			});
 			await runBlueprintV1Steps(compiled, playground);
 		}

packages/playground/client/src/index.ts

@@ -85,6 +85,10 @@ export interface StartPlaygroundOptions {
 	 * your Blueprint to replace all cross-origin URLs with the proxy URL.
 	 */
 	corsProxy?: string;
+	/**
+	 * Additional headers to pass to git operations.
+	 */
+	gitAdditionalHeaders?: Record<string, string>;
 	/**
 	 * The version of the SQLite driver to use.
 	 * Defaults to the latest development version.

packages/playground/storage/src/lib/git-sparse-checkout.ts

@@ -30,6 +30,18 @@ if (typeof globalThis.Buffer === 'undefined') {
 	globalThis.Buffer = BufferPolyfill;
 }
 
+/**
+ * Custom error class for git authentication failures.
+ */
+export class GitAuthenticationError extends Error {
+	constructor(public repoUrl: string, public status: number) {
+		super(
+			`Authentication required to access private repository: ${repoUrl}`
+		);
+		this.name = 'GitAuthenticationError';
+	}
+}
+
 /**
  * Downloads specific files from a git repository.
  * It uses the git protocol over HTTP to fetch the files. It only uses
@@ -67,14 +79,21 @@ export async function sparseCheckout(
 	filesPaths: string[],
 	options?: {
 		withObjects?: boolean;
+		additionalHeaders?: Record<string, string>;
 	}
 ): Promise<SparseCheckoutResult> {
-	const treesPack = await fetchWithoutBlobs(repoUrl, commitHash);
+	const treesPack = await fetchWithoutBlobs(
+		repoUrl,
+		commitHash,
+		options?.additionalHeaders
+	);
 	const objects = await resolveObjects(treesPack.idx, commitHash, filesPaths);
 
 	const blobOids = filesPaths.map((path) => objects[path].oid);
 	const blobsPack =
-		blobOids.length > 0 ? await fetchObjects(repoUrl, blobOids) : null;
+		blobOids.length > 0
+			? await fetchObjects(repoUrl, blobOids, options?.additionalHeaders)
+			: null;
 
 	const fetchedPaths: Record<string, any> = {};
 	await Promise.all(
@@ -177,9 +196,14 @@ const FULL_SHA_REGEX = /^[0-9a-f]{40}$/i;
  */
 export async function listGitFiles(
 	repoUrl: string,
-	commitHash: string
+	commitHash: string,
+	additionalHeaders?: Record<string, string>
 ): Promise<GitFileTree[]> {
-	const treesPack = await fetchWithoutBlobs(repoUrl, commitHash);
+	const treesPack = await fetchWithoutBlobs(
+		repoUrl,
+		commitHash,
+		additionalHeaders
+	);
 	const rootTree = await resolveAllObjects(treesPack.idx, commitHash);
 	if (!rootTree?.object) {
 		return [];
@@ -195,13 +219,17 @@ export async function listGitFiles(
  * @param ref The branch name or commit hash.
  * @returns The commit hash.
  */
-export async function resolveCommitHash(repoUrl: string, ref: GitRef) {
+export async function resolveCommitHash(
+	repoUrl: string,
+	ref: GitRef,
+	additionalHeaders?: Record<string, string>
+) {
 	const parsed = await parseGitRef(repoUrl, ref);
 	if (parsed.resolvedOid) {
 		return parsed.resolvedOid;
 	}
 
-	const oid = await fetchRefOid(repoUrl, parsed.refname);
+	const oid = await fetchRefOid(repoUrl, parsed.refname, additionalHeaders);
 	if (!oid) {
 		throw new Error(`Git ref "${parsed.refname}" not found at ${repoUrl}`);
 	}
@@ -239,7 +267,8 @@ function gitTreeToFileTree(tree: GitTree): GitFileTree[] {
  */
 export async function listGitRefs(
 	repoUrl: string,
-	fullyQualifiedBranchPrefix: string
+	fullyQualifiedBranchPrefix: string,
+	additionalHeaders?: Record<string, string>
 ) {
 	const packbuffer = Buffer.from(
 		(await collect([
@@ -260,10 +289,20 @@ export async function listGitRefs(
 			'content-type': 'application/x-git-upload-pack-request',
 			'Content-Length': `${packbuffer.length}`,
 			'Git-Protocol': 'version=2',
+			...additionalHeaders,
 		},
 		body: packbuffer as any,
 	});
 
+	if (!response.ok) {
+		if (response.status === 401 || response.status === 403) {
+			throw new GitAuthenticationError(repoUrl, response.status);
+		}
+		throw new Error(
+			`Failed to fetch git refs from ${repoUrl}: ${response.status} ${response.statusText}`
+		);
+	}
+
 	const refs: Record<string, string> = {};
 	for await (const line of parseGitResponseLines(response)) {
 		const spaceAt = line.indexOf(' ');
@@ -376,8 +415,12 @@ async function parseGitRef(
 	}
 }
 
-async function fetchRefOid(repoUrl: string, refname: string) {
-	const refs = await listGitRefs(repoUrl, refname);
+async function fetchRefOid(
+	repoUrl: string,
+	refname: string,
+	additionalHeaders?: Record<string, string>
+) {
+	const refs = await listGitRefs(repoUrl, refname, additionalHeaders);
 	const candidates = [refname, `${refname}^{}`];
 	for (const candidate of candidates) {
 		const sanitized = candidate.trim();
@@ -388,7 +431,11 @@ async function fetchRefOid(repoUrl: string, refname: string) {
 	return null;
 }
 
-async function fetchWithoutBlobs(repoUrl: string, commitHash: string) {
+async function fetchWithoutBlobs(
+	repoUrl: string,
+	commitHash: string,
+	additionalHeaders?: Record<string, string>
+) {
 	const packbuffer = Buffer.from(
 		(await collect([
 			GitPktLine.encode(
@@ -409,10 +456,20 @@ async function fetchWithoutBlobs(repoUrl: string, commitHash: string) {
 			Accept: 'application/x-git-upload-pack-advertisement',
 			'content-type': 'application/x-git-upload-pack-request',
 			'Content-Length': `${packbuffer.length}`,
+			...additionalHeaders,
 		},
 		body: packbuffer as any,
 	});
 
+	if (!response.ok) {
+		if (response.status === 401 || response.status === 403) {
+			throw new GitAuthenticationError(repoUrl, response.status);
+		}
+		throw new Error(
+			`Failed to fetch git objects from ${repoUrl}: ${response.status} ${response.statusText}`
+		);
+	}
+
 	const iterator = streamToIterator(response.body!);
 	const parsed = await parseUploadPackResponse(iterator);
 	const packfile = Buffer.from((await collect(parsed.packfile)) as any);
@@ -539,7 +596,11 @@ async function resolveObjects(
 }
 
 // Request oid for each resolvedRef
-async function fetchObjects(url: string, objectHashes: string[]) {
+async function fetchObjects(
+	url: string,
+	objectHashes: string[],
+	additionalHeaders?: Record<string, string>
+) {
 	const packbuffer = Buffer.from(
 		(await collect([
 			...objectHashes.map((objectHash) =>
@@ -558,10 +619,20 @@ async function fetchObjects(url: string, objectHashes: string[]) {
 			Accept: 'application/x-git-upload-pack-advertisement',
 			'content-type': 'application/x-git-upload-pack-request',
 			'Content-Length': `${packbuffer.length}`,
+			...additionalHeaders,
 		},
 		body: packbuffer as any,
 	});
 
+	if (!response.ok) {
+		if (response.status === 401 || response.status === 403) {
+			throw new GitAuthenticationError(url, response.status);
+		}
+		throw new Error(
+			`Failed to fetch git objects from ${url}: ${response.status} ${response.statusText}`
+		);
+	}
+
 	const iterator = streamToIterator(response.body!);
 	const parsed = await parseUploadPackResponse(iterator);
 	const packfile = Buffer.from((await collect(parsed.packfile)) as any);