Skip to content

Commit d54a856

Browse files
mrgrainmrgrain
authored
feat(golang): optimize git operations for large repositories (#1725)
feat(golang): add configurable git clone depth support - Add GIT_CLONE_DEPTH environment variable to control clone depth - Refactor git.clone() to accept depth parameter with default of 1 - Add git.rm() helper function for proper file removal - Update GoReleaser to use git.rm() instead of fs.removeSync() - Add comprehensive test coverage for new functionality
1 parent 4a2bd5d commit d54a856

File tree

7 files changed

+149
-16
lines changed

7 files changed

+149
-16
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,7 @@ Repository tags will be in the following format:
274274
| `GIT_USER_NAME` | Optional | Username to perform the commit with. Defaults to the git user.name config in the current directory. Fails if it doesn't exist. |
275275
| `GIT_USER_EMAIL` | Optional | Email to perform the commit with. Defaults to the git user.email config in the current directory. Fails if it doesn't exist. |
276276
| `GIT_COMMIT_MESSAGE` | Optional | The commit message. Defaults to 'chore(release): $VERSION'. |
277+
| `GIT_CLONE_DEPTH` | Optional | The git clone depth. Usually only the latest commit is required. Defaults to 1. |
277278
| `DRYRUN` | Optional | Set to "true" for a dry run. |
278279

279280
## Publish to CodeArtifact for testing

‎src/bin/publib-golang.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ const releaser = new go.GoReleaser({
88
email: process.env.GIT_USER_EMAIL,
99
username: process.env.GIT_USER_NAME,
1010
version: process.env.VERSION,
11+
cloneDepth: process.env.GIT_CLONE_DEPTH ? parseInt(process.env.GIT_CLONE_DEPTH) || 1 : 1,
1112
});
1213

13-
releaser.release();
14+
releaser.release();

‎src/help/git.ts‎

Lines changed: 68 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,61 @@
11
import * as shell from './shell';
22

3+
export interface CloneOptions {
4+
/**
5+
* @default 1
6+
*/
7+
readonly depth?: number;
8+
9+
/**
10+
* @default false
11+
*/
12+
readonly tags?: boolean;
13+
14+
/**
15+
* @default - default branch
16+
*/
17+
readonly branch?: string;
18+
}
19+
320
/**
421
* Clones a repository from GitHub. Requires a `GITHUB_TOKEN` env variable.
522
*
623
* @param repositoryUrl the repository to clone.
724
* @param targetDir the clone directory.
825
*/
9-
export function clone(repositoryUrl: string, targetDir: string) {
26+
export function clone(repositoryUrl: string, targetDir: string, { depth = 1, tags = false, branch }: CloneOptions = {}) {
27+
const cmd = ['git', 'clone'];
28+
29+
if (depth) {
30+
cmd.push(`--depth ${depth}`);
31+
}
32+
33+
if (branch) {
34+
cmd.push(`--branch ${branch}`);
35+
}
36+
37+
if (tags) {
38+
cmd.push('--tags');
39+
}
40+
41+
cmd.push(tryDetectRepositoryUrl(repositoryUrl));
42+
cmd.push(targetDir);
43+
44+
shell.run(cmd.join(' '));
45+
}
46+
47+
function tryDetectRepositoryUrl(repositoryUrl: string): string {
1048
const gitHubUseSsh = detectSSH();
1149
if (gitHubUseSsh) {
1250
const sshRepositoryUrl = repositoryUrl.replace('/', ':');
13-
shell.run(`git clone git@${sshRepositoryUrl}.git ${targetDir}`);
14-
} else {
15-
const gitHubToken = getToken(detectGHE());
16-
if (!gitHubToken) {
17-
throw new Error('GITHUB_TOKEN env variable is required when GITHUB_USE_SSH env variable is not used');
18-
}
19-
shell.run(`git clone https://${gitHubToken}@${repositoryUrl}.git ${targetDir}`);
51+
return `git@${sshRepositoryUrl}.git`;
52+
}
2053

54+
const gitHubToken = getToken(detectGHE());
55+
if (!gitHubToken) {
56+
throw new Error('GITHUB_TOKEN env variable is required when GITHUB_USE_SSH env variable is not used');
2157
}
58+
return `https://${gitHubToken}@${repositoryUrl}.git`;
2259
}
2360

2461
/**
@@ -84,6 +121,22 @@ export function add(p: string) {
84121
shell.run(`git add ${p}`);
85122
}
86123

124+
/**
125+
* Remove files from the working tree and from the index
126+
*
127+
* @param p the path.
128+
*/
129+
export function rm(p: string, options: { recursive?: boolean } = {}) {
130+
const cmd = ['git', 'rm'];
131+
if (options.recursive) {
132+
cmd.push('-r');
133+
}
134+
135+
cmd.push(p);
136+
137+
shell.run(cmd.join(' '));
138+
}
139+
87140
/**
88141
* Commit.
89142
*
@@ -188,3 +241,10 @@ export function identify(user: string, address: string) {
188241
shell.run(`git config user.name "${user}"`);
189242
shell.run(`git config user.email "${address}"`);
190243
}
244+
245+
/**
246+
* Does the given branch exists on the remote.
247+
*/
248+
export function branchExistsOnRemote(repositoryUrl: string, branch: string): boolean {
249+
return shell.check(`git ls-remote --exit-code --heads ${tryDetectRepositoryUrl(repositoryUrl)} ${branch}`, { capture: true });
250+
}

‎src/help/shell.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,22 @@ export function run(command: string, options: RunOptions = {}): string {
6666
return stdout;
6767
}
6868

69+
/**
70+
* Run a shell command and return a boolean indicating success or failure.
71+
*
72+
* Use this is when the result of the command informs a decision but is otherwise inconsequential.
73+
*
74+
* @param command command (e.g 'git ls-remote --exit-code')
75+
*/
76+
export function check(command: string, options: RunOptions = {}): boolean {
77+
try {
78+
run(command, options);
79+
return true;
80+
} catch (e) {
81+
return false;
82+
}
83+
}
84+
6985
/**
7086
* Return the path under which a program is available.
7187
* Empty string if the program is not installed.

‎src/targets/go.ts‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,15 @@ export interface GoReleaserProps {
7676
* The message to use for the commit marking the release.
7777
*/
7878
readonly message?: string;
79+
80+
/**
81+
* The git clone depth.
82+
*
83+
* Usually only the latest commit is required.
84+
*
85+
* @default 1
86+
*/
87+
readonly cloneDepth?: number;
7988
}
8089

8190
/**
@@ -168,7 +177,17 @@ export class GoReleaser {
168177

169178
const repoURL = this.extractRepoURL(modules);
170179
const repoDir = path.join(os.mkdtempSync(), 'repo');
171-
git.clone(repoURL, repoDir);
180+
181+
const branchExists = git.branchExistsOnRemote(repoURL, this.gitBranch);
182+
if (!branchExists) {
183+
console.log(`Remote branch '${this.gitBranch}' not found, continuing with default branch.`);
184+
}
185+
const cloneOptions = {
186+
tags: true, // we need to know about all tags to not re-create an existing one
187+
branch: branchExists ? this.gitBranch : undefined,
188+
};
189+
190+
git.clone(repoURL, repoDir, cloneOptions);
172191

173192
const cwd = process.cwd();
174193
try {
@@ -308,13 +327,13 @@ export class GoReleaser {
308327
// so we just empty it out
309328
fs.readdirSync(repoDir)
310329
.filter(f => f !== '.git')
311-
.forEach(f => fs.removeSync(path.join(repoDir, f)));
330+
.forEach(f => git.rm(path.join(repoDir, f), { recursive: true }));
312331
} else {
313332
// otherwise, we selectively remove the submodules only.
314333
for (const p of fs.readdirSync(repoDir)) {
315334
const submodule = path.join(repoDir, p, 'go.mod');
316335
if (fs.existsSync(submodule)) {
317-
fs.removeSync(path.join(repoDir, p));
336+
git.rm(path.join(repoDir, p), { recursive: true });
318337
}
319338
}
320339
}

‎test/targets/git-mocked.test.ts‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ test('clone with token', () => {
2424
git.clone('github.com/cdklabs/publib', 'target');
2525

2626
expect(mockedShellRun.mock.calls).toHaveLength(1);
27-
expect(mockedShellRun.mock.calls[0]).toEqual(['git clone https://[email protected]/cdklabs/publib.git target']);
27+
expect(mockedShellRun.mock.calls[0]).toEqual(['git clone --depth 1 https://[email protected]/cdklabs/publib.git target']);
2828
});
2929

3030
test('clone with ssh', () => {
@@ -33,7 +33,7 @@ test('clone with ssh', () => {
3333
git.clone('github.com/cdklabs/publib', 'target');
3434

3535
expect(mockedShellRun.mock.calls).toHaveLength(1);
36-
expect(mockedShellRun.mock.calls[0]).toEqual(['git clone [email protected]:cdklabs/publib.git target']);
36+
expect(mockedShellRun.mock.calls[0]).toEqual(['git clone --depth 1 [email protected]:cdklabs/publib.git target']);
3737
});
3838

3939
test('throw exception without token or ssh', () => {
@@ -57,7 +57,7 @@ test('clone with provided ghe authentication for github enterprise repo but no s
5757
process.env.GH_HOST = 'github.corporate-enterprise.com';
5858
git.clone('github.corporate-enterprise.com/cdklabs/publib', 'target');
5959
expect(mockedShellRun.mock.calls).toHaveLength(1);
60-
expect(mockedShellRun.mock.calls[0]).toEqual(['git clone https://[email protected]/cdklabs/publib.git target']);
60+
expect(mockedShellRun.mock.calls[0]).toEqual(['git clone --depth 1 https://[email protected]/cdklabs/publib.git target']);
6161
});
6262

6363
test('clone with provided ghe authentication for github enterprise repo and with non-public github api url', () => {
@@ -66,5 +66,36 @@ test('clone with provided ghe authentication for github enterprise repo and with
6666
process.env.GITHUB_API_URL = 'https://api.github.corporate-enterprise.com';
6767
git.clone('github.corporate-enterprise.com/cdklabs/publib', 'target');
6868
expect(mockedShellRun.mock.calls).toHaveLength(1);
69-
expect(mockedShellRun.mock.calls[0]).toEqual(['git clone https://[email protected]/cdklabs/publib.git target']);
69+
expect(mockedShellRun.mock.calls[0]).toEqual(['git clone --depth 1 https://[email protected]/cdklabs/publib.git target']);
70+
});
71+
72+
test('clone with depth option', () => {
73+
process.env.GITHUB_TOKEN = 'test-token';
74+
git.clone('github.com/owner/repo', 'target-dir', { depth: 5 });
75+
expect(mockedShellRun).toHaveBeenCalledWith('git clone --depth 5 https://[email protected]/owner/repo.git target-dir');
76+
delete process.env.GITHUB_TOKEN;
77+
});
78+
79+
test('clone with tags', () => {
80+
process.env.GITHUB_TOKEN = 'test-token';
81+
git.clone('github.com/owner/repo', 'target-dir', { tags: true });
82+
expect(mockedShellRun).toHaveBeenCalledWith('git clone --depth 1 --tags https://[email protected]/owner/repo.git target-dir');
83+
delete process.env.GITHUB_TOKEN;
84+
});
85+
86+
test('clone with branch', () => {
87+
process.env.GITHUB_TOKEN = 'test-token';
88+
git.clone('github.com/owner/repo', 'target-dir', { branch: 'foobar' });
89+
expect(mockedShellRun).toHaveBeenCalledWith('git clone --depth 1 --branch foobar https://[email protected]/owner/repo.git target-dir');
90+
delete process.env.GITHUB_TOKEN;
91+
});
92+
93+
test('rm without options', () => {
94+
git.rm('file.txt');
95+
expect(mockedShellRun).toHaveBeenCalledWith('git rm file.txt');
96+
});
97+
98+
test('rm with recursive option', () => {
99+
git.rm('directory', { recursive: true });
100+
expect(mockedShellRun).toHaveBeenCalledWith('git rm -r directory');
70101
});

‎test/targets/go.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@ function createReleaser(
6060
initRepo(targetDir, initializers.postInit);
6161
};
6262

63+
(git as any).branchExistsOnRemote = function() {
64+
// skip logic for comparing against remote since we don't have one
65+
return true;
66+
};
67+
6368
(git as any).checkout = function(branch: string, options: { createIfMissing?: boolean }) {
6469
// skip logic for comparing against remote since we don't have one
6570
if (options.createIfMissing) {

0 commit comments

Comments
 (0)