Skip to content

Commit 433182c

Browse files
mammad2cmammad2c
committed
changed download mechanism
1 parent 9096046 commit 433182c

File tree

6 files changed

+185
-148
lines changed

6 files changed

+185
-148
lines changed

‎.eslintrc.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,6 @@ module.exports = {
1010
},
1111
rules: {
1212
"no-console": 0,
13+
"no-await-in-loop": 0,
1314
},
1415
};

‎lib/getRepos.js‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/* eslint-disable no-await-in-loop */
2+
const chalk = require("chalk");
3+
const fs = require("fs-extra");
4+
const shell = require("shelljs");
5+
const Utils = require("./utils");
6+
7+
/**
8+
* this function handle cloning repositories.
9+
* @param {array} repos repositories required for clone.
10+
* @param {object} utils utils instance.
11+
* @param {object} bar cli progress bar instance.
12+
* @param {object} scriptOptions pass cli arguments.
13+
*/
14+
const getRrepos = async (repos = [], utils, bar, scriptOptions = {}) => {
15+
/**
16+
* handle if there is no repository exists on given gitlab url.
17+
*/
18+
if (repos.length === 0) {
19+
console.log(
20+
chalk.yellow(
21+
`\nYou don't have any repositories on ${scriptOptions.baseUrl} \n`
22+
)
23+
);
24+
process.exit(0);
25+
}
26+
27+
console.log(chalk.green("\nStart cloning ... \n"));
28+
29+
if (!fs.existsSync(scriptOptions.output)) {
30+
fs.mkdirSync(scriptOptions.output);
31+
}
32+
33+
const notClonedRepos = repos.filter((item) => {
34+
const repoName = Utils.generateRepoName(item.name_with_namespace);
35+
return !utils.isRepoExist(repoName);
36+
});
37+
38+
bar.start(repos.length, 0);
39+
bar.update(repos.length - notClonedRepos.length);
40+
41+
for (let i = 0; i < notClonedRepos.length; i += 1) {
42+
const repo = notClonedRepos[i];
43+
const {
44+
name_with_namespace: nameWithNameSpace,
45+
http_url_to_repo: httpUrlToRepo,
46+
} = repo;
47+
48+
const repoName = Utils.generateRepoName(nameWithNameSpace);
49+
const repoNameColor = Utils.generateRepoNameColorized(repoName);
50+
const repoUrl = utils.generateRepoUrl(httpUrlToRepo);
51+
const command = `git clone ${repoUrl} "./repos/${repoName}" --progress`;
52+
53+
console.log(chalk.yellow(`\ncloning ${repoNameColor} \n`));
54+
55+
await new Promise((resolve, reject) => {
56+
shell.exec(command, (code, stdout, stderr) => {
57+
if (code === 0) {
58+
utils.cloneCompleted(repoName);
59+
resolve();
60+
} else {
61+
reject(new Error(stderr));
62+
}
63+
});
64+
});
65+
}
66+
67+
return true;
68+
};
69+
70+
module.exports = getRrepos;

‎lib/index.js‎

Lines changed: 48 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
#!/usr/bin/env node
22

3-
const clone = require("git-clone");
43
const axios = require("axios").default;
5-
const fs = require("fs-extra");
64
const chalk = require("chalk");
7-
const objChange = require("on-change");
85
const cliProgress = require("cli-progress");
96
const argv = require("yargs");
7+
const shell = require("shelljs");
108
const Utils = require("./utils");
9+
const getRepos = require("./getRepos");
1110

1211
/**
1312
* define script options.
@@ -33,79 +32,61 @@ argv
3332

3433
if (!argv.argv.token) {
3534
console.log(
36-
chalk.bgRed(
37-
"\n Please pass your gitlab personal access token. Check README.md to how grab it. \n"
35+
chalk.red(
36+
"\nPlease pass your gitlab personal access token. Check README.md to how grab it. \n"
3837
)
3938
);
4039
process.exit(0);
4140
}
4241

42+
/**
43+
* check for git commands exist.
44+
*/
45+
if (!shell.which("git")) {
46+
shell.echo(`${chalk.red("Sorry, this script requires git")}`);
47+
shell.exit(1);
48+
}
49+
4350
/**
4451
* defining our needs.
4552
*/
46-
const token = argv.argv.token || "";
47-
const baseUrl = argv.argv.url || "https://gitlab.com";
48-
const output = argv.argv.output || "./repos";
4953
const pagination = 100; // currently maximum gitlab pagination supports.
5054
const defaultAddress = `/api/v4/projects?simple=true&membership=true&pagination=keyset&order_by=id&sort=asc&per_page=${pagination}`;
5155
const bar = new cliProgress.SingleBar(
5256
{
53-
format: "progress [{bar}] {percentage}% | {value}/{total}",
57+
hideCursor: true,
58+
format:
59+
"\nprogress [{bar}] {percentage}% | {value}/{total} | this may take several minutes \n",
5460
},
5561
cliProgress.Presets.shades_classic
5662
);
5763

5864
/**
59-
* for saving next paginated data url.
65+
* create an object for storing arguments and use it every where.
6066
*/
61-
let next;
67+
const scriptOptions = {
68+
token: argv.argv.token || "",
69+
baseUrl: argv.argv.url || "https://gitlab.com",
70+
output: argv.argv.output || "./repos",
71+
};
6272

6373
/**
64-
* total repository for handling progress bar.
74+
* for saving next paginated data url.
6575
*/
66-
let total = 0;
76+
let next;
6777

6878
/**
69-
* to indicate when first cloning starts.
79+
* total repository for handling progress bar.
7080
*/
71-
let firstStart = true;
72-
73-
axios.defaults.baseURL = baseUrl;
74-
axios.defaults.headers.common["PRIVATE-TOKEN"] = token;
75-
76-
if (!fs.existsSync(output)) {
77-
fs.mkdirSync(output);
78-
}
81+
const totalRepos = [];
7982

80-
/**
81-
* listen for changes on cloned repos and we reached end of pagination or not.
82-
*/
83-
const observer = objChange(
84-
{
85-
cloned: 0,
86-
hasNext: true,
87-
},
88-
() => {
89-
/**
90-
* if we cloned all repos based on total and cloned and also there is no link for next paginated data, so we cloned all repos and should stop progress bar and cleanup message.
91-
*/
92-
if (observer.cloned === total && !observer.hasNext) {
93-
bar.stop();
94-
console.log(chalk.green(`\n ${total} repo(s) saved!! \n`));
95-
}
96-
}
97-
);
83+
axios.defaults.baseURL = scriptOptions.baseUrl;
84+
axios.defaults.headers.common["PRIVATE-TOKEN"] = scriptOptions.token;
9885

9986
/**
10087
* generate our uitls.
10188
*/
102-
const utils = new Utils({
103-
token,
104-
url: baseUrl,
105-
output,
106-
bar,
107-
observer,
108-
});
89+
const utils = new Utils(bar, scriptOptions);
10990

11091
/**
11192
* main function:
@@ -119,59 +100,15 @@ function main() {
119100
.then((res) => {
120101
const repos = res.data;
121102
const { length } = repos;
122-
total += length;
123103

124104
/**
125105
* based on gitlab pagination document, the next url to be called come here, so we grab it and save it
126106
* to handle our pagination process.
127107
*/
128108
const { link } = res.headers;
129109

130-
if (firstStart) {
131-
/**
132-
* so at the beginning of cloning we start our cli progress.
133-
*/
134-
bar.start(total, 0);
135-
136-
firstStart = false;
137-
} else {
138-
/**
139-
* we update our total repos because of in new paginated data we should update total repos.
140-
*/
141-
bar.setTotal(total);
142-
}
143-
144110
for (let i = 0; i < length; i += 1) {
145-
const repo = repos[i];
146-
const {
147-
name_with_namespace: nameWithNameSpace,
148-
http_url_to_repo: httpUrlToRepo,
149-
} = repo;
150-
151-
const repoName = Utils.generateRepoName(nameWithNameSpace);
152-
const repoNameColor = Utils.generateRepoNameColorized(repoName);
153-
const repoUrl = utils.generateRepoUrl(httpUrlToRepo);
154-
155-
/**
156-
* if we had already cloned this repo, we do action after cloning a repo without showing any message to update our process.
157-
*/
158-
if (utils.isRepoExist(repoName)) {
159-
utils.cloneCompleted(repoNameColor, false);
160-
} else {
161-
/**
162-
* we start cloning the repo
163-
*/
164-
console.log(chalk.yellow(`\n cloning ${repoNameColor} ... \n`));
165-
166-
clone(`${repoUrl}`, `${output}/${repoName}`, undefined, () => {
167-
/**
168-
* here again, check if clone completed, do action after completing clone.
169-
*/
170-
if (utils.isRepoExist(repoName)) {
171-
utils.cloneCompleted(repoNameColor);
172-
}
173-
});
174-
}
111+
totalRepos.push(repos[i]);
175112
}
176113

177114
/**
@@ -187,13 +124,29 @@ function main() {
187124
main();
188125
} else {
189126
/**
190-
* if we don't have any other repos, so we stop our pagination process.
127+
* if we don't have any other repos url, so we stop our pagination process and
128+
* start cloning the repositories.
191129
*/
192-
observer.hasNext = false;
130+
getRepos(totalRepos, utils, bar, scriptOptions)
131+
.then(() => {
132+
bar.stop();
133+
console.clear();
134+
console.log(
135+
chalk.green(
136+
`\n \n \n ${totalRepos.length} repo(s) saved!! \n \n \n`
137+
)
138+
);
139+
process.exit(0);
140+
})
141+
.catch((err) => {
142+
console.log(`\n ${chalk.red(err.message)}`);
143+
process.exit(0);
144+
});
193145
}
194146
})
195147
.catch((err) => {
196-
console.log(`\n ${chalk.red(`trace: ${err.stack}`)} \n`);
148+
console.log(`\n${chalk.red(err.message)} \n`);
149+
console.log(`\n${chalk.red(`trace: ${err.stack}`)} \n`);
197150
process.exit(0);
198151
});
199152
}

‎lib/utils.js‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ class Utils {
99
* TODO: this should be configurable by passing an option.
1010
*/
1111
static generateRepoName(name) {
12-
return name.replace(/\//g, "-");
12+
return name.replace(/\//g, "-").replace(/\s+/g, "");
1313
}
1414

1515
/**
@@ -29,13 +29,12 @@ class Utils {
2929
return link.replace("<http", "http").replace(`>; rel="next"`, "");
3030
}
3131

32-
constructor(scriptState) {
33-
const { token, url, output, bar, observer } = scriptState;
32+
constructor(bar, scriptOptions) {
33+
const { token, url, output } = scriptOptions;
3434
this.token = token;
3535
this.url = url;
3636
this.output = output;
3737
this.bar = bar;
38-
this.observer = observer;
3938
}
4039

4140
/**
@@ -53,14 +52,13 @@ class Utils {
5352
*/
5453
cloneCompleted(repoName, showMessage = true) {
5554
if (showMessage) {
56-
console.log(chalk.green(`\n clone completed ${repoName} \n`));
55+
console.log(
56+
`\n${chalk.green(
57+
`successfully cloned`
58+
)} ${Utils.generateRepoNameColorized(repoName)} \n`
59+
);
5760
}
5861

59-
/**
60-
* we increase cloned repo to keep tracking how much cloned.
61-
*/
62-
this.observer.cloned += 1;
63-
6462
/**
6563
* increase cli progress bar by 1 when a repo cloned.
6664
*/

0 commit comments

Comments
 (0)