Skip to content

Commit 4bd7e53

Browse files
shhdgitshhdgit
committed
Refactor HTTP module and enhance variable handling in download utilities. Improved error handling and added support for variables from zip files.
1 parent 49a339c commit 4bd7e53

File tree

3 files changed

+126
-59
lines changed

3 files changed

+126
-59
lines changed

‎packages/download/http.js‎

Lines changed: 51 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
1-
import axios from 'axios'
2-
import dotenv from 'dotenv'
3-
import nPath, { resolve } from 'path'
4-
import sig from 'signale'
5-
import fs from 'fs'
1+
import axios from "axios";
2+
import dotenv from "dotenv";
3+
import nPath, { resolve } from "path";
4+
import sig from "signale";
5+
import fs from "fs";
66

7-
dotenv.config()
7+
dotenv.config();
88

9-
const GITHUB_AUTHORIZATION_TOKEN = process.env.GITHUB_AUTHORIZATION_TOKEN
10-
const baseURL = 'https://api.github.com'
9+
const GITHUB_AUTHORIZATION_TOKEN = process.env.GITHUB_AUTHORIZATION_TOKEN;
10+
const baseURL = "https://api.github.com";
1111
const defaultHeaders = {
12-
Accept: 'application/vnd.github.v3+json',
13-
}
12+
Accept: "application/vnd.github.v3+json",
13+
};
1414

1515
export const http = axios.create({
1616
baseURL,
@@ -20,67 +20,73 @@ export const http = axios.create({
2020
Authorization: `token ${GITHUB_AUTHORIZATION_TOKEN}`,
2121
}
2222
: defaultHeaders,
23-
})
23+
});
2424

2525
export function getContent(repo, ref, path) {
26-
const url = nPath.join(`/repos/${repo}/contents`, path)
26+
const url = nPath.join(`/repos/${repo}/contents`, path);
2727

28-
sig.start(`Get content(ref: ${ref}) from:`, url)
28+
sig.start(`Get content(ref: ${ref}) from:`, url);
2929

3030
return http.get(url, {
3131
params: {
3232
ref,
3333
},
34-
})
34+
});
3535
}
3636

3737
export function compare(repo, base, head) {
38-
const url = `/repos/${repo}/compare/${base}...${head}`
38+
const url = `/repos/${repo}/compare/${base}...${head}`;
3939

40-
sig.info(`Compare: ${base}...${head}`)
40+
sig.info(`Compare: ${base}...${head}`);
4141

42-
return http.get(url)
42+
return http.get(url);
4343
}
4444

45-
export function downloadFile(reqUrl, fileName = '') {
45+
export function downloadFile(reqUrl, fileName = "") {
4646
return axios({
47-
method: 'GET',
47+
method: "GET",
4848
url: reqUrl,
49-
responseType: 'stream',
49+
responseType: "stream",
5050
headers: GITHUB_AUTHORIZATION_TOKEN
5151
? {
5252
...defaultHeaders,
5353
Authorization: `token ${GITHUB_AUTHORIZATION_TOKEN}`,
5454
}
5555
: defaultHeaders,
5656
})
57-
.then(res => {
57+
.then((res) => {
5858
if (res.status == 200) {
59-
fileName = fileName || reqUrl.split('/').pop()
60-
const dir = resolve(fileName)
61-
sig.start(`Download file(fileName: ${fileName}) reqUrl:${reqUrl}`)
62-
res.data.pipe(fs.createWriteStream(dir))
63-
// res.data.on('end', () => {
64-
// sig.success('download completed')
65-
// })
59+
fileName = fileName || reqUrl.split("/").pop();
60+
const dir = resolve(fileName);
61+
sig.start(`Download file(fileName: ${fileName}) reqUrl:${reqUrl}`);
62+
63+
const writeStream = fs.createWriteStream(dir);
64+
65+
res.data.pipe(writeStream);
66+
6667
return new Promise((resolve, reject) => {
67-
res.data.on('end', () => {
68-
sig.success('download completed')
69-
resolve()
70-
})
71-
72-
res.data.on('error', (err) => {
73-
sig.error('Failed to save file: ', err)
74-
reject()
75-
})
76-
})
68+
writeStream.on("finish", () => {
69+
sig.success("download completed");
70+
resolve();
71+
});
72+
73+
writeStream.on("error", (err) => {
74+
sig.error("Failed to save file: ", err);
75+
reject(err);
76+
});
77+
78+
res.data.on("error", (err) => {
79+
sig.error("Failed to download data: ", err);
80+
reject(err);
81+
});
82+
});
7783
} else {
78-
sig.error(`ERROR >> ${res.status}`)
84+
sig.error(`ERROR >> ${res.status}`);
7985
}
8086
})
81-
.catch(err => {
82-
sig.error('Error ', err)
83-
})
87+
.catch((err) => {
88+
sig.error("Error ", err);
89+
});
8490
}
8591

8692
/**
@@ -91,7 +97,7 @@ export function downloadFile(reqUrl, fileName = '') {
9197
* @returns Promise
9298
*/
9399
export function getArchiveFile(repo, ref, fileName) {
94-
const url = `https://api.github.com/repos/${repo}/zipball/${ref}`
95-
sig.start(`Get content(ref: ${ref}) from:`, url)
96-
return downloadFile(url, fileName)
100+
const url = `https://api.github.com/repos/${repo}/zipball/${ref}`;
101+
sig.start(`Get content(ref: ${ref}) from:`, url);
102+
return downloadFile(url, fileName);
97103
}

‎packages/download/utils.js‎

Lines changed: 42 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import { gfm } from "micromark-extension-gfm";
1313
import { mdxFromMarkdown } from "mdast-util-mdx";
1414
import { gfmFromMarkdown } from "mdast-util-gfm";
1515
import { visit } from "unist-util-visit";
16-
16+
import { getVariablesFromZip, variablesReplaceStream } from "./variable.js";
1717
const IMAGE_CDN_PREFIX = "https://docs-download.pingcap.com/media/images";
1818
export const imageCDNs = {
1919
docs: IMAGE_CDN_PREFIX + "/docs",
@@ -36,10 +36,38 @@ export const imageCDNs = {
3636
* @param {string[]} [options.ignore] - Specify the files to be ignored
3737
* @param {Array} [options.pipelines]
3838
*/
39-
export async function retrieveAllMDs(metaInfo, destDir, options) {
39+
export async function retrieveAllMDs(
40+
metaInfo,
41+
destDir,
42+
options,
43+
defaultVariables
44+
) {
4045
const { repo, ref, path = "" } = metaInfo;
4146
const { ignore = [], pipelines = [] } = options;
4247

48+
// Get variables.json from root directory
49+
let variables = defaultVariables;
50+
if (!defaultVariables) {
51+
try {
52+
const variablesResponse = await getContent(repo, ref, "variables.json");
53+
if (variablesResponse.data && variablesResponse.data.content) {
54+
const content = Buffer.from(
55+
variablesResponse.data.content,
56+
"base64"
57+
).toString();
58+
variables = JSON.parse(content);
59+
}
60+
} catch (error) {
61+
sig.warn(
62+
"Failed to get variables.json from root directory:",
63+
error.message
64+
);
65+
}
66+
}
67+
68+
// Add variablesReplaceStream to pipelines
69+
const ppls = [...pipelines, variablesReplaceStream(variables)];
70+
4371
const data = (await getContent(repo, ref, path)).data;
4472

4573
if (Array.isArray(data)) {
@@ -59,11 +87,12 @@ export async function retrieveAllMDs(metaInfo, destDir, options) {
5987
path: `${path}/${name}`,
6088
},
6189
nextDest,
62-
options
90+
options,
91+
variables
6392
);
6493
} else {
6594
if (name.endsWith(".md")) {
66-
writeContent(download_url, nextDest, pipelines);
95+
writeContent(download_url, nextDest, ppls);
6796
}
6897
}
6998
});
@@ -72,7 +101,7 @@ export async function retrieveAllMDs(metaInfo, destDir, options) {
72101
writeContent(
73102
data.download_url,
74103
destDir.endsWith(".md") ? destDir : `${destDir}/${data.name}`,
75-
pipelines
104+
ppls
76105
);
77106
}
78107
}
@@ -179,6 +208,8 @@ export async function retrieveTiDBMDsFromZip(
179208
// Unzip archive
180209
const zip = new AdmZip(archiveFileName);
181210
const zipEntries = zip.getEntries();
211+
const variables = getVariablesFromZip(zip, "/variables.json");
212+
const ppls = [...pipelines, variablesReplaceStream(variables)];
182213

183214
zipEntries.forEach(function (zipEntry) {
184215
// console.log(zipEntry.toString()) // outputs zip entries information
@@ -208,7 +239,7 @@ export async function retrieveTiDBMDsFromZip(
208239
writeFile(
209240
`${destDir}/${relativePathNameList.join("/")}`,
210241
zipEntry.getData(),
211-
pipelines
242+
ppls
212243
);
213244
});
214245
} catch (error) {
@@ -222,8 +253,6 @@ export async function retrieveTiDBMDsFromZip(
222253
}
223254
}
224255

225-
const CONST_FILE_LIST = ["/_docHome.md"];
226-
227256
export async function retrieveCloudMDsFromZip(
228257
metaInfo,
229258
destDir,
@@ -243,15 +272,14 @@ export async function retrieveCloudMDsFromZip(
243272
// Unzip archive
244273
const zip = new AdmZip(archiveFileName);
245274
const zipEntries = zip.getEntries();
275+
const variables = getVariablesFromZip(zip, "/variables.json");
276+
const ppls = [...pipelines, variablesReplaceStream(variables)];
246277

247278
const cloudTocZipEntry = zipEntries.find((entry) =>
248279
entry.entryName.endsWith(`/TOC-tidb-cloud.md`)
249280
);
250281

251-
const cloudFileList = [
252-
...CONST_FILE_LIST,
253-
...getFileListFromToc(cloudTocZipEntry.getData()),
254-
];
282+
const cloudFileList = getFileListFromToc(cloudTocZipEntry.getData());
255283

256284
// console.log(cloudFileList);
257285

@@ -284,12 +312,12 @@ export async function retrieveCloudMDsFromZip(
284312
return;
285313
}
286314
if (relativePathInZip === `TOC-tidb-cloud.md`) {
287-
writeFile(`${destDir}/TOC.md`, zipEntry.getData(), pipelines);
315+
writeFile(`${destDir}/TOC.md`, zipEntry.getData(), ppls);
288316
} else {
289317
writeFile(
290318
`${destDir}/${relativePathNameList.join("/")}`,
291319
zipEntry.getData(),
292-
pipelines
320+
ppls
293321
);
294322
}
295323
});

‎packages/download/variable.js‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import replaceStream from "replacestream";
2+
3+
export const getVariablesFromZip = (zip, filePath) => {
4+
let variables = {};
5+
6+
try {
7+
const zipEntries = zip.getEntries();
8+
const variablesEntry = zipEntries.find((entry) =>
9+
entry.entryName.endsWith(filePath)
10+
);
11+
variables = JSON.parse(variablesEntry.getData().toString());
12+
} catch (error) {
13+
console.error("Read variables error: ", error);
14+
}
15+
16+
return variables;
17+
};
18+
19+
function getValueByPath(obj, path) {
20+
return path.split(".").reduce((acc, key) => (acc ? acc[key] : ""), obj) ?? "";
21+
}
22+
const variablePattern = /{{{\s*\.(.+?)\s*}}}/g;
23+
24+
export const variablesReplaceStream = (variables) => {
25+
return () =>
26+
replaceStream(variablePattern, (match, path) => {
27+
const value = getValueByPath(variables, path.trim());
28+
if (value) {
29+
return String(value);
30+
}
31+
return match;
32+
});
33+
};

0 commit comments

Comments
 (0)