|
| 1 | +import fs from "fs"; |
| 2 | +import path from "path"; |
| 3 | +import { spawnSync } from "child_process"; |
| 4 | +import { globSync } from "glob"; |
| 5 | +import assert from 'assert/strict'; |
| 6 | +import { fileURLToPath } from "url"; |
| 7 | + |
| 8 | +const __dirname = path.dirname(fileURLToPath(import.meta.url)); |
| 9 | + |
| 10 | + |
| 11 | +class Importer { |
| 12 | + constructor({ projectName, size, repoUrl, srcFolder, extraFiles, extraDirs }) { |
| 13 | + this.projectName = projectName; |
| 14 | + assert(projectName.endsWith(`-${size}`), "missing size annotation in projectName"); |
| 15 | + this.repoUrl = repoUrl; |
| 16 | + this.baseDir = path.resolve(__dirname); |
| 17 | + let repoName = path.basename(this.repoUrl); |
| 18 | + if (!repoName.endsWith(".git")) { |
| 19 | + repoName = `${repoName}.git`; |
| 20 | + } |
| 21 | + this.repoDir = path.resolve(__dirname, repoName); |
| 22 | + this.srcFolder = srcFolder; |
| 23 | + this.outputDir = path.resolve(__dirname, `../src/gen/${this.projectName}`); |
| 24 | + fs.mkdirSync(this.outputDir, { recursive: true }); |
| 25 | + this.srcFileData = Object.create(null); |
| 26 | + this.extraFiles = extraFiles; |
| 27 | + this.extraDirs = extraDirs; |
| 28 | + } |
| 29 | + run() { |
| 30 | + this.cloneRepo(); |
| 31 | + this.readSrcFileData(); |
| 32 | + this.addExtraFilesFromDirs(); |
| 33 | + this.addSpecificFiles(); |
| 34 | + this.writeTsConfig(); |
| 35 | + this.writeSrcFileData(); |
| 36 | + } |
| 37 | + |
| 38 | + cloneRepo() { |
| 39 | + if (fs.existsSync(this.repoDir)) return; |
| 40 | + console.info(`Cloning src data repository to ${this.repoDir}`); |
| 41 | + spawnSync("git", ["clone", this.repoUrl, this.repoDir]); |
| 42 | + } |
| 43 | + |
| 44 | + readSrcFileData() { |
| 45 | + const patterns = [`${this.srcFolder}/**/*.ts`, `${this.srcFolder}/**/*.d.ts`, `${this.srcFolder}/*.d.ts`]; |
| 46 | + patterns.forEach(pattern => { |
| 47 | + const files = globSync(pattern, { cwd: this.repoDir, nodir: true }); |
| 48 | + files.forEach(file => { |
| 49 | + const filePath = path.join(this.repoDir, file); |
| 50 | + const relativePath = path.relative(this.repoDir, filePath).toLowerCase(); |
| 51 | + const fileContents = fs.readFileSync(filePath, "utf8"); |
| 52 | + this.addFileContents(relativePath, fileContents); |
| 53 | + }); |
| 54 | + }); |
| 55 | + } |
| 56 | + |
| 57 | + addExtraFilesFromDirs() { |
| 58 | + this.extraDirs.forEach(({ dir, nameOnly = false }) => { |
| 59 | + const absoluteSourceDir = path.resolve(__dirname, dir); |
| 60 | + let allFiles = globSync("**/*.d.ts", { cwd: absoluteSourceDir, nodir: true }); |
| 61 | + allFiles = allFiles.concat(globSync("**/*.d.mts", { cwd: absoluteSourceDir, nodir: true })); |
| 62 | + |
| 63 | + allFiles.forEach(file => { |
| 64 | + const filePath = path.join(absoluteSourceDir, file); |
| 65 | + let relativePath = path.join(dir, path.relative(absoluteSourceDir, filePath)); |
| 66 | + if (nameOnly) { |
| 67 | + relativePath = path.basename(relativePath); |
| 68 | + } |
| 69 | + this.addFileContents(relativePath, fs.readFileSync(filePath, "utf8")) |
| 70 | + }); |
| 71 | + }); |
| 72 | + } |
| 73 | + |
| 74 | + addFileContents(relativePath, fileContents) { |
| 75 | + if (relativePath in this.srcFileData) { |
| 76 | + if (this.srcFileData[relativePath] !== fileContents) { |
| 77 | + throw new Error(`${relativePath} was previously added with different contents.`); |
| 78 | + } |
| 79 | + } else { |
| 80 | + this.srcFileData[relativePath] = fileContents; |
| 81 | + } |
| 82 | + } |
| 83 | + |
| 84 | + addSpecificFiles() { |
| 85 | + this.extraFiles.forEach(file => { |
| 86 | + const filePath = path.join(this.baseDir, file); |
| 87 | + this.srcFileData[file] = fs.readFileSync(filePath, "utf8"); |
| 88 | + }); |
| 89 | + } |
| 90 | + |
| 91 | + writeSrcFileData() { |
| 92 | + const filesDataPath = path.join(this.outputDir, "files.json"); |
| 93 | + fs.writeFileSync( |
| 94 | + filesDataPath, |
| 95 | + JSON.stringify(this.srcFileData, null, 2) |
| 96 | + ); |
| 97 | + const stats = fs.statSync(filesDataPath); |
| 98 | + const fileSizeInKiB = (stats.size / 1024) | 0; |
| 99 | + console.info(`Exported ${this.projectName}`); |
| 100 | + console.info(` File Contents: ${path.relative(process.cwd(), filesDataPath)}`); |
| 101 | + console.info(` Total Size: ${fileSizeInKiB} KiB`); |
| 102 | + } |
| 103 | + |
| 104 | + writeTsConfig() { |
| 105 | + const tsconfigInputPath = path.join(this.repoDir, "tsconfig.json"); |
| 106 | + const mergedTsconfig = this.loadAndMergeTsconfig(tsconfigInputPath); |
| 107 | + const tsconfigOutputPath = path.join(this.outputDir, "tsconfig.json"); |
| 108 | + fs.writeFileSync( |
| 109 | + tsconfigOutputPath, |
| 110 | + JSON.stringify(mergedTsconfig, null, 2) |
| 111 | + ); |
| 112 | + } |
| 113 | + |
| 114 | + loadAndMergeTsconfig(configPath) { |
| 115 | + const tsconfigContent = fs.readFileSync(configPath, "utf8"); |
| 116 | + const tsconfigContentWithoutComments = tsconfigContent.replace(/(?:^|\s)\/\/.*$|\/\*[\s\S]*?\*\//gm, ""); |
| 117 | + const tsconfig = JSON.parse(tsconfigContentWithoutComments); |
| 118 | + let baseConfigPath = tsconfig.extends; |
| 119 | + if (!baseConfigPath) return tsconfig; |
| 120 | + if (!baseConfigPath.startsWith('./') && !baseConfigPath.startsWith('../')) return tsconfig; |
| 121 | + |
| 122 | + baseConfigPath = path.resolve(path.dirname(configPath), baseConfigPath); |
| 123 | + const baseConfig = this.loadAndMergeTsconfig(baseConfigPath); |
| 124 | + |
| 125 | + const mergedConfig = { ...baseConfig, ...tsconfig }; |
| 126 | + if (baseConfig.compilerOptions && tsconfig.compilerOptions) { |
| 127 | + mergedConfig.compilerOptions = { ...baseConfig.compilerOptions, ...tsconfig.compilerOptions }; |
| 128 | + } |
| 129 | + delete mergedConfig.extends; |
| 130 | + return mergedConfig; |
| 131 | + } |
| 132 | +} |
| 133 | + |
| 134 | +const jest = new Importer({ |
| 135 | + projectName: "jestjs-large", |
| 136 | + size: "large", |
| 137 | + repoUrl: "https://github.com/jestjs/jest.git", |
| 138 | + srcFolder: "packages", |
| 139 | + extraFiles: [ |
| 140 | + "../../node_modules/@babel/types/lib/index.d.ts", |
| 141 | + "../../node_modules/callsites/index.d.ts", |
| 142 | + "../../node_modules/camelcase/index.d.ts", |
| 143 | + "../../node_modules/chalk/types/index.d.ts", |
| 144 | + "../../node_modules/execa/index.d.ts", |
| 145 | + "../../node_modules/fast-json-stable-stringify/index.d.ts", |
| 146 | + "../../node_modules/get-stream/index.d.ts", |
| 147 | + "../../node_modules/strip-json-comments/index.d.ts", |
| 148 | + "../../node_modules/tempy/index.d.ts", |
| 149 | + "../../node_modules/tempy/node_modules/type-fest/index.d.ts", |
| 150 | + "../node_modules/@jridgewell/trace-mapping/types/trace-mapping.d.mts", |
| 151 | + "../node_modules/@types/eslint/index.d.ts", |
| 152 | + "../node_modules/ansi-regex/index.d.ts", |
| 153 | + "../node_modules/ansi-styles/index.d.ts", |
| 154 | + "../node_modules/glob/dist/esm/index.d.ts", |
| 155 | + "../node_modules/jest-worker/build/index.d.ts", |
| 156 | + "../node_modules/lru-cache/dist/esm/index.d.ts", |
| 157 | + "../node_modules/minipass/dist/esm/index.d.ts", |
| 158 | + "../node_modules/p-limit/index.d.ts", |
| 159 | + "../node_modules/path-scurry/dist/esm/index.d.ts", |
| 160 | + "../node_modules/typescript/lib/lib.dom.d.ts", |
| 161 | + ], |
| 162 | + extraDirs: [ |
| 163 | + { dir: "../node_modules/@types/" }, |
| 164 | + { dir: "../node_modules/typescript/lib/", nameOnly: true }, |
| 165 | + { dir: "../node_modules/jest-worker/build/" }, |
| 166 | + { dir: "../node_modules/@jridgewell/trace-mapping/types/" }, |
| 167 | + { dir: "../node_modules/minimatch/dist/esm/" }, |
| 168 | + { dir: "../node_modules/glob/dist/esm/" }, |
| 169 | + { dir: "../../node_modules/tempy/node_modules/type-fest/source/" } |
| 170 | + ], |
| 171 | +}); |
| 172 | + |
| 173 | +const zod = new Importer({ |
| 174 | + projectName: "zod-medium", |
| 175 | + size: "medium", |
| 176 | + repoUrl: "https://github.com/colinhacks/zod.git", |
| 177 | + srcFolder: "packages", |
| 178 | + extraFiles: [], |
| 179 | + extraDirs: [ |
| 180 | + { dir: "../node_modules/typescript/lib/", nameOnly: true }, |
| 181 | + ], |
| 182 | +}); |
| 183 | + |
| 184 | +const immer =new Importer({ |
| 185 | + projectName: "immer-tiny", |
| 186 | + size: "tiny", |
| 187 | + repoUrl: "https://github.com/immerjs/immer.git", |
| 188 | + srcFolder: "src", |
| 189 | + extraFiles: [], |
| 190 | + extraDirs: [ |
| 191 | + { dir: "../node_modules/typescript/lib/", nameOnly: true }, |
| 192 | + ], |
| 193 | +}); |
| 194 | + |
| 195 | +// Skip jest since it produces a hugh in-memory FS. |
| 196 | +// jest.run(); |
| 197 | +zod.run(); |
| 198 | +immer.run(); |
0 commit comments