From 53be86c6d19a7726c9c815f6b0c8beba26810c46 Mon Sep 17 00:00:00 2001 From: Tatsuya Saito Date: Sun, 18 Jun 2023 15:58:35 +0900 Subject: [PATCH] Adds a feature to merge multiple build folders. --- package-lock.json | 5 ++ package.json | 4 +- utils/merge.js | 149 ++++++++++++++++++++++++++++++++++++++++++++++ yarn.lock | 5 ++ 4 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 utils/merge.js diff --git a/package-lock.json b/package-lock.json index 8cd526b9b..f0454b677 100644 --- a/package-lock.json +++ b/package-lock.json @@ -91,6 +91,11 @@ "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==" }, + "commander": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.0.0.tgz", + "integrity": "sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ==" + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", diff --git a/package.json b/package.json index 141abf44e..9cafff490 100644 --- a/package.json +++ b/package.json @@ -19,12 +19,14 @@ "pixelate": "node utils/pixelate.js", "update_info": "node utils/update_info.js", "preview_gif": "node utils/preview_gif.js", - "generate_metadata": "node utils/generate_metadata.js" + "generate_metadata": "node utils/generate_metadata.js", + "merge": "node utils/merge.js" }, "author": "Daniel Eugene Botha (HashLips)", "license": "MIT", "dependencies": { "canvas": "^2.8.0", + "commander": "^11.0.0", "gif-encoder-2": "^1.0.5", "sha1": "^1.1.1" } diff --git a/utils/merge.js b/utils/merge.js new file mode 100644 index 000000000..a4f6e39ee --- /dev/null +++ b/utils/merge.js @@ -0,0 +1,149 @@ +const basePath = process.cwd(); +const { NETWORK } = require(`${basePath}/constants/network.js`); +const fs = require("fs"); +const path = require("path"); +const { program } = require("commander"); + +const { baseUri, namePrefix, network } = require(`${basePath}/src/config.js`); + +const mergeSetup = (output_folder) => { + if (fs.existsSync(output_folder)) { + fs.rmSync(output_folder, { recursive: true }); + } + fs.mkdirSync(output_folder); + fs.mkdirSync(`${output_folder}/json`); + fs.mkdirSync(`${output_folder}/images`); +}; + +const parse_argv = () => { + program + .name("merge") + .description("Merge metadata and images in the folders.") + .version("0.0.1") + .requiredOption("-o, --output ", "output folder") + .option("-s, --shuffle", "after merging, shuffle the images") + .argument( + "", + "input folders, like ./build1 ./build2 ./build3" + ); + program.parse(process.argv); + const options = program.opts(); + const output_folder = path.resolve(options.output); + const input_folders = program.args.map((arg) => path.resolve(arg)); + + return { output_folder, input_folders, shuffle: !!options.shuffle }; +}; + +const file_sort_by_number = (a, b) => { + const num_a = parseInt(a, 10); + const num_b = parseInt(b, 10); + + if (!isNaN(num_a) && !isNaN(num_b)) { + return num_a - num_b; + } else if (!isNaN(num_a)) { + return -1; + } else if (!isNaN(num_b)) { + return 1; + } else { + return a.localeCompare(b); + } +}; + +const create_input_list = (input_folders) => { + let metadata = []; + let images = []; + let missings = []; + let dnas = []; + + input_folders.forEach((folder) => { + const metadata_folder = `${folder}/json`; + const image_folder = `${folder}/images`; + + const files = fs.readdirSync(metadata_folder); + files.sort(file_sort_by_number); + files.forEach((file) => { + if (path.extname(file).toLowerCase() !== ".json") return; + if (path.basename(file).toLowerCase() === "_metadata.json") return; + + const metadata_file = `${metadata_folder}/${file}`; + const f = fs.readFileSync(metadata_file); + const json = JSON.parse(f); + const dna = json.dna; + if (dnas.indexOf(dna) !== -1) { + console.log("Duplicated data: ", metadata_file); + return; + } + dnas.push(dna); + metadata.push(metadata_file); + + const { name } = path.parse(file); + const image_file = `${image_folder}/${name}.png`; + if (fs.existsSync(image_file)) { + images.push(image_file); + } else { + missings.push(image_file); + } + }); + }); + + return { metadata, images, missings }; +}; + +const copy_to_output_folder = (output_folder, images, metadata, shuffle) => { + const metadata_folder = `${output_folder}/json`; + const image_folder = `${output_folder}/images`; + const origin = network == NETWORK.sol ? 0 : 1; + + const num_list = [...Array(images.length).keys()].map((k) => k + origin); + if (shuffle) { + num_list.sort(() => Math.random() - 0.5); + } + + images.forEach((_, i) => { + fs.copyFileSync(images[i], `${image_folder}/${num_list[i]}.png`); + fs.copyFileSync(metadata[i], `${metadata_folder}/${num_list[i]}.json`); + }); +}; + +const update_metadata = (output_folder, metadata) => { + let new_metadata = []; + metadata.forEach((meta) => { + const f = fs.readFileSync(meta); + let item = JSON.parse(f); + + const { name } = path.parse(meta); + item.edition = parseInt(name); + item.name = `${namePrefix} #${item.edition}`; + item.image = `${baseUri}/${item.edition}.png`; + + fs.writeFileSync( + `${output_folder}/json/${item.edition}.json`, + JSON.stringify(item, null, 2) + ); + new_metadata.push(item); + }); + + fs.writeFileSync( + `${output_folder}/json/_metadata.json`, + JSON.stringify(new_metadata, null, 2) + ); +}; + +/* Main process */ +const { output_folder, input_folders, shuffle } = parse_argv(); +mergeSetup(output_folder); + +const { metadata, images, missings } = create_input_list(input_folders); +if (missings.length > 0) { + console.error("ERROR: There is no images corresponding to the metadata."); + console.error(missings); + return; +} + +copy_to_output_folder(output_folder, images, metadata, shuffle); +const { metadata: out_metadata, images: out_images } = create_input_list([ + output_folder, +]); +update_metadata(output_folder, out_metadata); + +console.log("Merge done."); diff --git a/yarn.lock b/yarn.lock index 14200dc80..9b44f0d33 100644 --- a/yarn.lock +++ b/yarn.lock @@ -89,6 +89,11 @@ color-support@^1.1.2: resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2" integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg== +commander@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-11.0.0.tgz#43e19c25dbedc8256203538e8d7e9346877a6f67" + integrity sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ== + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"