diff --git a/.gitignore b/.gitignore index 59e5ef98..b5bf5111 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ training/lstm/data/t dist /dist_examples .yarn +src/offline/models website/translated_docs website/build/ diff --git a/package.json b/package.json index cd90354e..6c869302 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "test": "jest --config tests/jest.config.js", "upload-examples": "node scripts/uploadExamples.js", "update-p5-version": "node scripts/updateP5Version.js", - "update-readme": "node scripts/updateReadme.js" + "update-readme": "node scripts/updateReadme.js", + "fetch-model-files": "node scripts/fetchModelFiles.js" }, "files": [ "dist" @@ -45,6 +46,7 @@ "postinstall-postinstall": "^2.1.0", "prettier": "2.8.8", "rimraf": "^5.0.5", + "tar": "^7.4.3", "terser-webpack-plugin": "^5.3.10", "webpack": "^5.76.1", "webpack-cli": "^5.0.1", diff --git a/scripts/fetchModelFiles.js b/scripts/fetchModelFiles.js new file mode 100644 index 00000000..60c2f7bb --- /dev/null +++ b/scripts/fetchModelFiles.js @@ -0,0 +1,175 @@ +const fs = require("fs"); +const tar = require("tar"); +const path = require("path"); +const rimraf = require("rimraf"); +const { Readable } = require("stream"); + +const outputDir = "src/offline/models"; +const tmpDir = "src/offline/models/tmp"; + +/** + * URLs of the models to fetch. + */ +const modelURLs = { + HandPose: { + detectorLite: + "https://www.kaggle.com/api/v1/models/mediapipe/handpose-3d/tfJs/detector-lite/1/download", + landmarkLite: + "https://www.kaggle.com/api/v1/models/mediapipe/handpose-3d/tfJs/landmark-lite/1/download", + }, +}; + +/** + * Fetch a compressed model from a given URL and save it to a given output directory. + * @param {string} url - The URL of the .tar.gz file to fetch. + * @param {string} outputDir - The directory path to save the fetched file. + */ +async function fetchCompressedModel(url, outputDir) { + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + const res = await fetch(url); + + const fileStream = fs.createWriteStream( + path.resolve(outputDir, "model.tar.gz"), + { flags: "w" } + ); + Readable.fromWeb(res.body).pipe(fileStream); + + return new Promise((resolve, reject) => { + fileStream.on("finish", resolve); + fileStream.on("error", reject); + }); +} + +/** + * Unzips a compressed file to a given output. + * @param {string} filePath + * @param {string} outputDir + */ +function unzip(filePath, outputDir) { + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + tar.x({ + file: filePath, + cwd: outputDir, + gzip: true, + sync: true, + }); +} + +/** + * Convert a JSON file to a JS file. + * Pad the JSON content with `export default` so it could be imported as a module. + * @param {string} jsonPath - Path to the JSON file to convert to JS. + * @param {string} outputDir - The directory path to save the JS representation of the JSON file. + * @param {string} outputName - The name of the output JS file. + */ +function jsonToJs(jsonPath, outputDir, outputName) { + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + // Read the model.json file + const content = fs.readFileSync(jsonPath, "utf-8"); + // Pad the content with export default + const padded = `const modelJson=${content}; export default modelJson;`; + // Write the content to a js file + fs.writeFileSync(path.resolve(outputDir, outputName), padded); +} + +/** + * Create a JS file from a binary file. + * The binary file is converted to a Uint8Array so it could be written in a JS file. + * Add `export default` to the Uint8Array so it could be imported as a module. + * @param {string} binaryPath - Path to the binary file to convert to JS. + * @param {string} outputDir - The directory path to save the JS representation of the binary file. + * @param {string} outputName - The name of the output JS file. + */ +function binaryToJs(binaryPath, outputDir, outputName) { + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + // Read the model.bin file + const content = fs.readFileSync(binaryPath); + // Convert the binary file to a Uint8Array so it could be written in a js file + const arrayBuffer = content.buffer.slice( + content.byteOffset, + content.byteOffset + content.byteLength + ); + const uint8Array = new Uint8Array(arrayBuffer); + // Write the Uint8Array to a js file + fs.writeFileSync( + path.resolve(outputDir, outputName), + `const modelBin=new Uint8Array([${uint8Array}]); export default modelBin;` + ); +} +/** + * Download the model binary files and convert them to JS files. + * The function will make a JS file for the model.json and model.bin files. + * The files are converted to JS so they could be bundled into the ml5.js library. + * + * @param {string} url - The URL of the .tar.gz file to fetch. + * @param {string} outputDir - The directory path to save the JS representation of the model.json and model.bin files. + * @returns + */ +async function makeJsModelFiles(url, ml5ModelName, modelName) { + // temporary directory for processing the model files + const modelTmpDir = path.resolve(tmpDir, ml5ModelName, modelName); + const modelDir = path.resolve(outputDir, ml5ModelName, modelName); + + await fetchCompressedModel(url, modelTmpDir); + unzip(path.resolve(modelTmpDir, "model.tar.gz"), modelTmpDir); + + // Convert model.json to model.json.js + jsonToJs(path.resolve(modelTmpDir, "model.json"), modelDir, "model.json.js"); + + // Convert all model.bin files to model.bin.js + const binFiles = fs + .readdirSync(modelTmpDir) + .filter((file) => file.endsWith(".bin")); + + binFiles.forEach((binFile) => { + binaryToJs(path.resolve(modelTmpDir, binFile), modelDir, `${binFile}.js`); + }); +} + +/** + * Remove the temporary directory. + */ +function cleanup() { + rimraf.sync(tmpDir); +} + +/** + * Check if the model files (js) already + * @param {string} ml5Model + * @param {string} model + * @returns + */ +function modelJsExists(ml5Model, model) { + const hasDir = fs.existsSync(path.resolve(outputDir, ml5Model, model)); + if (!hasDir) return false; + + const files = fs.readdirSync(path.resolve(outputDir, ml5Model, model)); + const hasJson = files.includes("model.json.js"); + const hasBin = files.some((file) => file.endsWith(".bin.js")); + return hasJson && hasBin; +} + +/** + * Point of entry to the script. + */ +async function main() { + for (ml5Model in modelURLs) { + for (model in modelURLs[ml5Model]) { + if (!modelJsExists(ml5Model, model)) { + await makeJsModelFiles(modelURLs[ml5Model][model], ml5Model, model); + } + } + } + cleanup(); +} +main(); diff --git a/src/HandPose/index.js b/src/HandPose/index.js index 52481b3e..64ac81a2 100644 --- a/src/HandPose/index.js +++ b/src/HandPose/index.js @@ -149,6 +149,10 @@ class HandPose { "handPose" ); + if (this.loadOfflineModel) { + this.loadOfflineModel(modelConfig); + } + // Load the Tensorflow.js detector instance await tf.ready(); this.model = await handPoseDetection.createDetector(pipeline, modelConfig); @@ -308,3 +312,4 @@ const handPose = (...inputs) => { }; export default handPose; +export { HandPose }; diff --git a/src/offline/HandPose/index.js b/src/offline/HandPose/index.js new file mode 100644 index 00000000..5d4a92b5 --- /dev/null +++ b/src/offline/HandPose/index.js @@ -0,0 +1,66 @@ +import { uint8ArrayToFile } from "../../utils/io.js"; +import landmarkLiteJson from "../models/HandPose/landmarkLite/model.json.js"; +import landmarkLiteBinArray from "../models/HandPose/landmarkLite/group1-shard1of1.bin.js"; +import detectorLiteJson from "../models/HandPose/detectorLite/model.json.js"; +import detectorLiteBinArray from "../models/HandPose/detectorLite/group1-shard1of1.bin.js"; + +/** + * Define the loadOfflineModel function. + * The function will inject the model URLs into the config object. + * This function will be called by HandPose during initialization. + * @param {Object} configObject - The configuration object to mutate. + */ +function loadOfflineModel(configObject) { + let landmarkJson; + let landmarkBinArray; + let detectorJson; + let detectorBinArray; + + // Select the correct model to load based on the config object. + if (configObject.modelType === "lite") { + landmarkJson = landmarkLiteJson; + landmarkBinArray = landmarkLiteBinArray; + detectorJson = detectorLiteJson; + detectorBinArray = detectorLiteBinArray; + } + + // Convert the binary data to a file object. + const landmarkBinFile = uint8ArrayToFile( + landmarkBinArray, + "group1-shard1of1.bin" + ); + const detectorBinFile = uint8ArrayToFile( + detectorBinArray, + "group1-shard1of1.bin" + ); + + // Give the detector model binary data a URL. + const landmarkBinURL = URL.createObjectURL(landmarkBinFile); + const detectorBinURL = URL.createObjectURL(detectorBinFile); + + // Change the path to the binary file in the model json data. + landmarkJson.weightsManifest[0].paths[0] = landmarkBinURL.split("/").pop(); + detectorJson.weightsManifest[0].paths[0] = detectorBinURL.split("/").pop(); + + // Convert the json data to file objects. + const landmarkJsonFile = new File( + [JSON.stringify(landmarkJson)], + "model.json", + { type: "application/json" } + ); + const detectorJsonFile = new File( + [JSON.stringify(detectorJson)], + "model.json", + { type: "application/json" } + ); + + // Give the json data URLs. + const landmarkJsonURL = URL.createObjectURL(landmarkJsonFile); + const detectorJsonURL = URL.createObjectURL(detectorJsonFile); + + // Inject the URLs into the config object. + configObject.landmarkModelUrl = landmarkJsonURL; + configObject.detectorModelUrl = detectorJsonURL; +} + +export default loadOfflineModel; diff --git a/src/offline/index.js b/src/offline/index.js new file mode 100644 index 00000000..2dfeab13 --- /dev/null +++ b/src/offline/index.js @@ -0,0 +1,8 @@ +import ml5 from "../index"; + +import { HandPose } from "../HandPose"; +import loadHandPoseOfflineModel from "./HandPose"; + +HandPose.prototype.loadOfflineModel = loadHandPoseOfflineModel; + +export default ml5; diff --git a/src/utils/io.js b/src/utils/io.js index 09b2cd67..92e50e94 100644 --- a/src/utils/io.js +++ b/src/utils/io.js @@ -35,4 +35,15 @@ const loadFile = async (path, callback) => throw error; }); -export { saveBlob, loadFile }; +/** + * Convert a Uint8Array to a binary File object. + * @param {Uint8Array} uint8Array - The Uint8Array to convert to a file. + * @param {string} fileName - The name of the file. + * @returns {File} A file object. + */ +function uint8ArrayToFile(uint8Array, fileName) { + const blob = new Blob([uint8Array], { type: "application/octet-stream" }); + return new File([blob], fileName, { type: "application/octet-stream" }); +} + +export { saveBlob, loadFile, uint8ArrayToFile }; diff --git a/webpack.config.js b/webpack.config.js index ff90e03d..a6ec69f6 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -5,15 +5,19 @@ const TerserPlugin = require("terser-webpack-plugin"); const commonConfig = { context: __dirname, - entry: "./src/index.js", + entry: { + ml5: "./src/index.js", + "ml5-offline": "./src/offline/index.js", + }, output: { - filename: "ml5.js", + filename: "[name].js", path: resolve(__dirname, "dist"), library: { name: "ml5", type: "umd", export: "default", }, + globalObject: "this", }, }; @@ -48,18 +52,20 @@ const developmentConfig = { resolve: { fallback: { fs: false, - util: false + util: false, }, - } + }, }; const productionConfig = { mode: "production", + devtool: "source-map", entry: { ml5: "./src/index.js", "ml5.min": "./src/index.js", + "ml5-offline": "./src/offline/index.js", + "ml5-offline.min": "./src/offline/index.js", }, - devtool: "source-map", output: { publicPath: "/", filename: "[name].js", @@ -68,8 +74,7 @@ const productionConfig = { minimize: true, minimizer: [ new TerserPlugin({ - include: "ml5.min.js", - exclude: "ml5.js", + include: /\.min\.js$/, extractComments: false, }), ], @@ -77,9 +82,9 @@ const productionConfig = { resolve: { fallback: { fs: false, - util: false + util: false, }, - } + }, }; module.exports = function (env, args) { diff --git a/yarn.lock b/yarn.lock index 59a80b8e..e7723c8e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1429,6 +1429,15 @@ __metadata: languageName: node linkType: hard +"@isaacs/fs-minipass@npm:^4.0.0": + version: 4.0.1 + resolution: "@isaacs/fs-minipass@npm:4.0.1" + dependencies: + minipass: "npm:^7.0.4" + checksum: 10c0/c25b6dc1598790d5b55c0947a9b7d111cfa92594db5296c3b907e2f533c033666f692a3939eadac17b1c7c40d362d0b0635dc874cbfe3e70db7c2b07cc97a5d2 + languageName: node + linkType: hard + "@istanbuljs/load-nyc-config@npm:^1.0.0": version: 1.1.0 resolution: "@istanbuljs/load-nyc-config@npm:1.1.0" @@ -3413,6 +3422,13 @@ __metadata: languageName: node linkType: hard +"chownr@npm:^3.0.0": + version: 3.0.0 + resolution: "chownr@npm:3.0.0" + checksum: 10c0/43925b87700f7e3893296c8e9c56cc58f926411cce3a6e5898136daaf08f08b9a8eb76d37d3267e707d0dcc17aed2e2ebdf5848c0c3ce95cf910a919935c1b10 + languageName: node + linkType: hard + "chrome-trace-event@npm:^1.0.2": version: 1.0.3 resolution: "chrome-trace-event@npm:1.0.3" @@ -6786,7 +6802,7 @@ __metadata: languageName: node linkType: hard -"minipass@npm:^7.0.2, minipass@npm:^7.0.3, minipass@npm:^7.1.2": +"minipass@npm:^7.0.2, minipass@npm:^7.0.3, minipass@npm:^7.0.4, minipass@npm:^7.1.2": version: 7.1.2 resolution: "minipass@npm:7.1.2" checksum: 10c0/b0fd20bb9fb56e5fa9a8bfac539e8915ae07430a619e4b86ff71f5fc757ef3924b23b2c4230393af1eda647ed3d75739e4e0acb250a6b1eb277cf7f8fe449557 @@ -6803,6 +6819,16 @@ __metadata: languageName: node linkType: hard +"minizlib@npm:^3.0.1": + version: 3.0.1 + resolution: "minizlib@npm:3.0.1" + dependencies: + minipass: "npm:^7.0.4" + rimraf: "npm:^5.0.5" + checksum: 10c0/82f8bf70da8af656909a8ee299d7ed3b3372636749d29e105f97f20e88971be31f5ed7642f2e898f00283b68b701cc01307401cdc209b0efc5dd3818220e5093 + languageName: node + linkType: hard + "mkdirp@npm:^1.0.3": version: 1.0.4 resolution: "mkdirp@npm:1.0.4" @@ -6812,6 +6838,15 @@ __metadata: languageName: node linkType: hard +"mkdirp@npm:^3.0.1": + version: 3.0.1 + resolution: "mkdirp@npm:3.0.1" + bin: + mkdirp: dist/cjs/src/bin.js + checksum: 10c0/9f2b975e9246351f5e3a40dcfac99fcd0baa31fbfab615fe059fb11e51f10e4803c63de1f384c54d656e4db31d000e4767e9ef076a22e12a641357602e31d57d + languageName: node + linkType: hard + "ml5@workspace:.": version: 0.0.0-use.local resolution: "ml5@workspace:." @@ -6844,6 +6879,7 @@ __metadata: postinstall-postinstall: "npm:^2.1.0" prettier: "npm:2.8.8" rimraf: "npm:^5.0.5" + tar: "npm:^7.4.3" terser-webpack-plugin: "npm:^5.3.10" webpack: "npm:^5.76.1" webpack-cli: "npm:^5.0.1" @@ -8441,6 +8477,20 @@ __metadata: languageName: node linkType: hard +"tar@npm:^7.4.3": + version: 7.4.3 + resolution: "tar@npm:7.4.3" + dependencies: + "@isaacs/fs-minipass": "npm:^4.0.0" + chownr: "npm:^3.0.0" + minipass: "npm:^7.1.2" + minizlib: "npm:^3.0.1" + mkdirp: "npm:^3.0.1" + yallist: "npm:^5.0.0" + checksum: 10c0/d4679609bb2a9b48eeaf84632b6d844128d2412b95b6de07d53d8ee8baf4ca0857c9331dfa510390a0727b550fd543d4d1a10995ad86cdf078423fbb8d99831d + languageName: node + linkType: hard + "terser-webpack-plugin@npm:^5.3.10": version: 5.3.10 resolution: "terser-webpack-plugin@npm:5.3.10" @@ -9844,6 +9894,13 @@ __metadata: languageName: node linkType: hard +"yallist@npm:^5.0.0": + version: 5.0.0 + resolution: "yallist@npm:5.0.0" + checksum: 10c0/a499c81ce6d4a1d260d4ea0f6d49ab4da09681e32c3f0472dee16667ed69d01dae63a3b81745a24bd78476ec4fcf856114cb4896ace738e01da34b2c42235416 + languageName: node + linkType: hard + "yaml@npm:^2.2.2": version: 2.3.2 resolution: "yaml@npm:2.3.2"