|
| 1 | +// Copyright (c) 2018 ml5 |
| 2 | +// |
| 3 | +// This software is released under the MIT License. |
| 4 | +// https://opensource.org/licenses/MIT |
| 5 | + |
| 6 | +/* eslint prefer-destructuring: ["error", {AssignmentExpression: {array: false}}] */ |
| 7 | +/* eslint no-await-in-loop: "off" */ |
| 8 | +/* |
| 9 | +* CVAE: Run conditional auto-encoder for pro-trained model |
| 10 | +*/ |
| 11 | + |
| 12 | +import * as tf from '@tensorflow/tfjs'; |
| 13 | +import callCallback from '../utils/callcallback'; |
| 14 | + |
| 15 | +class Cvae { |
| 16 | + /** |
| 17 | + * Create a Conditional Variational Autoencoder (CVAE). |
| 18 | + * @param {String} modelPath - Required. The url path to your model. |
| 19 | + * @param {function} callback - Required. A function to run once the model has been loaded. |
| 20 | + */ |
| 21 | + constructor(modelPath, callback) { |
| 22 | + /** |
| 23 | + * Boolean value that specifies if the model has loaded. |
| 24 | + * @type {boolean} |
| 25 | + * @public |
| 26 | + */ |
| 27 | + this.ready = false; |
| 28 | + this.model = {}; |
| 29 | + this.latentDim = tf.randomUniform([1, 16]); |
| 30 | + this.modelPath = modelPath; |
| 31 | + this.modelPathPrefix = ''; |
| 32 | + |
| 33 | + this.jsonLoader().then(val => { |
| 34 | + this.modelPathPrefix = this.modelPath.split('manifest.json')[0] |
| 35 | + this.ready = callCallback(this.loadCVAEModel(this.modelPathPrefix+val.model), callback); |
| 36 | + this.labels = val.labels; |
| 37 | + // get an array full of zero with the length of labels [0, 0, 0 ...] |
| 38 | + this.labelVector = Array(this.labels.length+1).fill(0); |
| 39 | + }); |
| 40 | + } |
| 41 | + |
| 42 | + // load tfjs model that is converted by tensorflowjs with graph and weights |
| 43 | + async loadCVAEModel(modelPath) { |
| 44 | + this.model = await tf.loadLayersModel(modelPath); |
| 45 | + return this; |
| 46 | + } |
| 47 | + |
| 48 | + /** |
| 49 | + * Generate a random result. |
| 50 | + * @param {String} label - A label of the feature your want to generate |
| 51 | + * @param {function} callback - A function to handle the results of ".generate()". Likely a function to do something with the generated image data. |
| 52 | + * @return {raw: ImageData, src: Blob, image: p5.Image} |
| 53 | + */ |
| 54 | + async generate(label, callback) { |
| 55 | + return callCallback(this.generateInternal(label), callback); |
| 56 | + } |
| 57 | + |
| 58 | + loadAsync(url){ |
| 59 | + return new Promise((resolve, reject) => { |
| 60 | + if(!this.ready) reject(); |
| 61 | + loadImage(url, (img) => { |
| 62 | + resolve(img); |
| 63 | + }); |
| 64 | + }); |
| 65 | + }; |
| 66 | + |
| 67 | + getBlob(inputCanvas) { |
| 68 | + return new Promise((resolve, reject) => { |
| 69 | + if (!this.ready) reject(); |
| 70 | + |
| 71 | + inputCanvas.toBlob((blob) => { |
| 72 | + resolve(blob); |
| 73 | + }); |
| 74 | + }); |
| 75 | + } |
| 76 | + |
| 77 | + checkP5() { |
| 78 | + if (typeof window !== 'undefined' && window.p5 && this |
| 79 | + && window.p5.Image && typeof window.p5.Image === 'function') return true; |
| 80 | + return false; |
| 81 | + } |
| 82 | + |
| 83 | + async generateInternal(label) { |
| 84 | + const res = tf.tidy(() => { |
| 85 | + this.latentDim = tf.randomUniform([1, 16]); |
| 86 | + const cursor = this.labels.indexOf(label); |
| 87 | + if (cursor < 0) { |
| 88 | + console.log('Wrong input of the label!'); |
| 89 | + return [undefined, undefined]; // invalid input just return; |
| 90 | + } |
| 91 | + |
| 92 | + this.labelVector = this.labelVector.map(() => 0); // clear vector |
| 93 | + this.labelVector[cursor+1] = 1; |
| 94 | + |
| 95 | + const input = tf.tensor([this.labelVector]); |
| 96 | + |
| 97 | + const temp = this.model.predict([this.latentDim, input]); |
| 98 | + return temp.reshape([temp.shape[1], temp.shape[2], temp.shape[3]]); |
| 99 | + }); |
| 100 | + |
| 101 | + const raws = await tf.browser.toPixels(res); |
| 102 | + |
| 103 | + const canvas = document.createElement('canvas'); // consider using offScreneCanvas |
| 104 | + const ctx = canvas.getContext('2d'); |
| 105 | + const [x, y] = res.shape; |
| 106 | + canvas.width = x; |
| 107 | + canvas.height = y; |
| 108 | + const imgData = ctx.createImageData(x, y); |
| 109 | + const data = imgData.data; |
| 110 | + for (let i = 0; i < x * y * 4; i += 1) data[i] = raws[i]; |
| 111 | + ctx.putImageData(imgData, 0, 0); |
| 112 | + |
| 113 | + const src = URL.createObjectURL(await this.getBlob(canvas)); |
| 114 | + let image; |
| 115 | + /* global loadImage */ |
| 116 | + if (this.checkP5()) image = await this.loadAsync(src); |
| 117 | + return { src, raws, image }; |
| 118 | + } |
| 119 | + |
| 120 | + async jsonLoader() { |
| 121 | + return new Promise((resolve, reject) => { |
| 122 | + const xhr = new XMLHttpRequest(); |
| 123 | + xhr.open('GET', this.modelPath); |
| 124 | + |
| 125 | + xhr.onload = () => { |
| 126 | + const json = JSON.parse(xhr.responseText); |
| 127 | + resolve(json); |
| 128 | + }; |
| 129 | + xhr.onerror = (error) => { |
| 130 | + reject(error); |
| 131 | + }; |
| 132 | + xhr.send(); |
| 133 | + }); |
| 134 | + } |
| 135 | +} |
| 136 | + |
| 137 | +const CVAE = (model, callback) => new Cvae(model, callback); |
| 138 | + |
| 139 | + |
| 140 | +export default CVAE; |
0 commit comments