Merge branch 'es6-rewrite' into docs

Author: xenova

jsconfig.json

@@ -5,6 +5,9 @@
     ],
     "compilerOptions": {
         // Tells the compiler to check JS files
-        "checkJs": true
+        "checkJs": true,
+        "target": "esnext",
+        "module": "esnext",
+        "moduleResolution": "nodenext",
     }
 }

package.json

@@ -4,6 +4,7 @@
   "description": "Run 🤗 Transformers in your browser! We currently support BERT, ALBERT, DistilBERT, MobileBERT, SqueezeBERT, T5, T5v1.1, FLAN-T5, mT5, BART, MarianMT, GPT2, GPT Neo, CodeGen, Whisper, CLIP, Vision Transformer, VisionEncoderDecoder, and DETR models, for a variety of tasks including: masked language modelling, text classification, token classification, zero-shot classification, text-to-text generation, translation, summarization, question answering, text generation, automatic speech recognition, image classification, zero-shot image classification, image-to-text, image segmentation, and object detection.",
   "main": "./src/transformers.js",
   "types": "./types/transformers.d.ts",
+  "type": "module",
   "directories": {
     "test": "tests"
   },

src/backends/onnx.js

@@ -1,13 +1,13 @@
-let ONNX;
+export let ONNX;
 
 // TODO support more execution providers (e.g., webgpu)
-const executionProviders = ['wasm'];
+export const executionProviders = ['wasm'];
 
 if (typeof process !== 'undefined') {
     // Running in a node-like environment.
     // Try to import onnxruntime-node, using onnxruntime-web as a fallback
     try {
-        ONNX = require('onnxruntime-node');
+        ONNX = (await import('onnxruntime-node')).default;
     } catch (err) {
         console.warn(
             "Node.js environment detected, but `onnxruntime-node` was not found. " +
@@ -20,7 +20,7 @@ if (typeof process !== 'undefined') {
         // @ts-ignore
         global.self = global;
 
-        ONNX = require('onnxruntime-web');
+        ONNX = (await import('onnxruntime-web')).default;
 
         // Disable spawning worker threads for testing.
         // This is done by setting numThreads to 1
@@ -33,10 +33,6 @@ if (typeof process !== 'undefined') {
 
 } else {
     // Running in a browser-environment, so we just import `onnxruntime-web`
-    ONNX = require('onnxruntime-web');
+    ONNX = (await import('onnxruntime-web')).default;
 }
 
-module.exports = {
-    ONNX,
-    executionProviders,
-}

src/env.js

@@ -1,7 +1,11 @@
-const fs = require('fs');
-const path = require('path');
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
 
-const { env: onnx_env } = require('./backends/onnx.js').ONNX;
+import { ONNX } from './backends/onnx.js';
+const { env: onnx_env } = ONNX;
+
+const __dirname = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
 
 
 
@@ -28,7 +32,7 @@ const localModelPath = RUNNING_LOCALLY
 // We use remote wasm files by default to make it easier for newer users.
 // In practice, users should probably self-host the necessary .wasm files.
 onnx_env.wasm.wasmPaths = RUNNING_LOCALLY
-    ? path.join(path.dirname(__dirname), '/dist/')
+    ? path.join(__dirname, '/dist/')
     : 'https://cdn.jsdelivr.net/npm/@xenova/transformers/dist/';
 
 
@@ -55,7 +59,9 @@ const env = {
 
     // Whether to use the file system to load files. By default, it is true available.
     useFS: FS_AVAILABLE,
-
+    
+    // Directory name of module. Useful for resolving local paths.
+    __dirname,
 
     /////////////////// Cache settings ///////////////////
     // Whether to use Cache API to cache models. By default, it is true if available.
@@ -78,6 +84,3 @@ function isEmpty(obj) {
     return Object.keys(obj).length === 0;
 }
 
-module.exports = {
-    env
-}

src/generation.js

@@ -1,11 +1,11 @@
-const { Tensor } = require("./tensor_utils.js");
-const {
+import { Tensor } from './tensor_utils.js';
+import {
     Callable,
     exists,
-} = require("./utils.js");
-const {
+} from './utils.js';
+import {
     log_softmax
-} = require('./math_utils.js');
+} from './math_utils.js';
 
 /**
  * A class representing a list of logits processors. A logits processor is a function that modifies the logits
@@ -14,7 +14,7 @@ const {
  *
  * @extends Callable
  */
-class LogitsProcessorList extends Callable {
+export class LogitsProcessorList extends Callable {
     /**
      * Constructs a new instance of `LogitsProcessorList`.
      */
@@ -68,7 +68,7 @@ class LogitsProcessorList extends Callable {
  * Base class for processing logits.
  * @extends Callable
  */
-class LogitsProcessor extends Callable {
+export class LogitsProcessor extends Callable {
     /**
      * Apply the processor to the input logits.
      *
@@ -87,7 +87,7 @@ class LogitsProcessor extends Callable {
  * 
  * @extends LogitsProcessor
  */
-class ForceTokensLogitsProcessor extends LogitsProcessor {
+export class ForceTokensLogitsProcessor extends LogitsProcessor {
     /**
      * Constructs a new instance of `ForceTokensLogitsProcessor`.
      * 
@@ -119,7 +119,7 @@ class ForceTokensLogitsProcessor extends LogitsProcessor {
  * A LogitsProcessor that forces a BOS token at the beginning of the generated sequence.
  * @extends LogitsProcessor
  */
-class ForcedBOSTokenLogitsProcessor extends LogitsProcessor {
+export class ForcedBOSTokenLogitsProcessor extends LogitsProcessor {
     /**
      * Create a ForcedBOSTokenLogitsProcessor.
      * @param {number} bos_token_id - The ID of the beginning-of-sequence token to be forced.
@@ -148,7 +148,7 @@ class ForcedBOSTokenLogitsProcessor extends LogitsProcessor {
  * 
  * @extends LogitsProcessor
  */
-class ForcedEOSTokenLogitsProcessor extends LogitsProcessor {
+export class ForcedEOSTokenLogitsProcessor extends LogitsProcessor {
     /**
      * Create a ForcedEOSTokenLogitsProcessor.
      * @param {number} max_length - Max length of the sequence.
@@ -176,7 +176,7 @@ class ForcedEOSTokenLogitsProcessor extends LogitsProcessor {
  * A LogitsProcessor that handles adding timestamps to generated text.
  * @extends LogitsProcessor
  */
-class WhisperTimeStampLogitsProcessor extends LogitsProcessor {
+export class WhisperTimeStampLogitsProcessor extends LogitsProcessor {
     /**
      * Constructs a new WhisperTimeStampLogitsProcessor.
      * @param {object} generate_config - The config object passed to the `generate()` method of a transformer model.
@@ -251,7 +251,7 @@ class WhisperTimeStampLogitsProcessor extends LogitsProcessor {
  * 
  * @extends LogitsProcessor
  */
-class NoRepeatNGramLogitsProcessor extends LogitsProcessor {
+export class NoRepeatNGramLogitsProcessor extends LogitsProcessor {
     /**
      * Create a NoRepeatNGramLogitsProcessor.
      * @param {number} no_repeat_ngram_size - The no-repeat-ngram size. All ngrams of this size can only occur once.
@@ -342,7 +342,7 @@ class NoRepeatNGramLogitsProcessor extends LogitsProcessor {
  * 
  * @extends LogitsProcessor
  */
-class RepetitionPenaltyLogitsProcessor extends LogitsProcessor {
+export class RepetitionPenaltyLogitsProcessor extends LogitsProcessor {
     /**
      * Create a RepetitionPenaltyLogitsProcessor.
      * @param {number} penalty - The penalty to apply for repeated tokens.
@@ -374,7 +374,7 @@ class RepetitionPenaltyLogitsProcessor extends LogitsProcessor {
 }
 
 
-class GenerationConfig {
+export class GenerationConfig {
     /**
      * Create a GenerationConfig object
      * @constructor
@@ -495,15 +495,3 @@ class GenerationConfig {
         this.generation_kwargs = kwargs.generation_kwargs ?? {};
     }
 }
-
-module.exports = {
-    LogitsProcessor,
-    LogitsProcessorList,
-    GenerationConfig,
-    ForcedBOSTokenLogitsProcessor,
-    ForcedEOSTokenLogitsProcessor,
-    WhisperTimeStampLogitsProcessor,
-    ForceTokensLogitsProcessor,
-    NoRepeatNGramLogitsProcessor,
-    RepetitionPenaltyLogitsProcessor
-};

src/image_utils.js

@@ -1,12 +1,12 @@
 
-const fs = require('fs');
-const { isString } = require('./utils.js');
-const { env } = require('./env.js');
+import fs from 'fs';
+import { isString } from './utils.js';
+import { env } from './env.js';
 
-const { getFile } = require('./utils/hub.js');
+import { getFile } from './utils/hub.js';
 
 // Will be empty (or not used) if running in browser or web-worker
-const sharp = require('sharp');
+import sharp from 'sharp';
 
 let CanvasClass;
 let ImageDataClass;
@@ -30,7 +30,7 @@ if (typeof self !== 'undefined') {
 }
 
 
-class CustomImage {
+export class CustomImage {
 
     /**
      * Create a new CustomImage object.
@@ -440,7 +440,3 @@ class CustomImage {
         fs.writeFileSync(path, buffer);
     }
 }
-
-module.exports = {
-    CustomImage,
-};

src/math_utils.js

@@ -8,7 +8,7 @@
 /**
  * @param {TypedArray} input
  */
-function interpolate(input, [in_channels, in_height, in_width], [out_height, out_width], mode = 'bilinear', align_corners = false) {
+export function interpolate(input, [in_channels, in_height, in_width], [out_height, out_width], mode = 'bilinear', align_corners = false) {
     // TODO use mode and align_corners
 
     // Output image dimensions
@@ -86,7 +86,7 @@ function interpolate(input, [in_channels, in_height, in_width], [out_height, out
  * @param {number[]} axes 
  * @returns {[T, number[]]} The transposed array and the new shape.
  */
-function transpose_data(array, dims, axes) {
+export function transpose_data(array, dims, axes) {
     // Calculate the new shape of the transposed array
     // and the stride of the original array
     const shape = new Array(axes.length);
@@ -125,7 +125,7 @@ function transpose_data(array, dims, axes) {
  * @param {number[]} arr - The array of numbers to compute the softmax of.
  * @returns {number[]} The softmax array.
  */
-function softmax(arr) {
+export function softmax(arr) {
     // Compute the maximum value in the array
     const maxVal = max(arr)[0];
 
@@ -146,7 +146,7 @@ function softmax(arr) {
  * @param {number[]} arr - The input array to calculate the log_softmax function for.
  * @returns {any} - The resulting log_softmax array.
  */
-function log_softmax(arr) {
+export function log_softmax(arr) {
     // Compute the softmax values
     const softmaxArr = softmax(arr);
 
@@ -162,7 +162,7 @@ function log_softmax(arr) {
  * @param {number[]} arr2 - The second array.
  * @returns {number} - The dot product of arr1 and arr2.
  */
-function dot(arr1, arr2) {
+export function dot(arr1, arr2) {
     return arr1.reduce((acc, val, i) => acc + val * arr2[i], 0);
 }
 
@@ -174,7 +174,7 @@ function dot(arr1, arr2) {
  * @param {number} [top_k=0] - The number of top items to return (default: 0 = return all)
  * @returns {Array} - The top k items, sorted by descending order
  */
-function getTopItems(items, top_k = 0) {
+export function getTopItems(items, top_k = 0) {
     // if top == 0, return all
 
     items = Array.from(items)
@@ -195,7 +195,7 @@ function getTopItems(items, top_k = 0) {
  * @param {number[]} arr2 - The second array.
  * @returns {number} The cosine similarity between the two arrays.
  */
-function cos_sim(arr1, arr2) {
+export function cos_sim(arr1, arr2) {
     // Calculate dot product of the two arrays
     const dotProduct = dot(arr1, arr2);
 
@@ -216,7 +216,7 @@ function cos_sim(arr1, arr2) {
  * @param {number[]} arr - The array to calculate the magnitude of.
  * @returns {number} The magnitude of the array.
  */
-function magnitude(arr) {
+export function magnitude(arr) {
     return Math.sqrt(arr.reduce((acc, val) => acc + val * val, 0));
 }
 
@@ -227,7 +227,7 @@ function magnitude(arr) {
  * @returns {number[]} - the value and index of the minimum element, of the form: [valueOfMin, indexOfMin]
  * @throws {Error} If array is empty.
  */
-function min(arr) {
+export function min(arr) {
     if (arr.length === 0) throw Error('Array must not be empty');
     let min = arr[0];
     let indexOfMin = 0;
@@ -247,7 +247,7 @@ function min(arr) {
  * @returns {number[]} - the value and index of the maximum element, of the form: [valueOfMax, indexOfMax]
  * @throws {Error} If array is empty.
  */
-function max(arr) {
+export function max(arr) {
     if (arr.length === 0) throw Error('Array must not be empty');
     let max = arr[0];
     let indexOfMax = 0;
@@ -265,7 +265,7 @@ function max(arr) {
  * FFT class provides functionality for performing Fast Fourier Transform on arrays
  * Code adapted from https://www.npmjs.com/package/fft.js
  */
-class FFT {
+export class FFT {
     /**
      * @param {number} size - The size of the input array. Must be a power of two and bigger than 1.
      * @throws {Error} FFT size must be a power of two and bigger than 1.
@@ -757,16 +757,3 @@ class FFT {
     }
 }
 
-module.exports = {
-    interpolate,
-    transpose: transpose_data,
-    softmax,
-    log_softmax,
-    getTopItems,
-    dot,
-    cos_sim,
-    magnitude,
-    min,
-    max,
-    FFT
-}