[moonshine] Update config values for transformers v4.48.0 (#1155)

* Update config values for transformers v4.48.0

* Separate wav2vec2 and wav2vec2 with lm processors

* Add moonshine modelling unit tests

Author: xenova

src/configs.js

@@ -198,12 +198,17 @@ function getNormalizedConfig(config) {
             mapping['encoder_hidden_size'] = mapping['decoder_hidden_size'] = 'd_model';
             break;
         case 'musicgen_decoder':
-        case 'moonshine':
             mapping['num_encoder_layers'] = mapping['num_decoder_layers'] = 'num_hidden_layers';
             mapping['num_encoder_heads'] = mapping['num_decoder_heads'] = 'num_attention_heads';
             mapping['encoder_hidden_size'] = mapping['decoder_hidden_size'] = 'hidden_size';
             break;
-
+        case 'moonshine':
+            mapping['num_decoder_layers'] = 'decoder_num_hidden_layers';
+            mapping['num_decoder_heads'] = 'decoder_num_key_value_heads';
+            mapping['num_encoder_layers'] = 'encoder_num_hidden_layers';
+            mapping['num_encoder_heads'] = 'encoder_num_key_value_heads';
+            mapping['encoder_hidden_size'] = mapping['decoder_hidden_size'] = 'hidden_size';
+            break;
         case 'vision-encoder-decoder':
             // @ts-expect-error TS2339
             const decoderConfig = getNormalizedConfig(config.decoder);

src/models/processors.js

@@ -13,4 +13,5 @@ export * from './qwen2_vl/processing_qwen2_vl.js';
 export * from './sam/processing_sam.js';
 export * from './speecht5/processing_speecht5.js';
 export * from './wav2vec2/processing_wav2vec2.js';
+export * from './wav2vec2_with_lm/processing_wav2vec2_with_lm.js';
 export * from './whisper/processing_whisper.js';

src/models/wav2vec2/processing_wav2vec2.js

@@ -1,7 +1,9 @@
-import { Processor } from "../../base/processing_utils.js";
+import { AutoTokenizer } from "../../tokenizers.js";
 import { AutoFeatureExtractor } from "../auto/feature_extraction_auto.js";
+import { Processor } from "../../base/processing_utils.js";
 
-export class Wav2Vec2ProcessorWithLM extends Processor {
+export class Wav2Vec2Processor extends Processor {
+    static tokenizer_class = AutoTokenizer
     static feature_extractor_class = AutoFeatureExtractor
 
     /**

src/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.js

@@ -0,0 +1,17 @@
+import { AutoTokenizer } from "../../tokenizers.js";
+import { AutoFeatureExtractor } from "../auto/feature_extraction_auto.js";
+import { Processor } from "../../base/processing_utils.js";
+
+export class Wav2Vec2ProcessorWithLM extends Processor {
+    static tokenizer_class = AutoTokenizer
+    static feature_extractor_class = AutoFeatureExtractor
+
+    /**
+     * Calls the feature_extractor function with the given audio input.
+     * @param {any} audio The audio input to extract features from.
+     * @returns {Promise<any>} A Promise that resolves with the extracted features.
+     */
+    async _call(audio) {
+        return await this.feature_extractor(audio)
+    }
+}

tests/models/moonshine/test_modeling_moonshine.js

@@ -0,0 +1,50 @@
+import { Wav2Vec2Processor, MoonshineForConditionalGeneration, full, ones } from "../../../src/transformers.js";
+
+import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js";
+
+export default () => {
+  describe("MoonshineForConditionalGeneration", () => {
+    const model_id = "hf-internal-testing/tiny-random-MoonshineForConditionalGeneration";
+
+    /** @type {MoonshineForConditionalGeneration} */
+    let model;
+    /** @type {Wav2Vec2Processor} */
+    let processor;
+    beforeAll(async () => {
+      model = await MoonshineForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS);
+      processor = await Wav2Vec2Processor.from_pretrained(model_id);
+    }, MAX_MODEL_LOAD_TIME);
+
+    const input_values = new Float32Array(16000);
+
+    it(
+      "forward",
+      async () => {
+        const inputs = await processor(input_values);
+        const { logits } = await model({
+          ...inputs,
+          decoder_input_ids: ones([1, 1]),
+        });
+        expect(logits.dims).toEqual([1, 1, 32768]);
+        expect(logits.mean().item()).toBeCloseTo(0.016709428280591965, 6);
+      },
+      MAX_TEST_EXECUTION_TIME,
+    );
+
+    it(
+      "batch_size=1",
+      async () => {
+        const inputs = await processor(input_values);
+        const generate_ids = await model.generate({ ...inputs, max_new_tokens: 3 });
+
+        const new_tokens = generate_ids;
+        expect(new_tokens.tolist()).toEqual([[/* Decoder start token */ 1n, /* Generated */ 6891n, 21892n, 14850n]]);
+      },
+      MAX_TEST_EXECUTION_TIME,
+    );
+
+    afterAll(async () => {
+      await model?.dispose();
+    }, MAX_MODEL_DISPOSE_TIME);
+  });
+};