fix: MP4Clip has not adapted to the video track’s matrix settings

Author: hughfenghen

.changeset/plain-bears-remain.md

@@ -0,0 +1,5 @@
+---
+'@webav/av-cliper': patch
+---
+
+fix: MP4Clip has not adapted to the video track’s matrix settings

packages/av-cliper/src/clips/__tests__/mp4-clip.test.ts

@@ -1,7 +1,6 @@
 import mp4box, { MP4ArrayBuffer } from '@webav/mp4box.js';
 import { file, write } from 'opfs-tools';
 import { expect, test, vi } from 'vitest';
-import { parseMatrix } from '../../mp4-utils/mp4box-utils';
 import { MP4Clip } from '../mp4-clip';
 
 const mp4_123 = `//${location.host}/video/123.mp4`;
@@ -182,15 +181,6 @@ test('get file header data', async () => {
   );
 
   expect(boxfile.moov?.mvhd.matrix.length).toBe(9);
-  expect(parseMatrix(boxfile.moov?.mvhd.matrix!)).toEqual({
-    perspective: 1,
-    rotationDeg: 0,
-    rotationRad: 0,
-    scaleX: 1,
-    scaleY: 1,
-    translateX: 0,
-    translateY: 0,
-  });
 });
 
 test('decode incorrectFrameTypeMp4', async () => {

packages/av-cliper/src/clips/mp4-clip.ts

@@ -3,6 +3,7 @@ import { MP4Info, MP4Sample } from '@webav/mp4box.js';
 import { file, tmpfile, write } from 'opfs-tools';
 import { audioResample, extractPCM4AudioData, sleep } from '../av-utils';
 import {
+  createVFRotater,
   extractFileConfig,
   parseMatrix,
   quickParseMP4File,
@@ -108,13 +109,14 @@ export class MP4Clip implements IClip {
   /**存储视频平移旋转信息，目前只还原旋转 */
   #parsedMatrix = {
     perspective: 1,
-    rotationDeg: 0,
     rotationRad: 0,
+    rotationDeg: 0,
     scaleX: 1,
     scaleY: 1,
     translateX: 0,
     translateY: 0,
   };
+  #vfRotater: (vf: VideoFrame | null) => VideoFrame | null = (vf) => vf;
 
   #volume = 1;
 
@@ -206,7 +208,22 @@ export class MP4Clip implements IClip {
         this.#videoFrameFinder = videoFrameFinder;
         this.#audioFrameFinder = audioFrameFinder;
 
-        this.#meta = genMeta(decoderConf, videoSamples, audioSamples);
+        const { codedWidth, codedHeight } = decoderConf.video ?? {};
+        if (codedWidth && codedHeight) {
+          this.#vfRotater = createVFRotater(
+            codedWidth,
+            codedHeight,
+            parsedMatrix.rotationDeg,
+          );
+        }
+
+        this.#meta = genMeta(
+          decoderConf,
+          videoSamples,
+          audioSamples,
+          parsedMatrix.rotationDeg,
+        );
+
         this.#log.info('MP4Clip meta:', this.#meta);
         return { ...this.#meta };
       },
@@ -243,7 +260,7 @@ export class MP4Clip implements IClip {
 
     const [audio, video] = await Promise.all([
       this.#audioFrameFinder?.find(time) ?? [],
-      this.#videoFrameFinder?.find(time),
+      this.#videoFrameFinder?.find(time).then(this.#vfRotater),
     ]);
 
     if (video == null) {
@@ -476,6 +493,7 @@ function genMeta(
   decoderConf: MP4DecoderConf,
   videoSamples: ExtMP4Sample[],
   audioSamples: ExtMP4Sample[],
+  rotationDeg: number,
 ) {
   const meta = {
     duration: 0,
@@ -487,6 +505,11 @@ function genMeta(
   if (decoderConf.video != null && videoSamples.length > 0) {
     meta.width = decoderConf.video.codedWidth ?? 0;
     meta.height = decoderConf.video.codedHeight ?? 0;
+    // 90, 270 度，需要交换宽高
+    const normalizedRotation = (Math.round(rotationDeg / 90) * 90 + 360) % 360;
+    if (normalizedRotation === 90 || normalizedRotation === 270) {
+      [meta.width, meta.height] = [meta.height, meta.width];
+    }
   }
   if (decoderConf.audio != null && audioSamples.length > 0) {
     meta.audioSampleRate = DEFAULT_AUDIO_CONF.sampleRate;
@@ -551,8 +574,8 @@ async function mp4FileToSamples(otFile: OPFSToolFile, opts: IMP4ClipOpts = {}) {
   let headerBoxPos: Array<{ start: number; size: number }> = [];
   const parsedMatrix = {
     perspective: 1,
-    rotationDeg: 0,
     rotationRad: 0,
+    rotationDeg: 0,
     scaleX: 1,
     scaleY: 1,
     translateX: 0,
@@ -571,7 +594,7 @@ async function mp4FileToSamples(otFile: OPFSToolFile, opts: IMP4ClipOpts = {}) {
       const moov = data.mp4boxFile.moov!;
       headerBoxPos.push({ start: moov.start, size: moov.size });
 
-      Object.assign(parsedMatrix, parseMatrix(moov.mvhd.matrix));
+      Object.assign(parsedMatrix, parseMatrix(mp4Info.videoTracks[0]?.matrix));
 
       let { videoDecoderConf: vc, audioDecoderConf: ac } = extractFileConfig(
         data.mp4boxFile,
@@ -1562,4 +1585,36 @@ if (import.meta.vitest) {
     expect(normalized.size).toBe(1000);
     expect(normalized.is_sync).toBe(normalized.is_idr);
   });
+
+  it('genMeta adjusts width and height based on rotation', () => {
+    const meta = genMeta(
+      {
+        video: {
+          codedWidth: 1920,
+          codedHeight: 1080,
+        },
+        audio: null,
+      } as any,
+      [{ cts: 0, duration: 1000 }] as any,
+      [],
+      90,
+    );
+    expect(meta.width).toBe(1080);
+    expect(meta.height).toBe(1920);
+
+    const meta2 = genMeta(
+      {
+        video: {
+          codedWidth: 1920,
+          codedHeight: 1080,
+        },
+        audio: null,
+      } as any,
+      [{ cts: 0, duration: 1000 }] as any,
+      [],
+      180,
+    );
+    expect(meta2.width).toBe(1920);
+    expect(meta2.height).toBe(1080);
+  });
 }

packages/av-cliper/src/mp4-utils/__tests__/mp4-utils.test.ts

@@ -1,8 +1,12 @@
-import { beforeAll, describe, expect, test, vi } from 'vitest';
-import mp4box from '@webav/mp4box.js';
 import { autoReadStream, file2stream } from '@webav/internal-utils';
+import mp4box from '@webav/mp4box.js';
 import { file, write } from 'opfs-tools';
-import { quickParseMP4File } from '../mp4box-utils';
+import { beforeAll, describe, expect, test, vi } from 'vitest';
+import {
+  createVFRotater,
+  parseMatrix,
+  quickParseMP4File,
+} from '../mp4box-utils';
 
 beforeAll(() => {
   vi.useFakeTimers();
@@ -95,3 +99,118 @@ test('quickParseMP4File', async () => {
   expect(sampleCount).toBe(40);
   await reader.close();
 });
+
+test('vfRotater can be rotate VideoFrame instance', () => {
+  const vf = new VideoFrame(new Uint8Array(200 * 100 * 4), {
+    codedHeight: 100,
+    codedWidth: 200,
+    format: 'RGBA',
+    timestamp: 0,
+  });
+
+  // Test 90 degree rotation
+  const rotater90 = createVFRotater(200, 100, 90);
+  const rotatedVF90 = rotater90(vf.clone());
+  expect(rotatedVF90).not.toBeNull();
+  if (rotatedVF90 == null) throw new Error('must not be null');
+  expect(rotatedVF90.codedWidth).toBe(100);
+  expect(rotatedVF90.codedHeight).toBe(200);
+  rotatedVF90.close();
+
+  // Test 180 degree rotation
+  const rotater180 = createVFRotater(200, 100, 180);
+  const rotatedVF180 = rotater180(vf.clone());
+  expect(rotatedVF180).not.toBeNull();
+  if (rotatedVF180 == null) throw new Error('must not be null');
+  expect(rotatedVF180.codedWidth).toBe(200);
+  expect(rotatedVF180.codedHeight).toBe(100);
+  rotatedVF180.close();
+
+  // Test 270 degree rotation
+  const rotater270 = createVFRotater(200, 100, 270);
+  const rotatedVF270 = rotater270(vf.clone());
+  expect(rotatedVF270).not.toBeNull();
+  if (rotatedVF270 == null) throw new Error('must not be null');
+  expect(rotatedVF270.codedWidth).toBe(100);
+  expect(rotatedVF270.codedHeight).toBe(200);
+  rotatedVF270.close();
+
+  // Test 0 degree rotation
+  const rotater0 = createVFRotater(200, 100, 0);
+  const vfClone = vf.clone();
+  const rotatedVF0 = rotater0(vfClone);
+  // For 0 rotation, it should return the original frame
+  expect(rotatedVF0).toBe(vfClone);
+  rotatedVF0?.close();
+
+  vf.close();
+});
+
+describe('parseMatrix', () => {
+  test('should throw error for invalid matrix length', () => {
+    const matrix = new Int32Array(8);
+    expect(parseMatrix(matrix)).toEqual({});
+  });
+
+  test('should parse 0 degree rotation matrix', () => {
+    const matrix = new Int32Array([65536, 0, 0, 0, 65536, 0, 0, 0, 1073741824]);
+    const result = parseMatrix(matrix);
+    expect(result.rotationDeg).toBe(0);
+    expect(result.scaleX).toBe(1);
+    expect(result.scaleY).toBe(1);
+    expect(result.translateX).toBe(0);
+    expect(result.translateY).toBe(0);
+  });
+
+  test('should parse 90 degree rotation matrix', () => {
+    // matrix for 90 deg rotation
+    const matrix = new Int32Array([
+      0, 65536, 0, -65536, 0, 0, 0, 0, 1073741824,
+    ]);
+    const result = parseMatrix(matrix);
+    expect(result.rotationDeg).toBe(-90);
+    expect(result.scaleX).toBe(1);
+    expect(result.scaleY).toBe(1);
+  });
+
+  test('should parse 180 degree rotation matrix', () => {
+    const matrix = new Int32Array([
+      -65536, 0, 0, 0, -65536, 0, 0, 0, 1073741824,
+    ]);
+    const result = parseMatrix(matrix);
+    expect(result.rotationDeg).toBe(180);
+    expect(result.scaleX).toBe(1);
+    expect(result.scaleY).toBe(1);
+  });
+
+  test('should parse 270 degree rotation matrix', () => {
+    const matrix = new Int32Array([
+      0, -65536, 0, 65536, 0, 0, 0, 0, 1073741824,
+    ]);
+    const result = parseMatrix(matrix);
+    expect(result.rotationDeg).toBe(90);
+    expect(result.scaleX).toBe(1);
+    expect(result.scaleY).toBe(1);
+  });
+
+  test('should parse matrix with translation', () => {
+    const width = 1920;
+    const height = 1080;
+    // 180 deg rotation + translation
+    const matrix = new Int32Array([
+      -65536,
+      0,
+      0,
+      0,
+      -65536,
+      0,
+      width * 65536,
+      height * 65536,
+      1073741824,
+    ]);
+    const result = parseMatrix(matrix);
+    expect(result.rotationDeg).toBe(180);
+    expect(result.translateX).toBe(width);
+    expect(result.translateY).toBe(height);
+  });
+});

packages/av-cliper/src/mp4-utils/mp4box-utils.ts

@@ -180,19 +180,19 @@ export async function quickParseMP4File(
   }
 }
 
-export function parseMatrix(matrix: Uint32Array) {
-  if (matrix.length !== 9) {
-    throw new Error('Matrix must have 9 elements');
-  }
+export function parseMatrix(matrix?: Int32Array) {
+  if (matrix?.length !== 9) return {};
+
+  const signedMatrix = new Int32Array(matrix.buffer);
 
   // 提取并转成浮点数
-  const a = matrix[0] / 65536.0;
-  const b = matrix[1] / 65536.0;
-  const c = matrix[3] / 65536.0;
-  const d = matrix[4] / 65536.0;
-  const tx = matrix[6] / 65536.0; // 一般是 0
-  const ty = matrix[7] / 65536.0; // 一般是 0
-  const w = matrix[8] / (1 << 30); // 一般是 1
+  const a = signedMatrix[0] / 65536.0;
+  const b = signedMatrix[1] / 65536.0;
+  const c = signedMatrix[3] / 65536.0;
+  const d = signedMatrix[4] / 65536.0;
+  const tx = signedMatrix[6] / 65536.0; // 一般是 0
+  const ty = signedMatrix[7] / 65536.0; // 一般是 0
+  const w = signedMatrix[8] / (1 << 30); // 一般是 1
 
   // 缩放
   const scaleX = Math.sqrt(a * a + c * c);
@@ -212,3 +212,39 @@ export function parseMatrix(matrix: Uint32Array) {
     perspective: w,
   };
 }
+
+/**
+ * 旋转 VideoFrame
+ */
+export function createVFRotater(
+  width: number,
+  height: number,
+  rotationDeg: number,
+) {
+  const normalizedRotation = (Math.round(rotationDeg / 90) * 90 + 360) % 360;
+  if (normalizedRotation === 0) return (vf: VideoFrame | null) => vf;
+
+  const rotatedWidth =
+    normalizedRotation === 90 || normalizedRotation === 270 ? height : width;
+  const rotatedHeight =
+    normalizedRotation === 90 || normalizedRotation === 270 ? width : height;
+
+  const canvas = new OffscreenCanvas(rotatedWidth, rotatedHeight);
+  const ctx = canvas.getContext('2d')!;
+
+  ctx.translate(rotatedWidth / 2, rotatedHeight / 2);
+  ctx.rotate((normalizedRotation * Math.PI) / 180);
+  ctx.translate(-width / 2, -height / 2);
+
+  return (vf: VideoFrame | null) => {
+    if (vf == null) return null;
+
+    ctx.drawImage(vf, 0, 0);
+    const newVF = new VideoFrame(canvas, {
+      timestamp: vf.timestamp,
+      duration: vf.duration ?? 0,
+    });
+    vf.close();
+    return newVF;
+  };
+}

types/mp4box.d.ts

@@ -18,6 +18,7 @@ declare module '@webav/mp4box.js' {
   }
 
   export interface MP4VideoTrack extends MP4MediaTrack {
+    matrix: Int32Array;
     video: {
       width: number;
       height: number;