feat(android): add video texture support

Author: triniwiz

packages/canvas-media/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nativescript/canvas-media",
-	"version": "0.4.0",
+	"version": "0.5.0",
 	"description": "Canvas media",
 	"main": "index",
 	"typings": "index.d.ts",

packages/canvas-media/video/index.android.ts

@@ -22,11 +22,11 @@ export class Video extends VideoBase {
 	#playerView: com.google.android.exoplayer2.ui.PlayerView;
 	#player: com.google.android.exoplayer2.SimpleExoPlayer;
 	#playerListener: com.google.android.exoplayer2.Player.EventListener;
+	#videoListener: com.google.android.exoplayer2.video.VideoListener;
 	#src: string;
 	#autoplay: boolean;
 	#loop: boolean;
 	#textureView: android.view.TextureView;
-
 	_isCustom: boolean = false;
 	_playing: boolean = false;
 	_timer: any;
@@ -37,9 +37,12 @@ export class Video extends VideoBase {
 	_canvas: any;
 	_frameTimer: any;
 	_readyState: number = 0;
+	_videoWidth = 0;
+	_videoHeight = 0;
 	constructor() {
 		super();
 		java.lang.System.loadLibrary('canvasnative');
+		//@ts-ignore
 		const builder = new com.google.android.exoplayer2.SimpleExoPlayer.Builder(Utils.ad.getApplicationContext());
 		this.#player = builder.build();
 		const ref = new WeakRef(this);
@@ -102,6 +105,17 @@ export class Video extends VideoBase {
 			onTimelineChanged(timeline: com.google.android.exoplayer2.Timeline, manifest: any, reason: number) {},
 			onTracksChanged: function (trackGroups, trackSelections) {},
 		});
+		this.#videoListener = new com.google.android.exoplayer2.video.VideoListener({
+			onRenderedFirstFrame() {},
+			onVideoSizeChanged(width: number, height: number, unappliedRotationDegrees: number, pixelWidthHeightRatio: number) {
+				const owner = ref.get();
+				if (owner) {
+					owner._videoWidth = width;
+					owner._videoHeight = height;
+				}
+			},
+			onSurfaceSizeChanged(width: number, height: number) {},
+		});
 		const activity: androidx.appcompat.app.AppCompatActivity = Application.android.foregroundActivity || Application.android.startActivity;
 		const inflator = activity.getLayoutInflater();
 		const layout = Video.getResourceId(Application.android.foregroundActivity || Application.android.startActivity, 'player');
@@ -111,9 +125,8 @@ export class Video extends VideoBase {
 		const params = new android.widget.LinearLayout.LayoutParams(android.widget.LinearLayout.LayoutParams.MATCH_PARENT, android.widget.LinearLayout.LayoutParams.MATCH_PARENT);
 		this.#textureView = new android.view.TextureView(Application.android.foregroundActivity || Application.android.startActivity);
 		this.#container.addView(this.#textureView as any, params);
-		//this.#playerView.setPlayer(this.#player);
-		//this.#player.setVideoTextureView(this.#textureView);
 		this.setNativeView(this.#container);
+		this.#player.addVideoListener(this.#videoListener);
 	}
 
 	get readyState() {
@@ -139,7 +152,6 @@ export class Video extends VideoBase {
 	}
 	static st_count = 0;
 	static createCustomView() {
-		const canvas = require('@nativescript/canvas');
 		const video = new Video();
 		video._isCustom = true;
 		video.width = 300;
@@ -164,40 +176,48 @@ export class Video extends VideoBase {
 
 	_hasFrame = false;
 	getCurrentFrame(context?: WebGLRenderingContext) {
-		const surfaceView = this.#playerView.getVideoSurfaceView();
-		if (surfaceView instanceof android.view.TextureView) {
-			if (!this._st) {
-				// @ts-ignore
-				const result = com.github.triniwiz.canvas.Utils.createSurfaceTexture(context);
-				this._st = result[0];
-				const ref = new WeakRef(this);
-				this._frameListener = new android.graphics.SurfaceTexture.OnFrameAvailableListener({
-					onFrameAvailable(param0: android.graphics.SurfaceTexture) {
-						const owner = ref.get();
-						if (owner) {
-							owner._hasFrame = true;
-							owner._notifyVideoFrameCallbacks();
-						}
-					},
-				});
-				this._st.setOnFrameAvailableListener(this._frameListener);
-
-				//this.#textureView.setSurfaceTexture(this._st);
-				this._surface = new android.view.Surface(this._st);
-				this.#player.setVideoSurface(this._surface);
-				this._render = result[1];
+		if (this.isLoaded) {
+			const surfaceView = this.#playerView.getVideoSurfaceView();
+			if (surfaceView instanceof android.view.TextureView) {
+				const st = surfaceView.getSurfaceTexture();
+				if (st) {
+					// @ts-ignore
+					this._render = com.github.triniwiz.canvas.Utils.createRenderAndAttachToGLContext(context, st);
+					this._st = st;
+				}
 			}
+		}
 
-			if (this._st) {
-				if (!this._hasFrame) {
-					return;
-				}
+		if (!this._st) {
+			// @ts-ignore
+			const result = com.github.triniwiz.canvas.Utils.createSurfaceTexture(context);
+			this._st = result[0];
+			const ref = new WeakRef(this);
+			this._frameListener = new android.graphics.SurfaceTexture.OnFrameAvailableListener({
+				onFrameAvailable(param0: android.graphics.SurfaceTexture) {
+					const owner = ref.get();
+					if (owner) {
+						owner._hasFrame = true;
+						owner._notifyVideoFrameCallbacks();
+					}
+				},
+			});
 
-				// @ts-ignore
-				com.github.triniwiz.canvas.Utils.updateTexImage(context, this._st, this._render);
+			this._st.setOnFrameAvailableListener(this._frameListener);
 
-				this._hasFrame = false;
+			this._surface = new android.view.Surface(this._st);
+			this.#player.setVideoSurface(this._surface);
+			this._render = result[1];
+		}
+
+		if (this._st) {
+			if (!this._hasFrame) {
+				return;
 			}
+			// @ts-ignore
+			com.github.triniwiz.canvas.Utils.updateTexImage(context, this._st, this._render, this._videoWidth, this._videoHeight, arguments[4], arguments[5]);
+
+			this._hasFrame = false;
 		}
 	}
 
@@ -274,6 +294,7 @@ export class Video extends VideoBase {
 		if (!this.#playerView) {
 			this.#playerView = new com.google.android.exoplayer2.ui.PlayerView(this._context);
 		}
+		this.#playerView.setPlayer(this.#player);
 		return this.#playerView;
 	}
 

packages/canvas/Canvas/common.ts

@@ -1,6 +1,6 @@
 import {CSSType, PercentLength, View, Screen, GestureStateTypes, Utils, Application} from '@nativescript/core';
 import {CanvasRenderingContext, TouchList} from '../common';
-import {Pointer} from '@nativescript/core/ui/gestures';
+import {PinchGestureEventData, Pointer} from '@nativescript/core/ui/gestures';
 
 export interface ICanvasBase {
 	on(eventName: 'ready', callback: (data: any) => void, thisArg?: any): void;
@@ -92,6 +92,7 @@ export abstract class CanvasBase extends View implements ICanvasBase {
 
 	__touchStart?: Pointer;
 
+	_isPinching = false;
 	_touchEventsFN(event: any) {
 		if (event.eventName === 'touch') {
 			switch (event.action) {
@@ -114,8 +115,15 @@ export abstract class CanvasBase extends View implements ICanvasBase {
 					break;
 			}
 		} else if (event.eventName === 'pinch') {
+			this._isPinching = true;
 			this._emitEvent('touchmove:pinch', event);
+			if(event.state === GestureStateTypes.ended || event.state === GestureStateTypes.cancelled){
+				this._isPinching = false;
+			}
 		} else if (event.eventName === 'pan') {
+			if(this._isPinching){
+				return;
+			}
 			if (event.state === GestureStateTypes.began || event.state === GestureStateTypes.changed) {
 				this._emitEvent('touchmove', event);
 			}
@@ -205,20 +213,21 @@ export abstract class CanvasBase extends View implements ICanvasBase {
 			};
 
 			pointers.push({
-				clientX: this.__touchStart.getX(),
-				clientY: this.__touchStart.getY(),
+				clientX: this.__touchStart?.getX() ?? 0,
+				clientY: this.__touchStart?.getY() ?? 0,
 				force: 0.0,
 				identifier: 0,
-				pageX: this.__touchStart.getX(),
-				pageY: this.__touchStart.getY(),
+				pageX: this.__touchStart?.getX() ?? 0,
+				pageY: this.__touchStart?.getY() ?? 0,
 				radiusX: 0,
 				radiusY: 0,
 				rotationAngle: 0,
-				screenX: this.__touchStart.getX(),
-				screenY: this.__touchStart.getY(),
+				screenX: this.__touchStart?.getX() ?? 0,
+				screenY: this.__touchStart?.getY() ?? 0,
 				target,
 			});
 
+
 			pointers.push({
 				clientX: x,
 				clientY: y,
@@ -233,6 +242,7 @@ export abstract class CanvasBase extends View implements ICanvasBase {
 				screenY: y,
 				target,
 			});
+
 		} else {
 			const count = event.getAllPointers().length;
 			const point = event.getActivePointers()[0];

packages/canvas/WebGL/WebGLRenderingContext/index.android.ts

@@ -133,10 +133,20 @@ export class WebGLRenderingContext extends WebGLRenderingContextBase {
 		this.context.bindRenderbuffer(target, value);
 	}
 
+	_lastTexture : {
+		target: number,
+		texture: number
+	} = {target: 0, texture: 0}
 	bindTexture(target: number, texture: WebGLTexture): void {
 		this._glCheckError('bindTexture');
 		this._checkArgs('bindTexture', arguments);
 		const value = texture ? texture.native : 0;
+		if(value > 0){
+			this._lastTexture = {
+				target, texture: value
+			};
+		}
+
 		this.context.bindTexture(target, value);
 	}
 
@@ -1026,9 +1036,9 @@ export class WebGLRenderingContext extends WebGLRenderingContextBase {
 			}
 		} else if (arguments.length === 6) {
 			if (border && typeof border.tagName === 'string' && (border.tagName === 'VID' || border.tagName === 'VIDEO') && typeof border._video.getCurrentFrame === 'function') {
-				border._video.getCurrentFrame(this.context);
+				border._video.getCurrentFrame(this.context, this, target, level, internalformat, width, height);
 			} else if (border && typeof border.getCurrentFrame === 'function') {
-				border.getCurrentFrame(this.context);
+				border.getCurrentFrame(this.context, this, target, level, internalformat, width, height);
 			} else if (border instanceof ImageAsset) {
 				this.context.texImage2D(target, level, internalformat, width, height, border.native);
 			} else if (border instanceof android.graphics.Bitmap) {

packages/canvas/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nativescript/canvas",
-	"version": "0.9.10",
+	"version": "0.9.12",
 	"description": "DOM Canvas API for NativeScript",
 	"main": "index",
 	"typings": "index.d.ts",

packages/canvas/platforms/android/include.gradle

@@ -1,3 +1,3 @@
 dependencies {
-	implementation 'com.github.triniwiz:canvas:0.9.10'
+	implementation 'com.github.triniwiz:canvas:0.9.11'
 }

packages/canvas/src-native/canvas-android/.idea/dictionaries/triniwiz.xml

@@ -36,4 +36,6 @@ dependencies {
 	androidTestImplementation 'androidx.test:runner:1.2.0'
 	androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0'
 	implementation project(path: ':canvas')
+	implementation 'com.google.android.exoplayer:exoplayer:2.13.2'
+	implementation 'com.google.android.exoplayer:exoplayer-ui:2.13.2'
 }