Merge pull request #59 from peerless2012/dev_overlay_gl

Add open gl render mode in overlay.

Author: peerless2012

app/src/main/java/io/github/peerless2012/ass/demo/MainActivity.kt

@@ -27,7 +27,7 @@ import androidx.core.net.toUri
 
 class MainActivity : AppCompatActivity() {
 
-    private var url = "http://192.168.0.254:80/files/c.mkv"
+    private var url = "http://192.168.3.6:8080/files/c.mkv"
 
     private lateinit var player: ExoPlayer
 
@@ -50,27 +50,27 @@ class MainActivity : AppCompatActivity() {
         player = ExoPlayer.Builder(this)
             .buildWithAssSupport(
                 this,
-                AssRenderType.OVERLAY,
+                AssRenderType.OVERLAY_OPEN_GL,
                 playerView.subtitleView
             )
         playerView.player = player
         val enConfig = MediaItem.SubtitleConfiguration
-            .Builder(Uri.parse("http://192.168.0.254:80/files/e.ass"))
+            .Builder(Uri.parse("http://192.168.3.6:8080/files/e.ass"))
             .setMimeType(MimeTypes.TEXT_SSA)
             .setLanguage("en")
             .setLabel("External ass en")
             .setId("129")
             .setSelectionFlags(C.SELECTION_FLAG_DEFAULT)
             .build()
         val jpConfig = MediaItem.SubtitleConfiguration
-            .Builder(Uri.parse("http://192.168.0.254:80/files/f-jp.ass"))
+            .Builder(Uri.parse("http://192.168.3.6:8080/files/f-jp.ass"))
             .setMimeType(MimeTypes.TEXT_SSA)
             .setLanguage("jp")
             .setLabel("External ass jp")
             .setId("130")
             .build()
         val zhConfig = MediaItem.SubtitleConfiguration
-            .Builder(Uri.parse("http://192.168.0.254:80/files/f-zh.ass"))
+            .Builder(Uri.parse("http://192.168.3.6:8080/files/f-zh.ass"))
             .setMimeType(MimeTypes.TEXT_SSA)
             .setLanguage("zh")
             .setLabel("External ass zh")

gradle.properties

@@ -24,7 +24,7 @@ android.nonTransitiveRClass=true
 
 # maven publish
 GROUP=io.github.peerless2012
-VERSION_NAME=0.3.0
+VERSION_NAME=0.4.0-alpha01
 
 POM_URL=https://github.com/peerless2012/libass-android
 POM_INCEPTION_YEAR=2025

gradle/libs.versions.toml

@@ -1,5 +1,5 @@
 [versions]
-agp = "8.12.0"
+agp = "8.13.1"
 kotlin = "2.2.0"
 coreKtx = "1.16.0"
 annotation = "1.9.1"

lib_ass_kt/src/main/cpp/AssKt.c

@@ -9,6 +9,8 @@
 #include <jni.h>
 #include "ass/ass.h"
 #include "fontconfig/fontconfig.h"
+#include "GLES2/gl2.h"
+#include "GLES2/gl2ext.h"
 
 #define LOG_TAG "SubtitleRenderer"
 
@@ -265,6 +267,24 @@ jobject createAlphaBitmap(JNIEnv* env, const ASS_Image* image) {
     return bitmap;
 }
 
+jint createTexture(JNIEnv* env, const ASS_Image* image) {
+    GLuint texture;
+    glGenTextures(1, &texture);
+    if (texture <= 0) return 0;
+    glBindTexture(GL_TEXTURE_2D, texture);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+
+    glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
+    glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, image->stride);
+    glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, image->w, image->h, 0, GL_ALPHA, GL_UNSIGNED_BYTE, image->bitmap);
+    glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, 0);
+    glBindTexture(GL_TEXTURE_2D, 0);
+    return texture;
+}
+
 static int count_ass_images(ASS_Image *images) {
     int count = 0;
     for (ASS_Image *img = images; img != NULL; img = img->next) {
@@ -273,7 +293,7 @@ static int count_ass_images(ASS_Image *images) {
     return count;
 }
 
-jobject nativeAssRenderFrame(JNIEnv* env, jclass clazz, jlong render, jlong track, jlong time, jboolean onlyAlpha) {
+jobject nativeAssRenderFrame(JNIEnv* env, jclass clazz, jlong render, jlong track, jlong time, jint type) {
     int changed;
     ASS_Image *image = ass_render_frame((ASS_Renderer *) render, (ASS_Track *) track, time, &changed);
     if (image == NULL) {
@@ -289,7 +309,7 @@ jobject nativeAssRenderFrame(JNIEnv* env, jclass clazz, jlong render, jlong trac
 
     int size = count_ass_images(image);
     jclass assTexClass = (*env)->FindClass(env, "io/github/peerless2012/ass/AssTex");
-    jmethodID assTexConstructor = (*env)->GetMethodID(env, assTexClass, "<init>", "(IIILandroid/graphics/Bitmap;)V");
+    jmethodID assTexConstructor = (*env)->GetMethodID(env, assTexClass, "<init>", "(IIIIILandroid/graphics/Bitmap;I)V");
 
     jobjectArray assTexArr = (*env)->NewObjectArray(env, size, assTexClass, NULL);
     if (assTexArr == NULL) {
@@ -299,13 +319,18 @@ jobject nativeAssRenderFrame(JNIEnv* env, jclass clazz, jlong render, jlong trac
     int index = 0;
     for (ASS_Image *img = image; img != NULL; img = img->next) {
         jobject bitmap = NULL;
+        jint tex = 0;
         if (img->w > 0 && img->h > 0) {
-            bitmap = onlyAlpha ? createAlphaBitmap(env, img) : createBitmap(env, img);
+            if (type == 0) {
+                bitmap = createBitmap(env, img);
+            } else if (type == 1) {
+                bitmap = createAlphaBitmap(env, img);
+            } else if (type == 2) {
+               tex = createTexture(env, img);
+            }
         }
         int32_t color = (int32_t) img->color;
-
-        jobject assTexObject = (*env)->NewObject(env, assTexClass, assTexConstructor, img->dst_x, img->dst_y, color, bitmap);
-
+        jobject assTexObject = (*env)->NewObject(env, assTexClass, assTexConstructor, img->dst_x, img->dst_y, img->w, img->h, color, bitmap, tex);
         (*env)->SetObjectArrayElement(env, assTexArr, index, assTexObject);
         (*env)->DeleteLocalRef(env, assTexObject);
         if (bitmap != NULL) {
@@ -329,7 +354,7 @@ static JNINativeMethod renderMethodTable[] = {
         {"nativeAssRenderSetCacheLimit", "(JII)V", (void*)nativeAssRenderSetCacheLimit},
         {"nativeAssRenderSetStorageSize", "(JII)V", (void*) nativeAssRenderSetStorageSize},
         {"nativeAssRenderSetFrameSize", "(JII)V", (void*)nativeAssRenderSetFrameSize},
-        {"nativeAssRenderFrame", "(JJJZ)Lio/github/peerless2012/ass/AssFrame;", (void*) nativeAssRenderFrame},
+        {"nativeAssRenderFrame", "(JJJI)Lio/github/peerless2012/ass/AssFrame;", (void*) nativeAssRenderFrame},
         {"nativeAssRenderDeinit", "(J)V", (void*)nativeAssRenderDeinit},
 };
 JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) {

lib_ass_kt/src/main/cpp/CMakeLists.txt

@@ -32,5 +32,6 @@ target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE
         lib_ass::ass
         android
         jnigraphics
+        GLESv2
         z
         log)

lib_ass_kt/src/main/java/io/github/peerless2012/ass/AssRender.kt

@@ -27,7 +27,7 @@ class AssRender(nativeAss: Long) {
         external fun nativeAssRenderSetFrameSize(render: Long, width: Int, height: Int)
 
         @JvmStatic
-        external fun nativeAssRenderFrame(render: Long, track: Long, time: Long, onlyAlpha: Boolean): AssFrame?
+        external fun nativeAssRenderFrame(render: Long, track: Long, time: Long, type: Int): AssFrame?
 
         @JvmStatic
         external fun nativeAssRenderDeinit(render: Long)
@@ -57,8 +57,8 @@ class AssRender(nativeAss: Long) {
         nativeAssRenderSetFrameSize(nativeRender, width, height)
     }
 
-    public fun renderFrame(time: Long, onlyAlpha: Boolean): AssFrame? {
-        return track?.let { nativeAssRenderFrame(nativeRender, it.nativeAssTrack, time, onlyAlpha) }
+    public fun renderFrame(time: Long, type: AssTexType): AssFrame? {
+        return track?.let { nativeAssRenderFrame(nativeRender, it.nativeAssTrack, time, type.ordinal) }
     }
 
     protected fun finalize() {

lib_ass_kt/src/main/java/io/github/peerless2012/ass/AssTex.kt

@@ -9,4 +9,4 @@ import android.graphics.Bitmap
  * @Version V1.0
  * @Description
  */
-data class AssTex(val x: Int, val y: Int, val color: Int, val bitmap: Bitmap? = null)
+data class AssTex(val x: Int, val y: Int, val w: Int, val h: Int, val color: Int, val bitmap: Bitmap? = null, val tex: Int = 0)

lib_ass_kt/src/main/java/io/github/peerless2012/ass/AssTexType.kt

@@ -0,0 +1,11 @@
+package io.github.peerless2012.ass
+
+enum class AssTexType {
+
+    BITMAP_RGBA,
+
+    BITMAP_ALPHA,
+
+    TEXTURE
+
+}

lib_ass_media/README.md

@@ -7,33 +7,50 @@ App use media3 can use this module to add ass for your player.
 There are three ways to render ass subtitle.
 Which is defined in `AssRenderType`.
 
-| Type | Feature | Anim | HDR/DV |
-| :----: | :----: | :----: | :----: |
-| LEGACY | SubtitleView & Cue | ❌ | ✅ |
-| CANVAS | Effect | ✅ | ❓ |
-| OPEN_GL | Effect | ✅ | ❓ |
+| Type | Feature | Anim | HDR/DV | Block render/UI |
+| :----: | :----: | :----: | :----: | :----: |
+| CUES | SubtitleView & Cue | ❌ | ✅ | ❌ |
+| EFFECTS_CANVAS | Effect | ✅ | ❓ | ✅ |
+| EFFECTS_OPEN_GL | Effect | ✅ | ❓ | ✅ |
+| OVERLAY_CANVAS | Overlay | ✅ | ✅ | ✅ |
+| OVERLAY_OPEN_GL | Overlay | ✅ | ✅ | ❌ |
 
 * [OverlayShaderProgram does not support HDR colors yet](https://github.com/androidx/media/issues/723)
 * [Why does TextOverLay support hdr, but Bitmap not support?](https://github.com/androidx/media/issues/2383)
 
-### 1. LEGACY
+### 1. CUES
 The ass/ssa subtitle will be parsed and transcode to bytes, and decode to bitmap when render.
 
 This type not support dynamic feature, because all subtitle and it time is static.
 
 But since the subtitle is transcode, it will not cost too much time when render. All work is done in parse thread.
 
-### 2. CANVAS
+### 2. EFFECTS_CANVAS
 The ass/ssa subtitle will be cal and render at runtime use media3 effect feature, and this will support all dynamic features.
 
 And this need to create a screen size offscreen bitmap to render the libass bitmap pieces.
 
 But when the dynamic feature is too complex, and libass will cost too much time to cal, the render will be blocked.
 
-### 3. OPEN_GL
-Just like `CANVAS`, but use OpenGL to render. and the offscreen tex is create to render the bitmap pieces.
+### 3. EFFECTS_OPEN_GL
+Just like `EFFECTS_CANVAS`, but use OpenGL to render. and the offscreen tex is create to render the bitmap pieces.
+
+Due to test, the `EFFECTS_OPEN_GL` will save 1/3 time when render.
+
+### 4. OVERLAY_CANVAS
+The ass/ssa subtitle will be cal at runtime, and add a `Overlay` widget in `SubtitleView` to render subtitle.
+
+The `libass` render result will copy to bitmap, and draw in `Canvas`.
+
+It will block UI thread when rendering.
+
+### 4. OVERLAY_OPEN_GL
+Just like `OVERLAY_CANVAS`, but the `libass` render result will pass to `OpenGL` texture, and avoid create tmp bitmap.
+
+It will save half memory than `OVERLAY_CANVAS`.
+
+And the `libass` render and `OpenGL` draw on another separate thread, it will not block the UI thread like `OVERLAY_CANVAS`.
 
-Due to test, the `OPEN_GL` will save 1/3 time when render.
 
 ## How to use
 1. Add MavenCenter to your project

lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandler.kt

@@ -19,8 +19,6 @@ import io.github.peerless2012.ass.Ass
 import io.github.peerless2012.ass.media.parser.AssHeaderParser
 import io.github.peerless2012.ass.media.render.AssOverlayManager
 import io.github.peerless2012.ass.media.type.AssRenderType
-import kotlin.math.floor
-import kotlin.math.roundToInt
 
 /**
  * Handles ASS subtitle rendering and integration with ExoPlayer.
@@ -98,8 +96,8 @@ class AssHandler(val renderType: AssRenderType) : Listener {
     fun init(player: ExoPlayer) {
         player.addListener(this)
         handler = Handler(player.applicationLooper)
-        if (renderType == AssRenderType.CANVAS || renderType == AssRenderType.OPEN_GL) {
-            overlayManager = AssOverlayManager(this, player, renderType == AssRenderType.OPEN_GL)
+        if (renderType == AssRenderType.EFFECTS_CANVAS || renderType == AssRenderType.EFFECTS_OPEN_GL) {
+            overlayManager = AssOverlayManager(this, player, renderType == AssRenderType.EFFECTS_OPEN_GL)
         }
     }
 
@@ -162,7 +160,7 @@ class AssHandler(val renderType: AssRenderType) : Listener {
         this.track = track
         val render = requireNotNull(render)
         render.setStorageSize(videoSize.width, videoSize.height)
-        if (renderType == AssRenderType.OVERLAY) {
+        if (renderType == AssRenderType.OVERLAY_CANVAS || renderType == AssRenderType.OVERLAY_OPEN_GL) {
             render.setFrameSize(surfaceSize.width, surfaceSize.height)
         } else {
             render.setFrameSize(videoSize.width, videoSize.height)
@@ -186,7 +184,7 @@ class AssHandler(val renderType: AssRenderType) : Listener {
         Log.i("AssHandler", "onSurfaceSizeChanged: width = $width, height = $height")
         if (surfaceSize.width == width && surfaceSize.height == height) return
         surfaceSize = Size(width, height)
-        if (renderType == AssRenderType.OVERLAY && surfaceSize.isValid) {
+        if ((renderType == AssRenderType.OVERLAY_CANVAS || renderType == AssRenderType.OVERLAY_OPEN_GL) && surfaceSize.isValid) {
             render?.setFrameSize(surfaceSize.width, surfaceSize.height)
         }
     }
@@ -229,7 +227,7 @@ class AssHandler(val renderType: AssRenderType) : Listener {
 
         val track = ass.createTrack()
         if (format.initializationData.size > 0) {
-            val header = AssHeaderParser.parse(format, renderType != AssRenderType.LEGACY)
+            val header = AssHeaderParser.parse(format, renderType != AssRenderType.CUES)
             track.readBuffer(header)
         }
         availableTracks[format.id!!] = track
@@ -252,7 +250,7 @@ class AssHandler(val renderType: AssRenderType) : Listener {
             if (videoSize.isValid) {
                 render.setFrameSize(videoSize.width, videoSize.height)
             }
-            if (renderType == AssRenderType.OVERLAY) {
+            if (renderType == AssRenderType.OVERLAY_CANVAS || renderType == AssRenderType.OVERLAY_OPEN_GL) {
                 if (surfaceSize.isValid) {
                     render.setFrameSize(surfaceSize.width, surfaceSize.height)
                 }