fix: Fix loading TFLite models from Resources in release on Android (#30)

* fix: Fix loading TFLite models from Resources in release on Android

* Update TfliteModule.java

* Try parsing URI

* Update TfliteModule.java

* Update Podfile.lock

* fix: Fix metro and scale

* Update AndroidManifest.xml

* Update worklets

Author: mrousavy

android/src/main/cpp/Tflite.cpp

@@ -46,13 +46,6 @@ struct TfliteModule : public jni::JavaClass<TfliteModule> {
 
       auto byteData = method(cls, url);
 
-      // TODO: to review by someone experienced much more in C++
-      // Detaching current thread causes app crash with exception:
-      // "Unable to retrieve jni environment. is the thread attached?"
-      // anyway, there is still a risk of memory leakage without calling the function below:
-
-      // java_machine->DetachCurrentThread();
-
       auto size = byteData->size();
       auto bytes = byteData->getRegion(0, size);
       void* data = malloc(size);

android/src/main/java/com/tflite/TfliteModule.java

@@ -1,18 +1,24 @@
 package com.tflite;
 
+import android.annotation.SuppressLint;
+import android.content.Context;
+import android.net.Uri;
 import android.util.Log;
 
 import androidx.annotation.NonNull;
 
 import com.facebook.proguard.annotations.DoNotStrip;
 import com.facebook.react.bridge.JavaScriptContextHolder;
-import com.facebook.react.bridge.Promise;
 import com.facebook.react.bridge.ReactApplicationContext;
 import com.facebook.react.bridge.ReactContextBaseJavaModule;
 import com.facebook.react.bridge.ReactMethod;
 import com.facebook.react.module.annotations.ReactModule;
 import com.facebook.react.turbomodule.core.CallInvokerHolderImpl;
 
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.lang.ref.WeakReference;
+
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import okhttp3.Response;
@@ -21,9 +27,12 @@
 @ReactModule(name = TfliteModule.NAME)
 public class TfliteModule extends ReactContextBaseJavaModule {
   public static final String NAME = "Tflite";
+  private static WeakReference<ReactApplicationContext> weakContext;
+  private static final OkHttpClient client = new OkHttpClient();
 
   public TfliteModule(ReactApplicationContext reactContext) {
     super(reactContext);
+    weakContext = new WeakReference<>(reactContext);
   }
 
   @Override
@@ -32,21 +41,64 @@ public String getName() {
     return NAME;
   }
 
+  @SuppressLint("DiscouragedApi")
+  private static int getResourceId(Context context, String name) {
+    return context.getResources().getIdentifier(
+            name,
+            "raw",
+            context.getPackageName()
+    );
+  }
+
+  /** @noinspection unused*/
   @DoNotStrip
-  public static byte[] fetchByteDataFromUrl(String url) {
-    OkHttpClient client = new OkHttpClient();
+  public static byte[] fetchByteDataFromUrl(String url) throws Exception {
+    Log.i(NAME, "Loading byte data from URL: " + url + "...");
 
-    Request request = new Request.Builder().url(url).build();
+    Uri uri = null;
+    Integer resourceId = null;
+    if (url.contains("://")) {
+      Log.i(NAME, "Parsing URL...");
+      uri = Uri.parse(url);
+      Log.i(NAME, "Parsed URL: " + uri.toString());
+    } else {
+      Log.i(NAME, "Parsing resourceId...");
+      resourceId = getResourceId(weakContext.get(), url);
+      Log.i(NAME, "Parsed resourceId: " + resourceId);
+    }
 
-    try (Response response = client.newCall(request).execute()) {
-      if (response.isSuccessful() && response.body() != null) {
-        return response.body().bytes();
-      } else {
-        throw new RuntimeException("Response was not successful!");
+    if (uri != null) {
+      // It's a URL/http resource
+      Request request = new Request.Builder().url(uri.toString()).build();
+      try (Response response = client.newCall(request).execute()) {
+        if (response.isSuccessful() && response.body() != null) {
+          return response.body().bytes();
+        } else {
+          throw new RuntimeException("Response was not successful!");
+        }
+      } catch (Exception ex) {
+        Log.e(NAME, "Failed to fetch URL " + url + "!", ex);
+        throw ex;
+      }
+    } else if (resourceId != null) {
+      // It's bundled into the Android resources/assets
+      Context context = weakContext.get();
+      if (context == null) {
+        throw new Exception("React Context has already been destroyed!");
+      }
+      try (InputStream stream = context.getResources().openRawResource(resourceId)) {
+        ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+        byte[] buffer = new byte[2048];
+        int length;
+        while ((length = stream.read(buffer)) != -1) {
+          byteStream.write(buffer, 0, length);
+        }
+        return byteStream.toByteArray();
       }
-    } catch (Exception e) {
-      Log.e(NAME, "Failed to fetch URL " + url + "!", e);
-      return null;
+    } else {
+      // It's a bird? it's a plane? not it's an error
+      throw new Exception("Input is neither a valid URL, nor a resourceId - " +
+              "cannot load TFLite model! (Input: " + url + ")");
     }
   }
 

example/android/app/src/main/AndroidManifest.xml

@@ -1,6 +1,7 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android">
 
     <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.CAMERA" />
 
     <application
       android:name=".MainApplication"

example/ios/Podfile.lock

@@ -890,7 +890,7 @@ PODS:
     - glog
     - RCT-Folly (= 2022.05.16.00)
     - React-Core
-  - react-native-worklets-core (0.2.4):
+  - react-native-worklets-core (0.3.0):
     - React
     - React-callinvoker
     - React-Core
@@ -1275,7 +1275,7 @@ SPEC CHECKSUMS:
   React-logger: 66b168e2b2bee57bd8ce9e69f739d805732a5570
   React-Mapbuffer: 9ee041e1d7be96da6d76a251f92e72b711c651d6
   react-native-fast-tflite: 369d7ded6d82a878fb481ab772c26804a2a797f0
-  react-native-worklets-core: b4094f51cb2bc649e297206425cb8956f4945e3e
+  react-native-worklets-core: a894d572639fcf37c6d284cc799882d25d00c93d
   React-nativeconfig: d753fbbc8cecc8ae413d615599ac378bbf6999bb
   React-NativeModulesApple: 964f4eeab1b4325e8b6a799cf4444c3fd4eb0a9c
   React-perflogger: 29efe63b7ef5fbaaa50ef6eaa92482f98a24b97e
@@ -1299,7 +1299,7 @@ SPEC CHECKSUMS:
   SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
   vision-camera-resize-plugin: 536345c29f42e04438d34d0cada7b4992a8fc104
   VisionCamera: edbcd00e27a438b2228f67823e2b8d15a189065f
-  Yoga: e64aa65de36c0832d04e8c7bd614396c77a80047
+  Yoga: 13c8ef87792450193e117976337b8527b49e8c03
 
 PODFILE CHECKSUM: 33c2f84b68c18fe6787cbe3e723675d15fcc7e66
 

example/metro.config.js

@@ -1,16 +1,19 @@
+const { getDefaultConfig, mergeConfig } = require('@react-native/metro-config');
 const path = require('path');
 const escape = require('escape-string-regexp');
 const exclusionList = require('metro-config/src/defaults/exclusionList');
 const pak = require('../package.json');
 
 const root = path.resolve(__dirname, '..');
+const modules = Object.keys({ ...pak.peerDependencies });
 
-const modules = Object.keys({
-  ...pak.peerDependencies,
-});
-
-module.exports = {
-  projectRoot: __dirname,
+/**
+ * Metro configuration
+ * https://facebook.github.io/metro/docs/configuration
+ *
+ * @type {import('metro-config').MetroConfig}
+ */
+const config = {
   watchFolders: [root],
 
   // We need to make sure that only one version is loaded for peerDependencies
@@ -39,3 +42,5 @@ module.exports = {
     }),
   },
 };
+
+module.exports = mergeConfig(getDefaultConfig(__dirname), config);

example/package.json

@@ -13,7 +13,7 @@
     "react": "18.2.0",
     "react-native": "0.73.2",
     "react-native-vision-camera": "^3.8.2",
-    "react-native-worklets-core": "^0.2.4",
+    "react-native-worklets-core": "^0.3.0",
     "vision-camera-resize-plugin": "^2.0.1"
   },
   "devDependencies": {

example/src/App.tsx

@@ -51,7 +51,7 @@ export default function App(): React.ReactNode {
 
       console.log(`Running inference on ${frame}`)
       const resized = resize(frame, {
-        size: {
+        scale: {
           width: 320,
           height: 320,
         },
@@ -79,6 +79,7 @@ export default function App(): React.ReactNode {
           style={StyleSheet.absoluteFill}
           isActive={true}
           frameProcessor={frameProcessor}
+          pixelFormat="yuv"
         />
       ) : (
         <Text>No Camera available.</Text>

example/yarn.lock

@@ -5779,10 +5779,10 @@ react-native-vision-camera@^3.8.2:
   resolved "https://registry.yarnpkg.com/react-native-vision-camera/-/react-native-vision-camera-3.8.2.tgz#f4f75f84c6a19e1c3474ddc0f7f785b5a526739b"
   integrity sha512-MY39l2e3hNRPUefn2JPShOFExcw0PblbAcUGvJrIfS9pMzdIyceo0umRAx8lOGXzDUAdb+xy/tFWb8zGxKimCQ==
 
-react-native-worklets-core@^0.2.4:
-  version "0.2.4"
-  resolved "https://registry.yarnpkg.com/react-native-worklets-core/-/react-native-worklets-core-0.2.4.tgz#a4a79c04f2c6769ff03e96b7529c8638c88fa560"
-  integrity sha512-NKqxLRDOYfO7RJRZHHDA/1yztdiSadSTTILgHw2VwSrcQcmnWAZGXWr9EYpzKblpbk/vXfJa6QpWIKHjNrbe4w==
+react-native-worklets-core@^0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/react-native-worklets-core/-/react-native-worklets-core-0.3.0.tgz#4d6c4c4999d3bf2f8c67fa46fdd70d8b5bc7ba0e"
+  integrity sha512-bOoo6xbl+f+mz0Q65bLkUP3FxgYt7f/Bzi0c5Yj8Ix4vQCKMtgHwafrqCNRhW32N7TpTv3+zOoyQMJwQVr/frQ==
   dependencies:
     string-hash-64 "^1.0.3"