feat: update to arcore 1.51.0, fix AR Markers Android

doranteseduardo · doranteseduardo · commit b02139ca95eb · 2025-11-26T15:50:34.000-06:00
diff --git a/android/libs/arcore/include/arcore_c_api.h b/android/libs/arcore/include/arcore_c_api.h
diff --git a/android/memoryleaktest/build.gradle b/android/memoryleaktest/build.gradle
@@ -64,7 +64,7 @@ dependencies {
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.6.1'
     implementation project(':virocore')
     implementation project(':libs:gvr')
-    implementation 'com.google.ar:core:1.45.0'
+    implementation 'com.google.ar:core:1.41.0'
     implementation "androidx.appcompat:appcompat:1.0.0"
     implementation "androidx.core:core-ktx:1.8.0"
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:2.0.20"
diff --git a/android/nativetest/build.gradle b/android/nativetest/build.gradle
@@ -65,7 +65,7 @@ android {
 
 dependencies {
     implementation fileTree(include: ['*.jar'], dir: 'libs')
-    implementation 'com.google.ar:core:1.45.0'
+    implementation 'com.google.ar:core:1.51.0'
     implementation project(':libs:gvr')
     implementation project(':virocore')
     implementation "androidx.appcompat:appcompat:1.0.0"
diff --git a/android/releasetest/build.gradle b/android/releasetest/build.gradle
@@ -83,7 +83,7 @@ dependencies {
     implementation fileTree(include: ['*.jar'], dir: 'libs')
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.6.1'
     implementation project(':virocore')
-    implementation 'com.google.ar:core:1.45.0'
+    implementation 'com.google.ar:core:1.51.0'
     implementation project(':libs:gvr')
     implementation "androidx.appcompat:appcompat:1.0.0"
     // testing dependencies
diff --git a/android/renderertest/build.gradle b/android/renderertest/build.gradle
@@ -66,7 +66,7 @@ android {
 
 dependencies {
     implementation fileTree(include: ['*.jar'], dir: 'libs')
-    implementation 'com.google.ar:core:1.45.0'
+    implementation 'com.google.ar:core:1.51.0'
     implementation project(':libs:gvr')
     implementation project(':virocore')
     implementation "androidx.appcompat:appcompat:1.0.0"
diff --git a/android/sharedCode/build.gradle b/android/sharedCode/build.gradle
@@ -46,7 +46,7 @@ dependencies {
     implementation 'androidx.media3:media3-exoplayer-hls:1.1.1'
     implementation 'androidx.media3:media3-exoplayer-smoothstreaming:1.1.1'
     implementation project(':libs:gvr')
-    implementation 'com.google.ar:core:1.45.0'
+    implementation 'com.google.ar:core:1.51.0'
     implementation 'com.google.protobuf.nano:protobuf-javanano:3.0.0-alpha-7'
     implementation "androidx.core:core-ktx:1.8.0"
     implementation "org.jetbrains.kotlin:kotlin-stdlib:2.0.0"
diff --git a/android/sharedCode/src/main/cpp/VROImageAndroid.cpp b/android/sharedCode/src/main/cpp/VROImageAndroid.cpp
@@ -96,15 +96,17 @@ unsigned char *VROImageAndroid::getData(size_t *length) {
  This function is used by VROARImageTargetAndroid (w/ ARCore). As such, we can
  make a few assumptions, that the data is from a Bitmap and is using the RGBA_8888
  format.
+
+ IMPORTANT: ARCore requires stride == width for augmented images. We always
+ return a tightly-packed grayscale buffer.
  */
 unsigned char *VROImageAndroid::getGrayscaleData(size_t *length, size_t *stride) {
-    *length = _dataLength / 4; // the length is 1/4th because RGBA -> Grayscale (1 byte)
+    *length = _width * _height; // tightly-packed grayscale (no padding)
+    *stride = _width; // ARCore requires stride == width
 
     if (_grayscaleData == nullptr) {
-        // we can make this assumption because VROPlatformConvertBitmap computes _dataLength = _stride * _height
+        // Calculate the RGBA stride from the original data
         int32_t rgbastride = _dataLength / _height;
-        *stride = rgbastride / 4;
-
         convertRgbaToGrayscale(rgbastride, &_grayscaleData);
     }
     return _grayscaleData;
@@ -113,17 +115,22 @@ unsigned char *VROImageAndroid::getGrayscaleData(size_t *length, size_t *stride)
 /*
  This function comes from the augmented_image_c example provided by ARCore (util.h's
  ConvertRgbaToGrayscale function).
+
+ IMPORTANT: ARCore requires stride == width for augmented images. We always
+ output a tightly-packed grayscale buffer (no row padding) regardless of the
+ input RGBA stride.
  */
 void VROImageAndroid::convertRgbaToGrayscale(int32_t stride, uint8_t **out_grayscale_buffer) {
-    int32_t grayscale_stride = stride / 4;  // Only support RGBA_8888 format
-    uint8_t *grayscale_buffer = new uint8_t[grayscale_stride * _height];
+    // Always use width as the output stride (no padding) - ARCore requires stride == width
+    uint8_t *grayscale_buffer = new uint8_t[_width * _height];
     for (int h = 0; h < _height; ++h) {
         for (int w = 0; w < _width; ++w) {
             const uint8_t *pixel = &_data[w * 4 + h * stride];
             uint8_t r = *pixel;
             uint8_t g = *(pixel + 1);
             uint8_t b = *(pixel + 2);
-            grayscale_buffer[w + h * grayscale_stride] =
+            // Write to tightly-packed buffer (stride == width)
+            grayscale_buffer[w + h * _width] =
                     static_cast<uint8_t>(0.213f * r + 0.715 * g + 0.072 * b);
         }
     }
diff --git a/android/sharedCode/src/main/cpp/arcore/ARCore_API.h b/android/sharedCode/src/main/cpp/arcore/ARCore_API.h
@@ -320,6 +320,7 @@ namespace arcore {
                                      FocusMode focusMode, DepthMode depthMode, SemanticMode semanticMode) = 0;
 
         virtual bool isDepthModeSupported(DepthMode depthMode) = 0;
+        virtual bool isSemanticModeSupported(SemanticMode semanticMode) = 0;
 
         virtual AugmentedImageDatabase *createAugmentedImageDatabase() = 0;
         virtual AugmentedImageDatabase *createAugmentedImageDatabase(uint8_t* raw_buffer, int64_t size) = 0;
diff --git a/android/sharedCode/src/main/cpp/arcore/VROARSessionARCore.cpp b/android/sharedCode/src/main/cpp/arcore/VROARSessionARCore.cpp
@@ -284,36 +284,37 @@ bool VROARSessionARCore::updateARCoreConfig() {
     }
   }
 
-  pinfo("Creating ARCore config with planeFindingMode=%d, depthMode=%d, semanticMode=%d",
-        (int)_planeFindingMode, (int)effectiveDepthMode, (int)_semanticMode); // DEBUG
+  // Check if semantic mode is supported on this device
+  arcore::SemanticMode effectiveSemanticMode = _semanticMode;
+  if (_semanticMode != arcore::SemanticMode::Disabled) {
+    if (!_session->isSemanticModeSupported(_semanticMode)) {
+      pwarn("⚠️ Requested semantic mode %d not supported on this device, falling back to DISABLED",
+            (int)_semanticMode);
+      effectiveSemanticMode = arcore::SemanticMode::Disabled;
+    }
+  }
+
   arcore::Config *config =
       _session->createConfig(_lightingMode, _planeFindingMode, _updateMode,
-                             _cloudAnchorMode, _focusMode, effectiveDepthMode, _semanticMode);
+                             _cloudAnchorMode, _focusMode, effectiveDepthMode, effectiveSemanticMode);
 
   if (getImageTrackingImpl() == VROImageTrackingImpl::ARCore &&
       _currentARCoreImageDatabase) {
     config->setAugmentedImageDatabase(_currentARCoreImageDatabase);
   }
 
-  pinfo("Applying config to ARCore session..."); // DEBUG
+  // ARCore requires the session to be paused before calling configure()
+  _session->pause();
+
   arcore::ConfigStatus status = _session->configure(config);
   delete (config);
 
   if (status == arcore::ConfigStatus::Success) {
-    pinfo("✅ Successfully configured AR session [lighting %d, planes %d, "
-          "update %d, focus %d, depth %d, semantic %d]",
-          _lightingMode, _planeFindingMode, _updateMode, _focusMode, effectiveDepthMode, _semanticMode);
     _session->resume();
     return true;
-  } else if (status == arcore::ConfigStatus::UnsupportedConfiguration) {
-    pwarn("❌ Failed to configure AR session: configuration not supported");
-    pwarn("   Your device may not support the requested features (vertical planes, depth, or semantic mode)!");
-    return false;
-  } else if (status == arcore::ConfigStatus::SessionNotPaused) {
-    pwarn("❌ Failed to change AR configuration: session must be paused");
-    return false;
   } else {
-    pwarn("❌ Unknown error updating AR configuration");
+    pwarn("Failed to configure AR session (status %d)", (int)status);
+    _session->resume();
     return false;
   }
 }
@@ -498,7 +499,7 @@ void VROARSessionARCore::addTargetToDatabase(
   rotateImageForOrientation(&grayscaleImage, &width, &height, &stride,
                             target->getOrientation());
 
-  database->addImageWithPhysicalSize(
+  arcore::AugmentedImageDatabaseStatus status = database->addImageWithPhysicalSize(
       targetAndroid->getId().c_str(), grayscaleImage, width, height,
       (int32_t)stride, target->getPhysicalWidth(), &outIndex);
 
@@ -918,7 +919,6 @@ void VROARSessionARCore::processUpdatedAnchors(VROARFrameARCore *frameAR) {
     arcore::TrackableList *imageList = _session->createTrackableList();
     frame->getUpdatedTrackables(imageList, arcore::TrackableType::Image);
     int imageSize = imageList->size();
-
     for (int i = 0; i < imageSize; i++) {
       arcore::Trackable *trackable = imageList->acquireItem(i);
       arcore::AugmentedImage *image = (arcore::AugmentedImage *)trackable;
@@ -968,8 +968,6 @@ void VROARSessionARCore::processUpdatedAnchors(VROARFrameARCore *frameAR) {
             target = std::dynamic_pointer_cast<VROARImageTargetAndroid>(
                 _imageTargets[j]);
             if (key == target->getId()) {
-              pinfo("Detected new anchor tied to image target [%s]",
-                    key.c_str());
               haveFoundTarget = true;
               // break out of the loop since we found a target id that matches
               // the key
diff --git a/android/sharedCode/src/main/java/com/viro/core/ARImageTarget.java b/android/sharedCode/src/main/java/com/viro/core/ARImageTarget.java
@@ -129,7 +129,27 @@ public long getNativeRef() {
      *                      <i>post</i>-orientation.
      */
     public ARImageTarget(Bitmap image, Orientation orientation, float physicalWidth) {
-        mId = String.valueOf(this.hashCode()); // the ID is simply the string of the hashCode().
+        this(image, orientation, physicalWidth, null);
+    }
+
+    /**
+     * Creates a ARImageTarget object to be given to {@link ViroViewARCore} to start looking
+     * for and tracking the specified image, with a custom ID.
+     *
+     * @param image         The {@link Bitmap} containing the image to track.
+     * @param orientation   The {@link Orientation} of the given Bitmap, which indicates where the
+     *                      top of the image is. For example, if the input image is an upside-down
+     *                      dollar bill, you would set the orientation to {@link Orientation#Down}
+     *                      to identify right-side-up dollar bills in the world. Set to {@link
+     *                      Orientation#Up} to indicate you want to identify the image as-is without
+     *                      any rotation.
+     * @param physicalWidth The real-world width of the image in meters. Note this is the width
+     *                      <i>post</i>-orientation.
+     * @param id            The custom ID for this target. If null, a hashCode-based ID will be used.
+     *                      This ID is used to match detected images with their corresponding targets.
+     */
+    public ARImageTarget(Bitmap image, Orientation orientation, float physicalWidth, String id) {
+        mId = (id != null) ? id : String.valueOf(this.hashCode());
         mOrientation = orientation;
         mPhysicalWidth = physicalWidth;
         mNativeRef = nativeCreateARImageTarget(image, mOrientation.getStringValue(), mPhysicalWidth, mId);
diff --git a/android/viroar/src/main/cpp/arcore/ARCore_Native.cpp b/android/viroar/src/main/cpp/arcore/ARCore_Native.cpp
@@ -926,6 +926,27 @@ namespace arcore {
         return isSupported != 0;
     }
 
+    bool SessionNative::isSemanticModeSupported(SemanticMode semanticMode) {
+        ArSemanticMode arSemanticMode;
+        switch (semanticMode) {
+            case SemanticMode::Disabled:
+                return true;  // Always supported
+            case SemanticMode::Enabled:
+                arSemanticMode = AR_SEMANTIC_MODE_ENABLED;
+                break;
+        }
+
+        int32_t isSupported = 0;
+        ArSession_isSemanticModeSupported(_session, arSemanticMode, &isSupported);
+
+        __android_log_print(ANDROID_LOG_INFO, "ViroARCore",
+                          "Semantic mode %d supported: %s",
+                          (int)semanticMode,
+                          isSupported ? "YES" : "NO");
+
+        return isSupported != 0;
+    }
+
     AugmentedImageDatabase *SessionNative::createAugmentedImageDatabase() {
         ArAugmentedImageDatabase *database;
         ArAugmentedImageDatabase_create(_session, &database);
diff --git a/android/viroar/src/main/cpp/arcore/ARCore_Native.h b/android/viroar/src/main/cpp/arcore/ARCore_Native.h
@@ -236,6 +236,7 @@ namespace arcore {
                                      UpdateMode updateMode, CloudAnchorMode cloudAnchorMode,
                                      FocusMode focusMode, DepthMode depthMode, SemanticMode semanticMode);
         virtual bool isDepthModeSupported(DepthMode depthMode);
+        virtual bool isSemanticModeSupported(SemanticMode semanticMode);
         virtual AugmentedImageDatabase *createAugmentedImageDatabase();
         virtual AugmentedImageDatabase *createAugmentedImageDatabase(uint8_t* raw_buffer, int64_t size);
         virtual Pose *createPose();
diff --git a/android/viroar/src/main/jniLibs/arm64-v8a/libarcore_sdk_c.so b/android/viroar/src/main/jniLibs/arm64-v8a/libarcore_sdk_c.so
diff --git a/android/viroar/src/main/jniLibs/armeabi-v7a/libarcore_sdk_c.so b/android/viroar/src/main/jniLibs/armeabi-v7a/libarcore_sdk_c.so
diff --git a/android/viroar/src/main/jniLibs/x86/libarcore_sdk_c.so b/android/viroar/src/main/jniLibs/x86/libarcore_sdk_c.so
diff --git a/android/viroar/src/main/jniLibs/x86_64/libarcore_sdk_c.so b/android/viroar/src/main/jniLibs/x86_64/libarcore_sdk_c.so
diff --git a/android/virocore/build.gradle b/android/virocore/build.gradle
@@ -94,7 +94,7 @@ dependencies {
     implementation 'androidx.media3:media3-exoplayer-hls:1.1.1'
     implementation 'androidx.media3:media3-exoplayer-smoothstreaming:1.1.1'
     api project(':libs:gvr')
-    implementation 'com.google.ar:core:1.45.0'
+    implementation 'com.google.ar:core:1.51.0'
     //implementation project(path: ':sharedCode')
     implementation 'com.google.protobuf.nano:protobuf-javanano:3.0.0-alpha-7'
 }
diff --git a/android/viroreact/build.gradle b/android/viroreact/build.gradle
@@ -99,7 +99,7 @@ dependencies {
     implementation 'androidx.media3:media3-exoplayer-hls:1.1.1'
     implementation 'androidx.media3:media3-exoplayer-smoothstreaming:1.1.1'
     implementation project(':libs:gvr')
-    implementation 'com.google.ar:core:1.45.0'
+    implementation 'com.google.ar:core:1.51.0'
     implementation 'com.google.protobuf.nano:protobuf-javanano:3.0.0-alpha-7'
 }
 
