feat: sqlite video player (#19792)

* feat: video player

* use remote asset id in local query

* fix: error from pre-caching beyond total assets

* fix: flipped local videos

* incorrect aspect ratio on iOS

* ignore other storage id during equals check

---------

Co-authored-by: shenlong-tanwen <[email protected]>

Author: shenlong-tanwen

mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt

@@ -87,7 +87,8 @@ data class PlatformAsset (
   val updatedAt: Long? = null,
   val width: Long? = null,
   val height: Long? = null,
-  val durationInSeconds: Long
+  val durationInSeconds: Long,
+  val orientation: Long
 )
  {
   companion object {
@@ -100,7 +101,8 @@ data class PlatformAsset (
       val width = pigeonVar_list[5] as Long?
       val height = pigeonVar_list[6] as Long?
       val durationInSeconds = pigeonVar_list[7] as Long
-      return PlatformAsset(id, name, type, createdAt, updatedAt, width, height, durationInSeconds)
+      val orientation = pigeonVar_list[8] as Long
+      return PlatformAsset(id, name, type, createdAt, updatedAt, width, height, durationInSeconds, orientation)
     }
   }
   fun toList(): List<Any?> {
@@ -113,6 +115,7 @@ data class PlatformAsset (
       width,
       height,
       durationInSeconds,
+      orientation,
     )
   }
   override fun equals(other: Any?): Boolean {

mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt

@@ -40,7 +40,8 @@ open class NativeSyncApiImplBase(context: Context) {
       MediaStore.MediaColumns.BUCKET_ID,
       MediaStore.MediaColumns.WIDTH,
       MediaStore.MediaColumns.HEIGHT,
-      MediaStore.MediaColumns.DURATION
+      MediaStore.MediaColumns.DURATION,
+      MediaStore.MediaColumns.ORIENTATION,
     )
 
     const val HASH_BUFFER_SIZE = 2 * 1024 * 1024
@@ -74,6 +75,8 @@ open class NativeSyncApiImplBase(context: Context) {
         val widthColumn = c.getColumnIndexOrThrow(MediaStore.MediaColumns.WIDTH)
         val heightColumn = c.getColumnIndexOrThrow(MediaStore.MediaColumns.HEIGHT)
         val durationColumn = c.getColumnIndexOrThrow(MediaStore.MediaColumns.DURATION)
+        val orientationColumn =
+          c.getColumnIndexOrThrow(MediaStore.MediaColumns.ORIENTATION)
 
         while (c.moveToNext()) {
           val id = c.getLong(idColumn).toString()
@@ -101,6 +104,7 @@ open class NativeSyncApiImplBase(context: Context) {
           val duration = if (mediaType == MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE) 0
           else c.getLong(durationColumn) / 1000
           val bucketId = c.getString(bucketIdColumn)
+          val orientation = c.getInt(orientationColumn)
 
           val asset = PlatformAsset(
             id,
@@ -110,7 +114,8 @@ open class NativeSyncApiImplBase(context: Context) {
             modifiedAt,
             width,
             height,
-            duration
+            duration,
+            orientation.toLong(),
           )
           yield(AssetResult.ValidAsset(asset, bucketId))
         }

mobile/drift_schemas/main/drift_schema_v1.json

@@ -138,6 +138,7 @@ struct PlatformAsset: Hashable {
   var width: Int64? = nil
   var height: Int64? = nil
   var durationInSeconds: Int64
+  var orientation: Int64
 
 
   // swift-format-ignore: AlwaysUseLowerCamelCase
@@ -150,6 +151,7 @@ struct PlatformAsset: Hashable {
     let width: Int64? = nilOrValue(pigeonVar_list[5])
     let height: Int64? = nilOrValue(pigeonVar_list[6])
     let durationInSeconds = pigeonVar_list[7] as! Int64
+    let orientation = pigeonVar_list[8] as! Int64
 
     return PlatformAsset(
       id: id,
@@ -159,7 +161,8 @@ struct PlatformAsset: Hashable {
       updatedAt: updatedAt,
       width: width,
       height: height,
-      durationInSeconds: durationInSeconds
+      durationInSeconds: durationInSeconds,
+      orientation: orientation
     )
   }
   func toList() -> [Any?] {
@@ -172,6 +175,7 @@ struct PlatformAsset: Hashable {
       width,
       height,
       durationInSeconds,
+      orientation,
     ]
   }
   static func == (lhs: PlatformAsset, rhs: PlatformAsset) -> Bool {

mobile/ios/Runner/Sync/Messages.g.swift

@@ -27,7 +27,8 @@ extension PHAsset {
       updatedAt: modificationDate.map { Int64($0.timeIntervalSince1970) },
       width: Int64(pixelWidth),
       height: Int64(pixelHeight),
-      durationInSeconds: Int64(duration)
+      durationInSeconds: Int64(duration),
+      orientation: 0
     )
   }
 }
@@ -169,7 +170,8 @@ class NativeSyncApiImpl: NativeSyncApi {
             id: asset.localIdentifier,
             name: "",
             type: 0,
-            durationInSeconds: 0
+            durationInSeconds: 0,
+            orientation: 0
           )
           if (updatedAssets.contains(AssetWrapper(with: predicate))) {
             continue

mobile/ios/Runner/Sync/MessagesImpl.swift

@@ -25,6 +25,7 @@ sealed class BaseAsset {
   final int? height;
   final int? durationInSeconds;
   final bool isFavorite;
+  final String? livePhotoVideoId;
 
   const BaseAsset({
     required this.name,
@@ -36,18 +37,12 @@ sealed class BaseAsset {
     this.height,
     this.durationInSeconds,
     this.isFavorite = false,
+    this.livePhotoVideoId,
   });
 
   bool get isImage => type == AssetType.image;
   bool get isVideo => type == AssetType.video;
 
-  double? get aspectRatio {
-    if (width != null && height != null && height! > 0) {
-      return width! / height!;
-    }
-    return null;
-  }
-
   bool get hasRemote =>
       storage == AssetState.remote || storage == AssetState.merged;
   bool get hasLocal =>

mobile/lib/domain/models/asset/base_asset.model.dart

@@ -3,6 +3,7 @@ part of 'base_asset.model.dart';
 class LocalAsset extends BaseAsset {
   final String id;
   final String? remoteId;
+  final int orientation;
 
   const LocalAsset({
     required this.id,
@@ -16,6 +17,8 @@ class LocalAsset extends BaseAsset {
     super.height,
     super.durationInSeconds,
     super.isFavorite = false,
+    super.livePhotoVideoId,
+    this.orientation = 0,
   });
 
   @override
@@ -38,18 +41,20 @@ class LocalAsset extends BaseAsset {
    durationInSeconds: ${durationInSeconds ?? "<NA>"},
    remoteId: ${remoteId ?? "<NA>"}
    isFavorite: $isFavorite,
+  orientation: $orientation,
  }''';
   }
 
   @override
   bool operator ==(Object other) {
     if (other is! LocalAsset) return false;
     if (identical(this, other)) return true;
-    return super == other && id == other.id && remoteId == other.remoteId;
+    return super == other && id == other.id && orientation == other.orientation;
   }
 
   @override
-  int get hashCode => super.hashCode ^ id.hashCode ^ remoteId.hashCode;
+  int get hashCode =>
+      super.hashCode ^ id.hashCode ^ remoteId.hashCode ^ orientation.hashCode;
 
   LocalAsset copyWith({
     String? id,
@@ -63,6 +68,7 @@ class LocalAsset extends BaseAsset {
     int? height,
     int? durationInSeconds,
     bool? isFavorite,
+    int? orientation,
   }) {
     return LocalAsset(
       id: id ?? this.id,
@@ -76,6 +82,7 @@ class LocalAsset extends BaseAsset {
       height: height ?? this.height,
       durationInSeconds: durationInSeconds ?? this.durationInSeconds,
       isFavorite: isFavorite ?? this.isFavorite,
+      orientation: orientation ?? this.orientation,
     );
   }
 }

mobile/lib/domain/models/asset/local_asset.model.dart

@@ -30,6 +30,7 @@ class RemoteAsset extends BaseAsset {
     super.isFavorite = false,
     this.thumbHash,
     this.visibility = AssetVisibility.timeline,
+    super.livePhotoVideoId,
   });
 
   @override
@@ -65,7 +66,6 @@ class RemoteAsset extends BaseAsset {
     return super == other &&
         id == other.id &&
         ownerId == other.ownerId &&
-        localId == other.localId &&
         thumbHash == other.thumbHash &&
         visibility == other.visibility;
   }

mobile/lib/domain/models/asset/remote_asset.model.dart

@@ -5,6 +5,7 @@ enum Setting<T> {
   groupAssetsBy<int>(StoreKey.groupAssetsBy, 0),
   showStorageIndicator<bool>(StoreKey.storageIndicator, true),
   loadOriginal<bool>(StoreKey.loadOriginal, false),
+  loadOriginalVideo<bool>(StoreKey.loadOriginalVideo, false),
   preferRemoteImage<bool>(StoreKey.preferRemoteImage, false),
   advancedTroubleshooting<bool>(StoreKey.advancedTroubleshooting, false),
   ;

mobile/lib/domain/models/setting.model.dart

@@ -2,16 +2,20 @@ import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
 import 'package:immich_mobile/domain/models/exif.model.dart';
 import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
 import 'package:immich_mobile/infrastructure/repositories/remote_asset.repository.dart';
+import 'package:immich_mobile/infrastructure/utils/exif.converter.dart';
+import 'package:platform/platform.dart';
 
 class AssetService {
   final RemoteAssetRepository _remoteAssetRepository;
   final DriftLocalAssetRepository _localAssetRepository;
+  final Platform _platform;
 
   const AssetService({
     required RemoteAssetRepository remoteAssetRepository,
     required DriftLocalAssetRepository localAssetRepository,
   })  : _remoteAssetRepository = remoteAssetRepository,
-        _localAssetRepository = localAssetRepository;
+        _localAssetRepository = localAssetRepository,
+        _platform = const LocalPlatform();
 
   Stream<BaseAsset?> watchAsset(BaseAsset asset) {
     final id = asset is LocalAsset ? asset.id : (asset as RemoteAsset).id;
@@ -21,10 +25,40 @@ class AssetService {
   }
 
   Future<ExifInfo?> getExif(BaseAsset asset) async {
-    if (asset is LocalAsset || asset is! RemoteAsset) {
+    if (!asset.hasRemote) {
       return null;
     }
 
-    return _remoteAssetRepository.getExif(asset.id);
+    final id =
+        asset is LocalAsset ? asset.remoteId! : (asset as RemoteAsset).id;
+    return _remoteAssetRepository.getExif(id);
+  }
+
+  Future<double> getAspectRatio(BaseAsset asset) async {
+    bool isFlipped;
+    double? width;
+    double? height;
+
+    if (asset.hasRemote) {
+      final exif = await getExif(asset);
+      isFlipped = ExifDtoConverter.isOrientationFlipped(exif?.orientation);
+      width = exif?.width ?? asset.width?.toDouble();
+      height = exif?.height ?? asset.height?.toDouble();
+    } else if (asset is LocalAsset) {
+      isFlipped = _platform.isAndroid &&
+          (asset.orientation == 90 || asset.orientation == 270);
+      width = asset.width?.toDouble();
+      height = asset.height?.toDouble();
+    } else {
+      isFlipped = false;
+    }
+
+    final orientedWidth = isFlipped ? height : width;
+    final orientedHeight = isFlipped ? width : height;
+    if (orientedWidth != null && orientedHeight != null && orientedHeight > 0) {
+      return orientedWidth / orientedHeight;
+    }
+
+    return 1.0;
   }
 }