How to use the OCR ?

[teto]

I feel bad for asking silly questions in the same vein as #301 but here I go.

So I think I've built with OCR successfully and I opened: https://www.youtube.com/watch?v=somEzKcCDVE which contains hardcoded subtitles at the top.

I dont know how to leverage the OCR though:

• how can one enable it ? how does it work ? should I select an area ? can it do live transcription or is it preprocessing ?
• is it reliable in your experience ? is that an experimental feature

NB: I've used manga-ocr with success in the past

[ripose-jp]

If you build with OCR support, you should see in Settings → Options → OCR the option Enable OCR. If that is checked, OCR is enabled. The first time it's enabled, you will need to wait a bit to download the model, but once that's done it should just work. You can click the Eye icon on the bottom right of the OSC to go into OCR mode which will allow you to select an area to scan. You can also use Ctrl+D to enter OCR mode. Text will be searched in the search box on the side.

I wouldn't say it's experimental, but it's not something I recommend most people use because linking C++ to Python is pretty sketchy in terms of reliability. It also uses a lot of RAM. I hardly ever use it, but sometimes I pair it with mpv-manga-reader and just OCR text, though I'm sure better options exist.

[teto]

arf I had commented out the -DOCR_SUPPORT=true ...

How can I tell memento where to find libmocr-populate ? I have it built with nix:

nix/store/sz0g6wv3g2ny4cc2a44v29i5jx12jcxx-libmocr-unstable-2023-11-15
├── include
│   ├── mocr++.h
│   └── mocr.h
└── lib
    ├── libmocr++.so
    └── libmocr.so

memento wants to build libmocr but it's not possible to fetch arbitrary code in a nix sandbox. Is there a cmake flag to tell memento where to find it ?

       > -- Found Threads: TRUE
       > -- Performing Test HAVE_STDATOMIC
       > -- Performing Test HAVE_STDATOMIC - Success
       > -- Found WrapAtomic: TRUE
       > -- Found OpenGL: /nix/store/lgrb82lslvxgcv01dvxxqx6cjc3l3qdy-libglvnd-1.7.0/lib/libOpenGL.so
       > -- Found WrapOpenGL: TRUE
       > -- Found WrapVulkanHeaders: /nix/store/rihwzzvwj3p26wvfrjnw1f4nb4h3m001-vulkan-headers-1.4.328.0/include
       > CMake Error at /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/ExternalProject/shared_internal_commands.cmake:928 (message):
       >   error: could not find git for clone of libmocr-populate
       > Call Stack (most recent call first):
       >   /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/ExternalProject.cmake:3080 (_ep_add_download_command)
       >   CMakeLists.txt:29 (ExternalProject_Add)
       >
       >
       > -- Configuring incomplete, errors occurred!
       > 
       > CMake Error at /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/FetchContent.cmake:1906 (message):
       >   CMake step for libmocr failed: 1
       > Call Stack (most recent call first):
       >   /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/FetchContent.cmake:1609 (__FetchContent_populateSubbuild)
       >   /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/FetchContent.cmake:2145:EVAL:2 (__FetchContent_doPopulation)
       >   /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/FetchContent.cmake:2145 (cmake_language)
       >   /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/FetchContent.cmake:2384 (__FetchContent_Populate)
       >   extern/CMakeLists.txt:7 (FetchContent_MakeAvailable)
       >
       > 
       > -- Configuring incomplete, errors occurred!
       

[ripose-jp]

For what you want to do, you'll need to modify the CMake to be something like this.

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 790e371..fc82d08 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -139,6 +139,9 @@ if(MECAB_SUPPORT)
 	find_package(MeCab REQUIRED)
 endif()
 find_package(mpv REQUIRED)
+if(OCR_SUPPORT)
+	find_package(mocr REQUIRED)
+endif()
 find_package(SQLite3 REQUIRED)
 if(UNIX AND NOT APPLE) # Linux and Unix
 	find_package(
diff --git a/cmake/Findmocr.cmake b/cmake/Findmocr.cmake
new file mode 100644
index 0000000..04a6800
--- /dev/null
+++ b/cmake/Findmocr.cmake
@@ -0,0 +1,40 @@
+include(FindPackageHandleStandardArgs)
+
+find_library(mocr_LIBRARY NAMES mocr)
+find_path(mocr_INCLUDE_DIR NAMES mocr.h)
+
+find_library(mocrxx_LIBRARY NAMES mocr++)
+find_path(mocrxx_INCLUDE_DIR NAMES mocr++.h)
+
+find_package_handle_standard_args(
+    mocr
+    REQUIRED_VARS
+        mocr_LIBRARY mocr_INCLUDE_DIR
+        mocrxx_LIBRARY mocrxx_INCLUDE_DIR
+)
+
+if(mocr_FOUND)
+    mark_as_advanced(mocr_LIBRARY)
+    mark_as_advanced(mocr_INCLUDE_DIR)
+    mark_as_advanced(mocrxx_LIBRARY)
+    mark_as_advanced(mocrxx_INCLUDE_DIR)
+endif()
+
+if(mocr_FOUND AND NOT TARGET mocr::mocr)
+    add_library(mocr::mocr UNKNOWN IMPORTED)
+    set_target_properties(
+        mocr::mocr PROPERTIES
+        IMPORTED_LOCATION "${mocr_LIBRARY}"
+        INTERFACE_INCLUDE_DIRECTORIES "${mocr_INCLUDE_DIR}"
+    )
+endif()
+
+if(mocr_FOUND AND NOT TARGET mocr::mocrxx)
+    add_library(mocr::mocrxx UNKNOWN IMPORTED)
+    set_target_properties(
+        mocr::mocrxx PROPERTIES
+        IMPORTED_LOCATION "${mocrxx_LIBRARY}"
+        INTERFACE_INCLUDE_DIRECTORIES "${mocrxx_INCLUDE_DIR}"
+    )
+    target_link_libraries(mocr::mocrxx INTERFACE mocr::mocr)
+endif()
diff --git a/extern/CMakeLists.txt b/extern/CMakeLists.txt
index 7e2db26..1a8391a 100644
--- a/extern/CMakeLists.txt
+++ b/extern/CMakeLists.txt
@@ -19,13 +19,3 @@ FetchContent_Declare(
     PATCH_COMMAND ${MEMENTO_QCORO_PATCH_COMMAND}
 )
 FetchContent_MakeAvailable(qcoro)
-
-if(OCR_SUPPORT)
-    set(BUILD_TESTING OFF CACHE BOOL "" FORCE)
-    FetchContent_Declare(
-        libmocr
-        GIT_REPOSITORY https://github.com/ripose-jp/libmocr.git
-        GIT_TAG 24ec081102208d0ff6954c3003df2fb691713bd5
-    )
-    FetchContent_MakeAvailable(libmocr)
-endif()
diff --git a/src/ocr/CMakeLists.txt b/src/ocr/CMakeLists.txt
index f942be1..3ffaf50 100644
--- a/src/ocr/CMakeLists.txt
+++ b/src/ocr/CMakeLists.txt
@@ -9,7 +9,7 @@ if(OCR_SUPPORT)
     target_include_directories(ocrmodel PRIVATE ${MEMENTO_INCLUDE_DIRS})
     target_link_libraries(
         ocrmodel
-        PRIVATE mocr++
+        PRIVATE mocr::mocrxx
         PRIVATE Qt6::Concurrent
         PRIVATE Qt6::Gui
         PUBLIC Qt6::Core

[teto]

sry I am dumb I had not git added the cmake/Findmocr.cmake (I mostly use jujutsu these days which autoadds files).
Before I realized that, I wrote this patch ripose-jp/libmocr#1 .

Now that cmake finds libmocr, it fails to find qcoro-populate:

       > -- Found JsonC: /nix/store/6wz56cnrij20lz1igjrf7kgadp398cqn-json-c-0.18/lib/libjson-c.so
       > -- Found libzip: /nix/store/w4x03r3rjb8sh7n5cq9f0jjch8lw7wd1-libzip-1.11.4/lib/libzip.so
       > -- Found mpv: /nix/store/2952ycl8j3q7lf3kgi4rdgg5cxfczjhc-mpv-with-scripts-0.40.0/lib/libmpv.so
       > -- Found mocr: /nix/store/kwj1nqaryyaghjrh8hifmgv64zqykv4z-libmocr-unstable-2023-11-15/lib/libmocr.so
       > -- Found SQLite3: /nix/store/90fmssznwpgs6a0wv6il4w2j5glpn1dw-sqlite-3.50.4/lib/libsqlite3.so (found version "3.50.4")
       > -- Performing Test CMAKE_HAVE_LIBC_PTHREAD
       > -- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Success
       > -- Found Threads: TRUE
       > -- Performing Test HAVE_STDATOMIC
       > -- Performing Test HAVE_STDATOMIC - Success
       > -- Found WrapAtomic: TRUE
       > -- Found OpenGL: /nix/store/lgrb82lslvxgcv01dvxxqx6cjc3l3qdy-libglvnd-1.7.0/lib/libOpenGL.so
       > -- Found WrapOpenGL: TRUE
       > -- Found WrapVulkanHeaders: /nix/store/rihwzzvwj3p26wvfrjnw1f4nb4h3m001-vulkan-headers-1.4.328.0/include
       > CMake Error at /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/ExternalProject/shared_internal_commands.cmake:928 (message):
       >   error: could not find git for clone of qcoro-populate
       > Call Stack (most recent call first):
       >   /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/ExternalProject.cmake:3080 (_ep_add_download_command)
       >   CMakeLists.txt:29 (ExternalProject_Add)
       >
       >
       > -- Configuring incomplete, errors occurred!
       > 
       > CMake Error at /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/FetchContent.cmake:1906 (message):
       >   CMake step for qcoro failed: 1
       > Call Stack (most recent call first):
       >   /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/FetchContent.cmake:1609 (__FetchContent_populateSubbuild)
       >   /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/FetchContent.cmake:2145:EVAL:2 (__FetchContent_doPopulation)
       >   /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/FetchContent.cmake:2145 (cmake_language)
       >   /nix/store/44rvi6xdaaj9ii92n1crrdvj73nw9pbg-cmake-4.1.2/share/cmake-4.1/Modules/FetchContent.cmake:2384 (__FetchContent_Populate)
       >   extern/CMakeLists.txt:21 (FetchContent_MakeAvailable)
       >
       > 

from extern/CMakeLists.txt .

FetchContent_Declare(
    qcoro
    GIT_REPOSITORY https://github.com/qcoro/qcoro.git
    GIT_TAG ef1231ae490465b4f592ca326eee6d56a3d802ae # v0.12.0
    PATCH_COMMAND ${MEMENTO_QCORO_PATCH_COMMAND}
)

Similarly it would be nice if I could use the package from nixpkgs instead, aka if memento could look for a prebuilt package.
Alternatively nix could clone the exact same revision and put it where FetchContent_Declare accept it ? the important thing is not accessing the network (for reproducibility reason) as nix forbids it in build mode. Nix needs to be aware and download all inputs in advance.

For information, the content of a qcoro install, It doesn't seem to have pkg-config support.

tree $(nix-build -A qt6Packages.qcoro)
/nix/store/p1fj2iybm3k0apj4ksgh4g0ziglx5yav-qcoro-0.12.0
└── lib
    ├── libQCoro6Core.a
    ├── libQCoro6DBus.a
    ├── libQCoro6Network.a
    ├── libQCoro6Qml.a
    ├── libQCoro6Quick.a
    └── libQCoro6WebSockets.a

[ripose-jp]

I added SYSTEM_QCORO and SYSTEM_MOCR options on master. That will disable FetchContent for those libraries.

[teto]

thanks for doing that (I like your commit message singling out my use case xD I think debian and other distribs have similar concerns so these are improvements not just for one distrib) .

I've "wrapped" the memento executable such that it always finds its dependencies (it's a common pattern in the nix world).

  preFixup = let
    pyEnv = python3.withPackages(p: [ p.manga-ocr p.jaconv ]);
  in ''
    wrapProgram "$out/bin/memento" \
      --prefix PATH : "${yt-dlp}/bin" \
      --suffix PYTHONPATH : "${pyEnv}/${pyEnv.sitePackages}"
  '';

On my first attempt, I used a wrong PYTHONPATH and was met with OCR model is invalid. in the terminal.
The message could be made more actionable. When I look at the options, I see that my model is "kha-white/manga-ocr-base"
OCR model "kha-white/manga-ocr-base" is invalid. "manga-ocr" can't be imported .
I had to add jaconv as well as a dependency.
I've done a quick test right now and the OCR worked instantaneously even though I have an old GPU.
Thanks for all the help. Would be cool to tag a release so I can bump the nix package with ocr available but optional.

[ripose-jp]

I just bumped the version. I was just waiting for mpv to update since that's generally a good to time to make a new Memento release.

[teto]

thanks ! the new release should reach nixpkgs unstable in a few days NixOS/nixpkgs#476399