feat: add comprehensive ebook library management system

This commit includes multiple related changes:

feat: add ebook library management with admin UI
- Add Libraries CRUD UI at /admin/libraries for managing library directories
- Support three scan modes: manual, auto_watch (filesystem monitoring), scheduled (cron)
- Include directory tree picker component for intuitive path selection
- Implement LibraryWatcher GenServer for automatic filesystem change detection
- Implement LibraryScheduler GenServer for cron-based scheduled scanning
- Add comprehensive test coverage (341 tests passing)

feat: display all books regardless of collection status
- Switch from INNER JOIN to LEFT JOIN on collection_items
- Make media types informational only, not governing visibility
- Add delete book button with proper confirmation

fix: resolve PDF processing hanging on large files
- Switch from in-memory pdf_info library to command-line pdfinfo
- Resolve 8+ hour hangs on 50MB+ PDF files
- Process large PDFs in sub-second timeframes

fix: handle semicolon-separated tags to prevent overflow
- Add normalize_tags/1 to split and process delimited tag strings
- Prevent varchar(255) overflow errors from long concatenated tags

fix: support namespaced XML elements in EPUB parsing
- Handle container.xml and OPF files with namespace prefixes
- Support various EPUB XML format variations

refactor: migrate EPUB parsing from erlang :zip to zstream
- Use zstream library for more flexible ZIP handling
- Add fallback to erlang :zip for unsupported ZIP formats

refactor: replace xmerl with Saxy for XML parsing
- Rewrite OpfParser to use Saxy instead of xmerl
- Improve parsing reliability and performance

chore: update dependencies and Docker configuration
- Add zstream, file_system, saxy dependencies
- Remove pdf_info dependency
- Add poppler-utils to Dockerfile for pdfinfo command

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: jfro

Dockerfile

@@ -85,7 +85,7 @@ LABEL org.opencontainers.image.licenses=MIT
 RUN echo "[$BUILD_TIME] [$GITREF] building on host that is $BUILDPLATFORM, for the target architecture $TARGETPLATFORM" > /build.log
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends libstdc++6 openssl libncurses5 locales ca-certificates \
+    && apt-get install -y --no-install-recommends libstdc++6 openssl libncurses5 locales ca-certificates poppler-utils \
     && rm -rf /var/lib/apt/lists/*
 
 # Set the locale

config/config.exs

@@ -118,16 +118,121 @@ config :fuzzy_catalog, Oban.Web.Resolver, pubsub: FuzzyCatalog.PubSub
 
 config :oban, :notifier, pubsub: FuzzyCatalog.PubSub
 
-# Ebook management configuration
+# =====================================================
+# Ebook Management Configuration
+# =====================================================
+#
+# This section configures the ebook library management system, including:
+# - File format support and limits
+# - Fuzzy matching for book linking
+# - Automatic filesystem watching (auto_watch mode)
+# - Scheduled scanning (scheduled mode)
+#
+# See lib/fuzzy_catalog/ebooks/library.ex for scan mode details
+
 config :fuzzy_catalog, :ebooks,
   # Supported file formats for ebook scanning
+  # Only files with these extensions will be processed during scans
   supported_formats: ["epub", "pdf"],
+
   # Maximum file size in bytes (100MB default)
+  # Files larger than this will be skipped during scanning
   max_file_size: 100 * 1024 * 1024,
+
   # Enable fuzzy matching for book linking by default
+  # When true, uses fuzzy string matching to link ebooks to existing books
   enable_fuzzy_matching: true,
+
   # Fuzzy matching threshold (0.0 - 1.0) - higher means stricter matching
-  fuzzy_threshold: 0.85
+  # 0.85 means 85% similarity required to consider a match
+  fuzzy_threshold: 0.85,
+
+  # FileSystem watcher configuration (for auto_watch mode)
+  # When enabled, monitors library directories for file changes
+  # Set to false in test environment to prevent interference
+  watcher_enabled: true,
+
+  # Debounce delay in milliseconds (wait for 5 seconds of no changes before scanning)
+  # This prevents rapid-fire scans during bulk file operations
+  # Libraries in auto_watch mode will wait this long after the last file change
+  watcher_debounce_ms: 5000,
+
+  # Library scheduler configuration (for scheduled mode)
+  # When enabled, processes libraries with cron-based schedules
+  # Set to false in test environment to prevent interference
+  scheduler_enabled: true
+
+# Note: Calibre library support is automatic - metadata.opf and cover.jpg
+# files are automatically detected and parsed when found alongside ebook files
+
+# =====================================================
+# Library Management Workflow
+# =====================================================
+#
+# ## Creating a Library
+#
+# 1. Navigate to /admin/libraries (admin users only)
+# 2. Click "New Library"
+# 3. Enter library name
+# 4. Use "Browse" button to select directory path via tree picker
+# 5. Choose scan mode:
+#    - Manual: Trigger scans via "Scan Now" button
+#    - Auto Watch: Automatic scans when files change (5s debounce)
+#    - Scheduled: Scans on cron schedule (e.g., "0 */6 * * *")
+# 6. If scheduled mode, enter cron expression
+# 7. Click "Create"
+#
+# ## Scan Modes Explained
+#
+# ### Manual Mode
+# - Scans only when you click "Scan Now" button
+# - Best for: One-time imports, testing, infrequent updates
+# - No background processes required
+#
+# ### Auto Watch Mode
+# - Monitors directory for file changes using FileSystem library
+# - Automatically triggers scan 5 seconds after last change
+# - Best for: Active libraries with frequent additions
+# - Requires: watcher_enabled: true (default)
+# - Background process: LibraryWatcher GenServer
+#
+# ### Scheduled Mode
+# - Runs scans on a cron schedule (e.g., every 6 hours)
+# - Cron format: minute hour day month weekday
+# - Examples:
+#   - "0 */6 * * *" = every 6 hours
+#   - "0 0 * * *" = daily at midnight
+#   - "0 9 * * 1" = Mondays at 9am
+# - Best for: Stable libraries with predictable update patterns
+# - Requires: scheduler_enabled: true (default)
+# - Background process: LibraryScheduler GenServer
+#
+# ## How Scanning Works
+#
+# 1. Library is marked as "scanning" to prevent concurrent scans
+# 2. ScanWorker job is enqueued in Oban
+# 3. ScanWorker recursively scans directory for .epub and .pdf files
+# 4. For each ebook file:
+#    - Extract metadata (title, author, ISBN, etc.)
+#    - Create or update Ebook record
+#    - Link to Library via library_id
+#    - If Calibre metadata.opf exists, parse it for rich metadata
+# 5. Library status updated to "idle" or "failed"
+# 6. last_scanned_at timestamp recorded
+#
+# ## Error Handling
+#
+# - If scan fails, library status set to "failed"
+# - Error message stored in last_scan_error field
+# - Can retry by clicking "Scan Now" or waiting for next scheduled scan
+# - Failed status does not prevent future scans
+#
+# ## Concurrent Scan Prevention
+#
+# - Only one scan can run per library at a time
+# - scanning_status field acts as a lock
+# - Manual, auto_watch, and scheduled modes all check this status
+# - "Scan Now" button disabled while scanning in progress
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.

config/test.exs

@@ -49,3 +49,8 @@ config :fuzzy_catalog, Oban,
   testing: :manual,
   queues: false,
   plugins: false
+
+# Disable FileSystem watcher and scheduler in tests
+config :fuzzy_catalog, :ebooks,
+  watcher_enabled: false,
+  scheduler_enabled: false

lib/fuzzy_catalog/application.ex

@@ -7,19 +7,40 @@ defmodule FuzzyCatalog.Application do
 
   @impl true
   def start(_type, _args) do
-    children = [
-      FuzzyCatalogWeb.Telemetry,
-      FuzzyCatalog.Repo,
-      {Oban, Application.fetch_env!(:fuzzy_catalog, Oban)},
-      {DNSCluster, query: Application.get_env(:fuzzy_catalog, :dns_cluster_query) || :ignore},
-      {Phoenix.PubSub, name: FuzzyCatalog.PubSub},
-      FuzzyCatalog.SyncStatusManager,
-      FuzzyCatalog.ProviderScheduler,
-      # Start a worker by calling: FuzzyCatalog.Worker.start_link(arg)
-      # {FuzzyCatalog.Worker, arg},
-      # Start to serve requests, typically the last entry
-      FuzzyCatalogWeb.Endpoint
-    ]
+    # Conditionally start LibraryWatcher based on config
+    library_watcher_child =
+      if watcher_enabled?() do
+        [FuzzyCatalog.Ebooks.LibraryWatcher]
+      else
+        []
+      end
+
+    # Conditionally start LibraryScheduler based on config
+    library_scheduler_child =
+      if scheduler_enabled?() do
+        [FuzzyCatalog.Ebooks.LibraryScheduler]
+      else
+        []
+      end
+
+    children =
+      [
+        FuzzyCatalogWeb.Telemetry,
+        FuzzyCatalog.Repo,
+        {Oban, Application.fetch_env!(:fuzzy_catalog, Oban)},
+        {DNSCluster, query: Application.get_env(:fuzzy_catalog, :dns_cluster_query) || :ignore},
+        {Phoenix.PubSub, name: FuzzyCatalog.PubSub},
+        FuzzyCatalog.SyncStatusManager,
+        FuzzyCatalog.ProviderScheduler
+      ] ++
+        library_watcher_child ++
+        library_scheduler_child ++
+        [
+          # Start a worker by calling: FuzzyCatalog.Worker.start_link(arg)
+          # {FuzzyCatalog.Worker, arg},
+          # Start to serve requests, typically the last entry
+          FuzzyCatalogWeb.Endpoint
+        ]
 
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options
@@ -34,4 +55,14 @@ defmodule FuzzyCatalog.Application do
     FuzzyCatalogWeb.Endpoint.config_change(changed, removed)
     :ok
   end
+
+  defp watcher_enabled? do
+    Application.get_env(:fuzzy_catalog, :ebooks, [])
+    |> Keyword.get(:watcher_enabled, true)
+  end
+
+  defp scheduler_enabled? do
+    Application.get_env(:fuzzy_catalog, :ebooks, [])
+    |> Keyword.get(:scheduler_enabled, true)
+  end
 end

lib/fuzzy_catalog/catalog/book.ex

@@ -34,6 +34,11 @@ defmodule FuzzyCatalog.Catalog.Book do
     field :original_title, :string
     field :language, :string
 
+    # Extended Metadata (from OPF files)
+    field :rating, :integer
+    field :tags, {:array, :string}
+    field :custom_metadata, :map
+
     has_many :collection_items, FuzzyCatalog.Collections.CollectionItem
 
     timestamps()
@@ -59,12 +64,16 @@ defmodule FuzzyCatalog.Catalog.Book do
       :series,
       :series_number,
       :original_title,
-      :language
+      :language,
+      :rating,
+      :tags,
+      :custom_metadata
     ])
     |> validate_required([:title, :author])
     |> validate_length(:title, min: 1, max: 255)
     |> validate_length(:author, min: 1, max: 255)
     |> validate_number(:pages, greater_than: 0)
+    |> validate_number(:rating, greater_than_or_equal_to: 0, less_than_or_equal_to: 10)
     |> validate_series_number()
     |> validate_publication_date()
   end

lib/fuzzy_catalog/collections.ex

@@ -337,12 +337,20 @@ defmodule FuzzyCatalog.Collections do
     # Remove search from Flop params to avoid validation issues
     flop_params = remove_search_param(params)
 
+    # Show all books regardless of collection items or ebook files
+    # Media types are just informational and can be used for filtering
     base_query =
       from b in Book,
-        join: ci in CollectionItem,
+        left_join: ci in CollectionItem,
         on: ci.book_id == b.id,
         group_by: [b.id],
-        select: {b, fragment("array_agg(? ORDER BY ?)::text[]", ci.media_type, ci.media_type)}
+        select:
+          {b,
+           fragment(
+             "array_remove(array_agg(DISTINCT ? ORDER BY ?), NULL)::text[]",
+             ci.media_type,
+             ci.media_type
+           )}
 
     # Apply search filter if present
     query_with_search =

lib/fuzzy_catalog/ebooks.ex

@@ -52,6 +52,24 @@ defmodule FuzzyCatalog.Ebooks do
     |> Repo.one()
   end
 
+  @doc """
+  Lists all ebooks associated with a book.
+
+  ## Examples
+
+      iex> list_ebooks_for_book(book)
+      [%Ebook{}, ...]
+
+      iex> list_ebooks_for_book(%Book{id: 123})
+      []
+  """
+  def list_ebooks_for_book(%{id: book_id}) do
+    Ebook
+    |> where([e], e.book_id == ^book_id)
+    |> order_by([e], asc: e.file_path)
+    |> Repo.all()
+  end
+
   @doc """
   Triggers a directory scan for ebook files.
 

lib/fuzzy_catalog/ebooks/ebook.ex

@@ -32,6 +32,9 @@ defmodule FuzzyCatalog.Ebooks.Ebook do
     # Optional relationship to Book
     belongs_to :book, FuzzyCatalog.Catalog.Book
 
+    # Optional relationship to Library
+    belongs_to :library, FuzzyCatalog.Ebooks.Library
+
     timestamps()
   end
 
@@ -54,13 +57,15 @@ defmodule FuzzyCatalog.Ebooks.Ebook do
       :processing_error,
       :last_processed_at,
       :cover_thumbnail_key,
-      :book_id
+      :book_id,
+      :library_id
     ])
     |> validate_required([:file_path, :file_format])
     |> validate_inclusion(:file_format, @file_formats)
     |> validate_inclusion(:processing_status, @processing_statuses)
     |> validate_number(:file_size, greater_than: 0)
     |> foreign_key_constraint(:book_id)
+    |> foreign_key_constraint(:library_id)
     |> unique_constraint(:file_path)
   end