Excel resource manager

Author: Quafadas

.github/copilot-instructions.md

@@ -5,7 +5,9 @@ ALWAYS reference these instructions first and fallback to search or bash command
 Scautable is a Scala 3 project using the mill build tool. It is a lightweight dataframe library with the twist that it expects the "dataframe" in question to have its structure identified by the compiler, at compile time. A dataframe here is modelled as an `Iterator[NamedTuple[K, V]]` where `K` is a compile time constant tuple of strings representing column names.
 
 ## Working Effectively
-- **NEVER CANCEL BUILDS**: Mill cold compilation takes 2 minutes or so. Tests take 1-3 minutes from cold. Set timeout to 2+ minutes.
+  On windows, use `mill scautable.jvm.test`, on MacOS/Linux use `./mill scautable.jvm.test` to run commands. Note the absence of `./` on windows.
+
+- **BUILDS**: Mill cold compilation takes 2 minutes or so. Tests take 1-3 minutes from cold. Set timeout to 2+ minutes.
 - Compile all modules:
   - `./mill __.compile` -- Compiles all modules (JVM, JS, tests). Takes 3-5 minutes cold, fast cached. NEVER CANCEL.
 - Compile specific platforms:

scautable/src-jvm/Excel.scala

@@ -44,6 +44,23 @@ object Excel:
 
   transparent inline def resource[K](filePath: String, sheetName: String) =
     ${ readExcelResource('filePath, 'sheetName, '{""}, '{TypeInferrer.FromAllRows}) }
-  
+
+  /** Cleanup cached workbook resources for a specific file.
+    * 
+    * Call this method when you know that no more operations will be performed
+    * on a specific Excel file to free up memory immediately.
+    * 
+    * @param filePath Path to the Excel file to cleanup
+    */
+  // def cleanup(filePath: String): Unit =
+    // ExcelResourceManager.cleanup(filePath)
+
+  /** Cleanup all cached Excel workbook resources.
+    * 
+    * This method is useful for application shutdown or when you want to
+    * free up all Excel-related memory immediately.
+    */
+  // def cleanupAll(): Unit =
+    // ExcelResourceManager.cleanupAll()
 
 end Excel

scautable/src-jvm/ExcelIterator.scala

@@ -1,11 +1,11 @@
 package io.github.quafadas.scautable
 
-import java.io.File
 import scala.NamedTuple.*
 import scala.collection.JavaConverters.*
-import org.apache.poi.ss.usermodel.{Row, WorkbookFactory}
+import org.apache.poi.ss.usermodel.Row
 import org.apache.poi.ss.util.CellRangeAddress
 import io.github.quafadas.scautable.BadTableException
+import io.github.quafadas.scautable.ExcelWorkbookCache
 
 /** Iterator for reading Excel files with compile-time type safety
   *
@@ -50,7 +50,9 @@ class ExcelIterator[K <: Tuple, V <: Tuple](filePath: String, sheetName: String,
 
   // Lazy-initialized sheet iterator to avoid opening file until needed
   private lazy val sheetIterator =
-    val workbook = WorkbookFactory.create(new File(filePath))
+    val workbook = ExcelWorkbookCache.getOrCreate(filePath).getOrElse(
+      throw new BadTableException(s"Failed to open Excel file: $filePath")
+    )
     val sheet = workbook.getSheet(sheetName)
     // Create an iterator that gives us rows by index for the specified range
     colRange match
@@ -88,16 +90,15 @@ class ExcelIterator[K <: Tuple, V <: Tuple](filePath: String, sheetName: String,
     */
   private inline def extractHeadersFromRange(range: String): List[String] =
     val (firstRow, _, firstCol, lastCol) = parseRange(range)
-    val workbook = WorkbookFactory.create(new File(filePath))
-    try 
-      val sheet = workbook.getSheet(sheetName)
-      val headerRow = sheet.getRow(firstRow)
-      val cells =
-        for (i <- firstCol.to(lastCol))
-          yield headerRow.getCell(i, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).toString
-      cells.toList
-    finally
-      workbook.close()
+    val workbook = ExcelWorkbookCache.getOrCreate(filePath).getOrElse(
+      throw new BadTableException(s"Failed to open Excel file: $filePath")
+    )
+    val sheet = workbook.getSheet(sheetName)
+    val headerRow = sheet.getRow(firstRow)
+    val cells =
+      for (i <- firstCol.to(lastCol))
+        yield headerRow.getCell(i, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).toString
+    cells.toList
   end extractHeadersFromRange
 
   /** Extract headers from the first row of the sheet This consumes the header row from the sheet iterator

scautable/src-jvm/ExcelMacros.scala

@@ -3,12 +3,11 @@ package io.github.quafadas.scautable
 import scala.quoted.*
 import io.github.quafadas.scautable.ColumnTyped.*
 import io.github.quafadas.table.TypeInferrer
-import org.apache.poi.ss.usermodel.{Row, WorkbookFactory, Cell, CellType, DateUtil}
+import org.apache.poi.ss.usermodel.{Row, Cell, CellType, DateUtil}
 import org.apache.poi.ss.util.CellRangeAddress
 import scala.collection.JavaConverters.*
-import java.io.File
 import io.github.quafadas.scautable.BadTableException
-import io.github.quafadas.scautable.InferrerOps
+import io.github.quafadas.scautable.ExcelWorkbookCache
 
 /** Compile-time macro functions for reading     val initial = ColumnTypeInfo()
     val finalInfo = cells.foldLeft(initial)(updateTypeInfo) files These macros perform Excel file inspection at compile time to determine structure
@@ -128,24 +127,23 @@ object ExcelMacros:
   /** Extracts headers from an Excel sheet, either from a specific range or the first row
     */
   private def extractHeaders(filePath: String, sheetName: String, colRange: Option[String]): List[String] =
-    val workbook = WorkbookFactory.create(new File(filePath))
-    try
-      val sheet = workbook.getSheet(sheetName)
-
-      colRange match
-        case Some(range) if range.nonEmpty =>
-          val cellRange = CellRangeAddress.valueOf(range)
-          val firstRow = sheet.getRow(cellRange.getFirstRow)
-          val cells =
-            for (i <- cellRange.getFirstColumn to cellRange.getLastColumn)
-              yield firstRow.getCell(i, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).toString
-          cells.toList
-        case _ =>
-          if sheet.iterator().hasNext then sheet.iterator().next().cellIterator().asScala.toList.map(_.toString)
-          else throw new BadTableException("No headers found in the first row of the sheet, and no range specified.")
-      end match
-    finally workbook.close()
-    end try
+    val workbook = ExcelWorkbookCache.getOrCreate(filePath).getOrElse(
+      throw new BadTableException(s"Failed to open Excel file: $filePath")
+    )
+    val sheet = workbook.getSheet(sheetName)
+
+    colRange match
+      case Some(range) if range.nonEmpty =>
+        val cellRange = CellRangeAddress.valueOf(range)
+        val firstRow = sheet.getRow(cellRange.getFirstRow)
+        val cells =
+          for (i <- cellRange.getFirstColumn to cellRange.getLastColumn)
+            yield firstRow.getCell(i, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).toString
+        cells.toList
+      case _ =>
+        if sheet.iterator().hasNext then sheet.iterator().next().cellIterator().asScala.toList.map(_.toString)
+        else throw new BadTableException("No headers found in the first row of the sheet, and no range specified.")
+    end match
   end extractHeaders
 
   /** Validates that headers are unique (no duplicates)
@@ -200,9 +198,10 @@ object ExcelMacros:
   ): quotes.reflect.TypeRepr =
     import quotes.reflect.*
 
-    val workbook = WorkbookFactory.create(new File(filePath))
-    try
-      val sheet = workbook.getSheet(sheetName)
+    val workbook = ExcelWorkbookCache.getOrCreate(filePath).getOrElse(
+      throw new BadTableException(s"Failed to open Excel file: $filePath")
+    )
+    val sheet = workbook.getSheet(sheetName)
 
       // Extract data based on column range or use all columns
       val columnData: List[List[Cell]] = colRange match
@@ -251,14 +250,6 @@ object ExcelMacros:
       }
 
       tupleType
-    finally 
-      try 
-        workbook.close()
-      catch
-        case _: Exception => 
-          // Workbook close can fail for Excel files with corrupted drawings - this is expected
-          println(s"Warning: Could not close workbook for file: $filePath")
-      end try
   end inferTypesFromExcelDataDirect
 
   /** Infer the most appropriate Scala type for a column based on Apache POI cell types

scautable/src-jvm/ExcelResourceManager.scala

@@ -0,0 +1,27 @@
+package io.github.quafadas.scautable
+
+/** Utility object for managing Excel workbook resources and lifecycle
+  */
+object ExcelResourceManager:
+  
+  /** Cleanup resources for a specific Excel file when no longer needed.
+    * 
+    * This method should be called when you know that no more operations
+    * will be performed on a specific Excel file to free up memory immediately.
+    * 
+    * @param filePath Path to the Excel file to cleanup
+    */
+  def cleanup(filePath: String): Unit =
+    ExcelWorkbookCache.closeAndRemove(filePath)
+  end cleanup
+  
+  /** Cleanup all cached Excel workbook resources.
+    * 
+    * This method is useful for application shutdown or when you want to
+    * free up all Excel-related memory immediately.
+    */
+  def cleanupAll(): Unit =
+    ExcelWorkbookCache.clearAll()
+  end cleanupAll
+
+end ExcelResourceManager

scautable/src-jvm/ExcelUtils.scala

@@ -1,10 +1,10 @@
 package io.github.quafadas.scautable
 
-import org.apache.poi.ss.usermodel.{Row, WorkbookFactory}
+import org.apache.poi.ss.usermodel.Row
 import org.apache.poi.ss.util.CellRangeAddress
 import scala.collection.JavaConverters.*
-import java.io.File
 import io.github.quafadas.scautable.BadTableException
+import io.github.quafadas.scautable.ExcelWorkbookCache
 
 /** Common utilities and exceptions for Excel processing
   */
@@ -22,24 +22,23 @@ object ExcelUtils:
     *   List of header strings
     */
   inline def extractHeaders(filePath: String, sheetName: String, colRange: Option[String]): List[String] =
-    val workbook = WorkbookFactory.create(new File(filePath))
-    try
-      val sheet = workbook.getSheet(sheetName)
+    val workbook = ExcelWorkbookCache.getOrCreate(filePath).getOrElse(
+      throw new BadTableException(s"Failed to open Excel file: $filePath")
+    )
+    val sheet = workbook.getSheet(sheetName)
 
-      colRange match
-        case Some(range) if range.nonEmpty =>
-          val cellRange = CellRangeAddress.valueOf(range)
-          val firstRow = sheet.getRow(cellRange.getFirstRow)
-          val cells =
-            for (i <- cellRange.getFirstColumn to cellRange.getLastColumn)
-              yield firstRow.getCell(i, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).toString
-          cells.toList
-        case _ =>
-          if sheet.iterator().hasNext then sheet.iterator().next().cellIterator().asScala.toList.map(_.toString)
-          else throw new BadTableException("No headers found in the first row of the sheet, and no range specified.")
-      end match
-    finally workbook.close()
-    end try
+    colRange match
+      case Some(range) if range.nonEmpty =>
+        val cellRange = CellRangeAddress.valueOf(range)
+        val firstRow = sheet.getRow(cellRange.getFirstRow)
+        val cells =
+          for (i <- cellRange.getFirstColumn to cellRange.getLastColumn)
+            yield firstRow.getCell(i, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).toString
+        cells.toList
+      case _ =>
+        if sheet.iterator().hasNext then sheet.iterator().next().cellIterator().asScala.toList.map(_.toString)
+        else throw new BadTableException("No headers found in the first row of the sheet, and no range specified.")
+    end match
   end extractHeaders
 
   /** Validates that headers are unique (no duplicates)

scautable/src-jvm/ExcelWorkbookCache.scala

@@ -0,0 +1,104 @@
+package io.github.quafadas.scautable
+
+import org.apache.poi.ss.usermodel.Workbook
+import org.apache.poi.ss.usermodel.WorkbookFactory
+import java.io.File
+import java.lang.ref.WeakReference
+import java.util.concurrent.ConcurrentHashMap
+import scala.util.Try
+
+/** Thread-safe cache for Excel workbook instances to avoid file contention and improve performance.
+  * 
+  * This cache uses weak references to allow workbooks to be garbage collected when no longer needed,
+  * while providing significant performance improvements when multiple operations access the same Excel file.
+  */
+object ExcelWorkbookCache:
+  
+  // Thread-safe cache using weak references to allow GC when workbooks are no longer used
+  private val cache = new ConcurrentHashMap[String, WeakReference[Workbook]]()
+  
+  /** Get or create a workbook for the specified file path.
+    * 
+    * This method is thread-safe and will reuse existing workbook instances when possible.
+    * If a workbook is garbage collected, a new one will be created automatically.
+    * 
+    * @param filePath The absolute path to the Excel file
+    * @return A Try containing the Workbook instance, or a Failure if the file cannot be opened
+    */
+  def getOrCreate(filePath: String): Try[Workbook] =
+    Try {
+      // Normalize the file path to handle different path representations
+      val normalizedPath = new File(filePath).getCanonicalPath
+      
+      // Try to get existing workbook from cache
+      val cachedRef = cache.get(normalizedPath)
+      val existingWorkbook = Option(cachedRef).flatMap(ref => Option(ref.get()))
+      
+      existingWorkbook match
+        case Some(workbook) =>
+          // Validate that the workbook is still usable (not closed)
+          try
+            // Simple validation - try to access the number of sheets
+            workbook.getNumberOfSheets
+            workbook
+          catch
+            case _: Exception =>
+              // Workbook is no longer valid, remove from cache and create new one
+              cache.remove(normalizedPath)
+              WorkbookFactory.create(new File(normalizedPath))
+        case None =>
+          // No cached workbook or it was garbage collected
+          val workbook = WorkbookFactory.create(new File(normalizedPath))
+          cache.put(normalizedPath, new WeakReference(workbook))
+          workbook
+    }
+  end getOrCreate
+  
+
+  
+  /** Explicitly close and remove a workbook from the cache.
+    * 
+    * This method should be called when you know a workbook will no longer be needed
+    * to free up resources immediately rather than waiting for garbage collection.
+    * 
+    * @param filePath The path to the Excel file
+    */
+  def closeAndRemove(filePath: String): Unit =
+    try
+      val normalizedPath = new File(filePath).getCanonicalPath
+      val cachedRef = cache.remove(normalizedPath)
+      
+      Option(cachedRef).flatMap(ref => Option(ref.get())).foreach { workbook =>
+        try
+          workbook.close()
+        catch
+          case _: Exception =>
+            // Ignore close errors - workbook might already be corrupted/closed
+            // This is expected for Excel files with certain drawing corruption issues
+        end try
+      }
+    catch
+      case _: Exception =>
+        // Ignore errors in cleanup - this is a best-effort operation
+    end try
+  end closeAndRemove
+  
+  /** Clear the entire cache and attempt to close all cached workbooks.
+    * 
+    * This is primarily useful for testing or application shutdown.
+    */
+  def clearAll(): Unit =
+    val keys = cache.keySet().toArray(Array.empty[String])
+    keys.foreach(closeAndRemove)
+  end clearAll
+  
+  /** Get the current number of cached workbook references.
+    * 
+    * Note that this includes weak references that may have been garbage collected.
+    * This method is primarily useful for testing and debugging.
+    * 
+    * @return The number of cached workbook references
+    */
+  def cacheSize: Int = cache.size()
+
+end ExcelWorkbookCache