Support for reading JPEG compressed TIFF files where JPEG tables are stored globally in JPEGTables tag and not per tile.

Author: smistad

source/FAST/Algorithms/Compression/JPEGCompression.cpp

@@ -18,6 +18,12 @@ void* JPEGCompression::decompress(uchar* compressedData, std::size_t bytes, int*
     cinfo.err = jpeg_std_error(&jerr);
     try {
         jpeg_create_decompress(&cinfo);
+        if(m_tableCount > 0) {
+            jpeg_mem_src(&cinfo, (const uchar*)m_tableData, m_tableCount);
+            if(jpeg_read_header(&cinfo, FALSE) != JPEG_HEADER_TABLES_ONLY) {
+                throw Exception("Error setting JPEG tables");
+            }
+        }
         jpeg_mem_src(&cinfo, compressedData, bytes);
         int ret = jpeg_read_header(&cinfo, TRUE);
         if(ret != JPEG_HEADER_OK) {
@@ -83,8 +89,9 @@ void JPEGCompression::compress(void *data, int width, int height, std::vector<ui
     free(resultBuffer);
     jpeg_destroy_compress(&cinfo);
 }
-JPEGCompression::JPEGCompression() {
-
+JPEGCompression::JPEGCompression(uint32_t tableCount, void* tableData) {
+    m_tableCount = tableCount;
+    m_tableData = tableData;
 }
 
 } // End namespace

source/FAST/Algorithms/Compression/JPEGCompression.hpp

@@ -11,7 +11,7 @@ namespace fast {
  */
 class JPEGCompression {
 public:
-    JPEGCompression();
+    explicit JPEGCompression(uint32_t JPEGTableCount = 0, void* tableData = nullptr);
     void compress(void* data, int width, int height, std::vector<uint8_t>* compressedData, int quality = 90);
     /**
      * @brief Decompress
@@ -24,5 +24,8 @@ class JPEGCompression {
      */
     void* decompress(uchar* compressedData, std::size_t bytes, int* widthOut, int* heightOut, uchar* outputBuffer = nullptr);
 
+private:
+    uint32_t m_tableCount = 0;
+    const void* m_tableData = nullptr;
 };
 }

source/FAST/Data/Access/ImagePyramidAccess.cpp

@@ -546,7 +546,7 @@ int ImagePyramidAccess::readTileFromTIFF(void *data, int x, int y, int level) {
             auto buffer = make_uninitialized_unique<char[]>(tileWidth*tileHeight*channels);
             bytesRead = TIFFReadRawTile(m_tiffHandle, tile_id, buffer.get(), tileWidth*tileHeight*channels);
 
-            JPEGCompression jpeg;
+            JPEGCompression jpeg(m_JPEGTablesCount, m_JPEGTablesData);
             int width, height;
             jpeg.decompress((uchar*)buffer.get(), bytesRead, &width, &height, (uchar*)data);
         } else if(m_compressionFormat == ImageCompression::JPEGXL) {
@@ -580,5 +580,10 @@ void ImagePyramidAccess::setBlankPatch(int level, int x, int y) {
     // TODO Propagate or not?
 }
 
+void ImagePyramidAccess::setJPEGTables(uint32_t tableCount, void *tableData) {
+    m_JPEGTablesCount = tableCount;
+    m_JPEGTablesData = tableData;
+}
+
 
 }

source/FAST/Data/Access/ImagePyramidAccess.hpp

@@ -101,6 +101,7 @@ class FAST_EXPORT ImagePyramidAccess : Object {
     std::shared_ptr<Image> getPatchAsImageForMagnification(float magnification, float offsetX, float offsetY, int width, int height, bool convertToRGB = true);
 	void release();
 	~ImagePyramidAccess();
+	void setJPEGTables(uint32_t tableCount, void* tableData);
 private:
     std::unique_ptr<uchar[]> getPatchDataChar(int level, int x, int y, int width, int height);
 	std::shared_ptr<ImagePyramid> m_image;
@@ -119,6 +120,9 @@ class FAST_EXPORT ImagePyramidAccess : Object {
     uint32_t writeTileToTIFFNeuralNetwork(int level, int x, int y, std::shared_ptr<Image> image);
     int readTileFromTIFF(void* data, int x, int y, int level);
     void propagatePatch(std::shared_ptr<Image> patch, int level, int x, int y);
+
+    uint32_t m_JPEGTablesCount = 0;
+    void* m_JPEGTablesData = nullptr;
 };
 
 template <class T>

source/FAST/Data/ImagePyramid.cpp

@@ -322,7 +322,10 @@ ImagePyramidAccess::pointer ImagePyramid::getAccess(accessType type) {
         std::unique_lock<std::mutex> lock(mDataIsBeingAccessedMutex);
         mDataIsBeingAccessed = true;
     }
-    return std::make_unique<ImagePyramidAccess>(m_levels, m_fileHandle, m_tiffHandle, std::static_pointer_cast<ImagePyramid>(mPtr.lock()), type == ACCESS_READ_WRITE, m_initializedPatchList, m_readMutex, m_compressionFormat);
+    auto access =  std::make_unique<ImagePyramidAccess>(m_levels, m_fileHandle, m_tiffHandle, std::static_pointer_cast<ImagePyramid>(mPtr.lock()), type == ACCESS_READ_WRITE, m_initializedPatchList, m_readMutex, m_compressionFormat);
+    if(m_JPEGTablesCount > 0)
+        access->setJPEGTables(m_JPEGTablesCount, m_JPEGTablesData);
+    return access;
 }
 
 void ImagePyramid::setDirtyPatch(int level, int patchIdX, int patchIdY) {
@@ -437,6 +440,20 @@ ImagePyramid::ImagePyramid(TIFF *fileHandle, std::vector<ImagePyramidLevel> leve
                     throw Exception("Unable to get JPEG quality from TIFF");
                 m_compressionQuality = quality;
             }
+            {
+                // Check if there is JPEG table data stored in the TIFF
+                // If so we use this table for all tiles
+                uint32_t tablesCount = 0;
+                void *tableData;
+                int res = TIFFGetField(m_tiffHandle, TIFFTAG_JPEGTABLES, &tablesCount, &tableData);
+                if(res == 1) {
+                    reportInfo() << "Got JPEG tables data from the TIFF file with size: " << tablesCount << reportEnd();
+                    m_JPEGTablesCount = tablesCount;
+                    // Have to copy the table
+                    m_JPEGTablesData = new uchar[tablesCount];
+                    std::memcpy(m_JPEGTablesData, tableData, sizeof(uchar)*tablesCount);
+                }
+            }
             break;
         case COMPRESSION_JXL:
             compression = ImageCompression::JPEGXL;

source/FAST/Data/ImagePyramid.hpp

@@ -139,6 +139,9 @@ class FAST_EXPORT ImagePyramid : public SpatialDataObject {
         float m_decompressionOutputScaleFactor = 1.0f;
 
         float m_magnification = -1.0f;
+
+        uint32_t m_JPEGTablesCount = 0;
+        void* m_JPEGTablesData = nullptr;
 };
 
 }

source/FAST/Importers/WholeSlideImageImporter.cpp

@@ -127,7 +127,8 @@ void WholeSlideImageImporter::execute() {
             lf.find(".ome.tiff") != std::string::npos ||
             lf.find(".ome.tif") != std::string::npos ||
             lf.find(".ome.btf") != std::string::npos ||
-            lf.find(".tiff") != std::string::npos
+            lf.find(".tiff") != std::string::npos ||
+            lf.find(".tif") != std::string::npos
             ) {
         // OpenSlide doesn't support reading OME-TIFF, use FAST and lbitiff instead.
         // Also FAST supports more TIFF compression formats, thus use FAST and libtiff for regular TIFF files as well.

source/FAST/Utility.cpp

@@ -1269,7 +1269,13 @@ void Progress::update(uint64_t current) {
         if(!m_unit.empty()) {
             ss << " " << round(m_current*m_unitScale, m_unitPrecision) << m_unit << " / " << round(m_max*m_unitScale, m_unitPrecision) << m_unit;
         }
-        ss << " | ETA " << ETA << " mins";
+        if(ETA > 60) {
+            ss << " | ETA " << round(ETA / 60.0f, 1) << " hours";
+        } else if(ETA > 1) {
+            ss << " | ETA " << round(ETA) << " mins";
+        } else {
+            ss << " | ETA " << std::floor(ETA * 60.0f) << " seconds";
+        }
         std::string startString = m_text + (m_text.empty() ? "" : " ") + "[";
         const int totalWidth = getConsoleWidth();
         const int progressbarWidth = totalWidth - ss.str().size() - startString.size() - 1; // have to leave last line open to avoid jumping to next line (windows)