Improved speed of ImagePyramidAccess and PatchGenerator with some optimizations and tile caching. Also moved runtime measurement stuff from ProcessObject to Object.

Author: smistad

CMakeLists.txt

@@ -7,7 +7,7 @@ project(FAST)
 
 set(VERSION_MAJOR 4)
 set(VERSION_MINOR 13)
-set(VERSION_PATCH 0)
+set(VERSION_PATCH 1)
 set(VERSION_SO 4)   # SO version, should be incremented by 1 every time the existing API changes
 set(FAST_VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}")
 

source/FAST/Algorithms/ImagePatch/PatchGenerator.cpp

@@ -109,6 +109,19 @@ void PatchGenerator::generateStream() {
             const int patchesX = std::ceil((float) levelWidth / (float) (patchWidthWithoutOverlap*resampleFactor));
             const int patchesY = std::ceil((float) levelHeight / (float) (patchHeightWithoutOverlap*resampleFactor));
 
+            bool useTileCache = true;
+            int tileCacheSize = -1;
+            // No need to use tile cache if we are only going to get each tile exactly 1 time
+            // This happens when the request patch size is the same as the tile size of the data
+            if((int)round(m_width*resampleFactor) == m_inputImagePyramid->getLevelTileWidth(level) && (int)round(m_height*resampleFactor) == m_inputImagePyramid->getLevelTileHeight(level)) {
+                useTileCache = false;
+            } else {
+                useTileCache = true;
+                int rowsPerPatch = std::ceil((float)m_inputImagePyramid->getLevelTileHeight(level) / m_height*resampleFactor);
+                tileCacheSize = patchesX*rowsPerPatch;
+                reportInfo() << "PatchGenerator is using a tile cache with size limit of " << tileCacheSize << reportEnd();
+            }
+            auto access = m_inputImagePyramid->getAccess(ACCESS_READ, useTileCache, tileCacheSize);
             for(int patchY = 0; patchY < patchesY; ++patchY) {
                 for(int patchX = 0; patchX < patchesX; ++patchX) {
                     mRuntimeManager->startRegularTimer("create patch");
@@ -150,7 +163,6 @@ void PatchGenerator::generateStream() {
                         }
                     }
                     reportInfo() << "Generating patch " << patchX << " " << patchY << reportEnd();
-                    auto access = m_inputImagePyramid->getAccess(ACCESS_READ);
                     if(patchWidth < overlapInPixelsX*2 || patchHeight < overlapInPixelsY*2)
                         continue;
                     mRuntimeManager->startRegularTimer("getPatchAsImage");
@@ -160,6 +172,7 @@ void PatchGenerator::generateStream() {
                                                          patchWidth + (patchOffsetX < 0 ? patchOffsetX : 0), // We have to reduce width and height if negative offset
                                                          patchHeight + (patchOffsetY < 0 ? patchOffsetY : 0));
                     mRuntimeManager->stopRegularTimer("getPatchAsImage");
+                    //access->getAllRuntimes()->printAll();
 
                     // If patch does not have correct size, pad it
                     int paddingValue = m_paddingValue;

source/FAST/Data/Access/ImagePyramidAccess.cpp

@@ -14,10 +14,10 @@ class Image;
 class ImagePyramid;
 class NeuralNetwork;
 
-
 /**
  * @brief Image compression types for ImagePyramids
  *
+ * @sa ImagePyramid
  * @ingroup wsi
  */
 enum class ImageCompression {
@@ -31,41 +31,34 @@ enum class ImageCompression {
     DEFLATE, // Zlib lossless
 };
 
-class FAST_EXPORT ImagePyramidPatch {
-public:
-	std::unique_ptr<uchar[]> data;
-	int width;
-	int height;
-	int offsetX;
-	int offsetY;
-};
-
+/**
+ * @brief Object for metadata of an ImagePyramid level
+ * @sa ImagePyramid
+ * @ingroup wsi
+ */
 class FAST_EXPORT ImagePyramidLevel {
 public:
 	int width;
 	int height;
-	int tileWidth = 256;
-	int tileHeight = 256;
+	int tileWidth = 1024;
+	int tileHeight = 1024;
 	int tilesX;
     int tilesY;
-	bool memoryMapped;
-	uint8_t* data;
 	uint64_t offset = 0; // subifd offset used by OME-TIFF
-#ifdef WIN32
-	void* fileHandle;
-#else
-	int fileHandle;
-#endif
 };
 
 /**
  * @brief CPU access to ImagePyramid
  * @ingroup access
  */
-class FAST_EXPORT ImagePyramidAccess : Object {
+#ifdef SWIG
+class FAST_EXPORT ImagePyramidAccess {
+#else
+class FAST_EXPORT ImagePyramidAccess : public Object {
+#endif
 public:
 	typedef std::unique_ptr<ImagePyramidAccess> pointer;
-	ImagePyramidAccess(std::vector<ImagePyramidLevel> levels, openslide_t* fileHandle, TIFF* tiffHandle, std::shared_ptr<ImagePyramid> imagePyramid, bool writeAccess, std::unordered_set<std::string>& initializedPatchList, std::mutex& readMutex, ImageCompression compressionFormat);
+	ImagePyramidAccess(std::vector<ImagePyramidLevel> levels, openslide_t* fileHandle, TIFF* tiffHandle, std::shared_ptr<ImagePyramid> imagePyramid, bool writeAccess, std::unordered_set<std::string>& initializedPatchList, std::mutex& readMutex, ImageCompression compressionFormat, bool useCache = false, int cacheLimit = -1);
 	/**
 	 * @brief Write a patch to the pyramid
 	 * @param level
@@ -85,8 +78,32 @@ class FAST_EXPORT ImagePyramidAccess : Object {
 	bool isPatchInitialized(int level, int x, int y);
 	template <class T>
 	std::unique_ptr<T[]> getPatchData(int level, int x, int y, int width, int height);
+	/**
+	 * @brief Get a specific level in an ImagePyramid as an Image object.
+	 * If requesting a level with a width or height higher than 16384 pixels this will throw an exception.
+	 * @param level
+	 * @return
+	 */
 	std::shared_ptr<Image> getLevelAsImage(int level);
+	/**
+	 * @brief Extract a patch from the image pyramid and return it as an Image
+	 * @param level Level to extract patch from
+	 * @param offsetX X offset
+	 * @param offsetY Y offset
+	 * @param width Width of patch
+	 * @param height Height of patch
+	 * @param convertToRGB convert to RGB when using OpenSlide, since it will return BGRA data
+	 * @return Patch as Image
+	 */
 	std::shared_ptr<Image> getPatchAsImage(int level, int offsetX, int offsetY, int width, int height, bool convertToRGB = true);
+	/**
+	 * @brief Extract a tile from the Image Pyramid
+	 * @param level Level to extract tile from
+	 * @param patchIdX Tile X id
+	 * @param patchIdY  Tile Y id
+	 * @param convertToRGB convert to RGB when using OpenSlide, since it will return BGRA data
+	 * @return Tile as Image
+	 */
 	std::shared_ptr<Image> getPatchAsImage(int level, int patchIdX, int patchIdY, bool convertToRGB = true);
 	/**
 	 * @brief Get patch as Image at a specific magnification
@@ -95,12 +112,12 @@ class FAST_EXPORT ImagePyramidAccess : Object {
 	 * @param offsetY Physical offset y position of patch
 	 * @param width Width of patch in pixels
 	 * @param height Height of patch in pixels
-	 * @param convertToRGB Convert from BGR to RGB if needed
+	 * @param convertToRGB convert to RGB when using OpenSlide, since it will return BGRA data
 	 * @return patch as Image object
 	 */
     std::shared_ptr<Image> getPatchAsImageForMagnification(float magnification, float offsetX, float offsetY, int width, int height, bool convertToRGB = true);
 	void release();
-	~ImagePyramidAccess();
+	~ImagePyramidAccess() override;
 	void setJPEGTables(uint32_t tableCount, void* tableData);
 private:
     std::unique_ptr<uchar[]> getPatchDataChar(int level, int x, int y, int width, int height);
@@ -123,6 +140,14 @@ class FAST_EXPORT ImagePyramidAccess : Object {
 
     uint32_t m_JPEGTablesCount = 0;
     void* m_JPEGTablesData = nullptr;
+
+    // Optional tile cache
+    void addTileToQueue(const std::string& id);
+    std::unordered_map<std::string, std::shared_ptr<char[]>> m_tileCache; // Should ideally be a unique_ptr here, but MSVC can't handle that
+    std::deque<std::string> m_tileCacheQueue;
+    std::unordered_map<std::string, int> m_tileCacheCounter;
+    uint64_t m_tileCacheSizeLimit;
+    bool m_useTileCache;
 };
 
 template <class T>

source/FAST/Data/Access/ImagePyramidAccess.hpp

@@ -263,35 +263,20 @@ void ImagePyramid::free(ExecutionDevice::pointer device) {
 }
 
 void ImagePyramid::freeAll() {
+    m_levels.clear();
     if(m_fileHandle != nullptr) {
-        m_levels.clear();
         openslide_close(m_fileHandle);
     } else if(m_tiffHandle != nullptr) {
-        m_levels.clear();
         TIFFClose(m_tiffHandle);
         if(m_tempFile) {
             // If this is a temp file created by FAST. Delete it.
             std::remove(m_tiffPath.c_str());
         }
-    } else {
-		for(auto& item : m_levels) {
-			if(item.memoryMapped) {
-#ifdef WIN32
-				UnmapViewOfFile(item.data);
-				CloseHandle(item.fileHandle);
-#else
-				munmap(item.data, item.width*item.height*m_channels);
-				close(item.fileHandle);
-#endif
-			} else {
-				delete[] item.data;
-			}
-		}
-        m_levels.clear();
     }
 
 	m_initialized = false;
 	m_fileHandle = nullptr;
+	m_tiffHandle = nullptr;
 }
 
 ImagePyramid::~ImagePyramid() {
@@ -301,7 +286,7 @@ int ImagePyramid::getNrOfChannels() const {
     return m_channels;
 }
 
-ImagePyramidAccess::pointer ImagePyramid::getAccess(accessType type) {
+ImagePyramidAccess::pointer ImagePyramid::getAccess(accessType type, bool useTileCache, int tileCacheSize) {
     if(!m_initialized)
         throw Exception("ImagePyramid has not been initialized.");
 
@@ -322,7 +307,7 @@ ImagePyramidAccess::pointer ImagePyramid::getAccess(accessType type) {
         std::unique_lock<std::mutex> lock(mDataIsBeingAccessedMutex);
         mDataIsBeingAccessed = true;
     }
-    auto access =  std::make_unique<ImagePyramidAccess>(m_levels, m_fileHandle, m_tiffHandle, std::static_pointer_cast<ImagePyramid>(mPtr.lock()), type == ACCESS_READ_WRITE, m_initializedPatchList, m_readMutex, m_compressionFormat);
+    auto access =  std::make_unique<ImagePyramidAccess>(m_levels, m_fileHandle, m_tiffHandle, std::static_pointer_cast<ImagePyramid>(mPtr.lock()), type == ACCESS_READ_WRITE, m_initializedPatchList, m_readMutex, m_compressionFormat, useTileCache, tileCacheSize);
     if(m_JPEGTablesCount > 0)
         access->setJPEGTables(m_JPEGTablesCount, m_JPEGTablesData);
     return access;
@@ -437,7 +422,7 @@ ImagePyramid::ImagePyramid(TIFF *fileHandle, std::vector<ImagePyramidLevel> leve
                 int quality = -1;
                 int res = TIFFGetField(m_tiffHandle, TIFFTAG_JPEGQUALITY, &quality);
                 if(res != 1)
-                    throw Exception("Unable to get JPEG quality from TIFF");
+                    reportInfo() << "Unable to get JPEG quality from TIFF" << reportEnd();
                 m_compressionQuality = quality;
             }
             {

source/FAST/Data/ImagePyramid.cpp

@@ -80,7 +80,7 @@ class FAST_EXPORT ImagePyramid : public SpatialDataObject {
         std::string getTIFFPath() const;
         void setSpacing(Vector3f spacing);
         Vector3f getSpacing() const;
-        ImagePyramidAccess::pointer getAccess(accessType type);
+        ImagePyramidAccess::pointer getAccess(accessType type, bool useTileCache = false, int tileCacheSize = -1);
         std::unordered_set<std::string> getDirtyPatches();
         bool isDirtyPatch(const std::string& tileID);
         bool isOMETIFF() const;

source/FAST/Data/ImagePyramid.hpp

@@ -92,6 +92,7 @@ Object::Object() {
         std::cout << "\033[0m" << std::flush; // Reset
 #endif
     });
+    mRuntimeManager = RuntimeMeasurementsManager::New();
 }
 
 Reporter& Object::reportError() {
@@ -273,4 +274,28 @@ std::unordered_map<std::string, std::shared_ptr<Attribute>> AttributeObject::get
     return mAttributes;
 }
 
+void Object::enableRuntimeMeasurements() {
+    mRuntimeManager->enable();
+}
+
+void Object::disableRuntimeMeasurements() {
+    mRuntimeManager->disable();
+}
+
+RuntimeMeasurement::pointer Object::getRuntime() {
+    return mRuntimeManager->getTiming("execute");
+}
+
+RuntimeMeasurement::pointer Object::getRuntime(std::string name) {
+    return mRuntimeManager->getTiming(name);
+}
+
+RuntimeMeasurementsManager::pointer Object::getAllRuntimes() {
+    return mRuntimeManager;
+}
+
+RuntimeMeasurementsManager::pointer Object::getRuntimeManager() {
+    return getAllRuntimes();
+}
+
 } // end namespace fast

source/FAST/Object.cpp

@@ -2,10 +2,12 @@
 
 #define NOMINMAX // Removes windows min and max macros
 #define _USE_MATH_DEFINES
-#include "FAST/Exception.hpp"
-#include "FAST/Reporter.hpp"
+#include <FAST/Exception.hpp>
+#include <FAST/Reporter.hpp>
 #include "FASTVersion.hpp"
-#include "FAST/Attribute.hpp"
+#include <FAST/RuntimeMeasurement.hpp>
+#include <FAST/RuntimeMeasurementManager.hpp>
+#include <FAST/Attribute.hpp>
 #include <unordered_map>
 #include <memory>
 
@@ -299,12 +301,23 @@ class FAST_EXPORT  Object {
             return "Object";
         }
         Reporter& getReporter();
+
+        // Runtime stuff
+        RuntimeMeasurement::pointer getRuntime();
+        RuntimeMeasurement::pointer getRuntime(std::string name);
+        RuntimeMeasurementsManager::pointer getAllRuntimes();
+        RuntimeMeasurementsManager::pointer getRuntimeManager();
+        void enableRuntimeMeasurements();
+        void disableRuntimeMeasurements();
+
     protected:
         Reporter& reportError();
         Reporter& reportWarning();
         Reporter& reportInfo();
         ReporterEnd reportEnd() const;
         std::weak_ptr<Object> mPtr;
+
+        RuntimeMeasurementsManager::pointer mRuntimeManager;
     private:
         Reporter mReporter;
 

source/FAST/Object.hpp

@@ -12,7 +12,6 @@ namespace fast {
 
 ProcessObject::ProcessObject() : mIsModified(false) {
     mDevices[0] = DeviceManager::getInstance()->getDefaultDevice();
-    mRuntimeManager = RuntimeMeasurementsManager::New();
 }
 
 static bool isStreamer(ProcessObject* po) {
@@ -255,26 +254,6 @@ void ProcessObject::validateOutputPortExists(uint portID) {
         throw Exception(getNameOfClass() + " has no output port with ID " + std::to_string(portID));
 }
 
-void ProcessObject::enableRuntimeMeasurements() {
-    mRuntimeManager->enable();
-}
-
-void ProcessObject::disableRuntimeMeasurements() {
-    mRuntimeManager->disable();
-}
-
-RuntimeMeasurement::pointer ProcessObject::getRuntime() {
-    return mRuntimeManager->getTiming("execute");
-}
-
-RuntimeMeasurement::pointer ProcessObject::getRuntime(std::string name) {
-    return mRuntimeManager->getTiming(name);
-}
-
-RuntimeMeasurementsManager::pointer ProcessObject::getAllRuntimes() {
-    return mRuntimeManager;
-}
-
 void ProcessObject::postExecute() {
     /*
     // TODO Release input data if they are marked as "release after execute"
@@ -437,10 +416,6 @@ int ProcessObject::getLastExecuteToken() const {
     return m_lastExecuteToken;
 }
 
-RuntimeMeasurementsManager::pointer ProcessObject::getRuntimeManager() {
-    return getAllRuntimes();
-}
-
 void ProcessObject::setExecuteOnLastFrameOnly(bool executeOnLastFrameOnly) {
     m_executeOnLastFrameOnly = true;
 }