diff --git a/.gitignore b/.gitignore index 91195179f..f4b1e7cff 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ tags ipch/ .vs/ .vscode/ +# Solution filters are sometimes handy but don't belong in official repo +*.slnf # GNU build system *.cache diff --git a/DeviceAdapters/AcquireZarr/AcqZarrAdapter.cpp b/DeviceAdapters/AcquireZarr/AcqZarrAdapter.cpp new file mode 100644 index 000000000..b927b787d --- /dev/null +++ b/DeviceAdapters/AcquireZarr/AcqZarrAdapter.cpp @@ -0,0 +1,55 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: AcqZarrAdapter.cpp +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope devices. Includes the experimental StorageDevice +// +// AUTHOR: Nenad Amodaj +// Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#include "AcqZarrAdapter.h" +#include "ModuleInterface.h" +#include "AcqZarrStorage.h" + + +/////////////////////////////////////////////////////////////////////////////// +// Exported MMDevice API +/////////////////////////////////////////////////////////////////////////////// +MODULE_API void InitializeModuleData() +{ + RegisterDevice(g_AcqZarrStorage, MM::StorageDevice, "Zarr storage based on Acquire"); +} + +MODULE_API MM::Device* CreateDevice(const char* deviceName) +{ + if(deviceName == 0) + return 0; + + if(strcmp(deviceName, g_AcqZarrStorage) == 0) + return new AcqZarrStorage(); + + return 0; +} + +MODULE_API void DeleteDevice(MM::Device* pDevice) +{ + delete pDevice; +} diff --git a/DeviceAdapters/AcquireZarr/AcqZarrAdapter.h b/DeviceAdapters/AcquireZarr/AcqZarrAdapter.h new file mode 100644 index 000000000..d9c076755 --- /dev/null +++ b/DeviceAdapters/AcquireZarr/AcqZarrAdapter.h @@ -0,0 +1,48 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: AcqZarrAdapter.h +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope devices. Includes the experimental StorageDevice +// +// AUTHORS: Milos Jovanovic +// Nenad Amodaj +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#pragma once +#include "MMDevice.h" +#include "DeviceBase.h" + +////////////////////////////////////////////////////////////////////////////// +// Error codes +// +////////////////////////////////////////////////////////////////////////////// +#define ERR_INTERNAL 144002 +#define ERR_FAILED_CREATING_FILE 144003 + +#define ERR_ZARR 140100 +#define ERR_ZARR_SETTINGS 140101 +#define ERR_ZARR_NUMDIMS 140102 +#define ERR_ZARR_STREAM_CREATE 140103 +#define ERR_ZARR_STREAM_CLOSE 140104 +#define ERR_ZARR_STREAM_LOAD 140105 +#define ERR_ZARR_STREAM_APPEND 140106 +#define ERR_ZARR_STREAM_ACCESS 140107 + +static const char* g_AcqZarrStorage = "AcquireZarrStorage"; diff --git a/DeviceAdapters/AcquireZarr/AcqZarrStorage.cpp b/DeviceAdapters/AcquireZarr/AcqZarrStorage.cpp new file mode 100644 index 000000000..dfb82a591 --- /dev/null +++ b/DeviceAdapters/AcquireZarr/AcqZarrStorage.cpp @@ -0,0 +1,538 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: AcqZarrStorage.cpp +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Zarr writer based on the CZI acquire-zarr library +// +// AUTHOR: Nenad Amodaj +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#include "AcqZarrAdapter.h" +#include "AcqZarrStorage.h" +#include "zarr.h" +#include +#include +#include +#include + + +using namespace std; + + + +/////////////////////////////////////////////////////////////////////////////// +// Zarr storage + +AcqZarrStorage::AcqZarrStorage() : + initialized(false), zarrStream(nullptr), currentImageNumber(0), dataType(MM::StorageDataType_UNKNOWN) +{ + InitializeDefaultErrorMessages(); + + // set device specific error messages + SetErrorText(ERR_ZARR, "Generic Zarr writer error. Check log for more information."); + SetErrorText(ERR_INTERNAL, "Internal driver error, see log file for details"); + SetErrorText(ERR_ZARR_SETTINGS, "Error in creating Zarr settings."); + SetErrorText(ERR_ZARR_NUMDIMS, "Number of Zarr dimensions is not valid."); + SetErrorText(ERR_ZARR_STREAM_CREATE, "Error creating Zarr stream. See log for more info."); + SetErrorText(ERR_ZARR_STREAM_CLOSE, "Error closing Zarr stream. See log for more info."); + SetErrorText(ERR_ZARR_STREAM_LOAD, "Error opening an existing Zarr stream."); + SetErrorText(ERR_ZARR_STREAM_APPEND, "Error appending image to Zarr stream."); + SetErrorText(ERR_ZARR_STREAM_ACCESS, "Error accessing Zarr stream. See log for more info."); + SetErrorText(ERR_ZARR_STREAM_LOAD, "Error opening an existing Zarr stream."); + + auto ver = Zarr_get_api_version(); + + // create pre-initialization properties + // ------------------------------------ + // + + // Name + CreateProperty(MM::g_Keyword_Name, g_AcqZarrStorage, MM::String, true); + // + // Description + ostringstream os; + os << "Acquire Zarr Storage v" << ver; + CreateProperty(MM::g_Keyword_Description, os.str().c_str(), MM::String, true); +} + +AcqZarrStorage::~AcqZarrStorage() +{ + Shutdown(); +} + +void AcqZarrStorage::GetName(char* Name) const +{ + CDeviceUtils::CopyLimitedString(Name, g_AcqZarrStorage); +} + +int AcqZarrStorage::Initialize() +{ + if (initialized) + return DEVICE_OK; + + int ret(DEVICE_OK); + + UpdateStatus(); + + initialized = true; + return DEVICE_OK; +} + +int AcqZarrStorage::Shutdown() +{ + if (initialized) + { + initialized = false; + } + destroyStream(); + + return DEVICE_OK; +} + +// Never busy because all commands block +bool AcqZarrStorage::Busy() +{ + return false; +} + +/** + * Creates Zarr dataset + * + * \param handle - handle to the dataset. + * \param path - parent directory of the dataset + * \param name - name of the dataset (the actual name will follow the micro-manager convention for not overwriting) + * \param numberOfDimensions - how many dimensions + * \param shape - array of dimension sizes, from slow to fast. Y and X are always the last two. + * \param pixType - pixel type + * \param meta - JSON encoded string representing "summary" metadata. Can be empty. + * \param metaLength - length of the metadata + * \return + */ +int AcqZarrStorage::Create(int handle, const char* path, const char* name, int numberOfDimensions, const int shape[], MM::StorageDataType pixType, const char* meta, int metaLength) +{ + if (zarrStream || datasetIsOpen) + { + LogMessage("Another stream is already open. Currently this device supports only one stream."); + return ERR_ZARR_NUMDIMS; + } + + if (numberOfDimensions < 3) + { + LogMessage("Number of dimensions is lower than 3."); + return ERR_ZARR_NUMDIMS; + } + + auto settings = ZarrStreamSettings_create(); + if (!settings) + { + LogMessage("Failed creating Zarr stream settings."); + return ERR_ZARR_SETTINGS; + } + + // set path + string savePrefix(name); + string savePrefixTmp(name); + string saveRoot(path); + string dsName = string(path) + "/" + savePrefixTmp; + int counter(1); + while (boost::filesystem::exists(dsName)) + { + savePrefixTmp = savePrefix + "_" + to_string(counter++); + dsName = saveRoot + "/" + savePrefixTmp; + } + boost::system::error_code errCode; + if (!boost::filesystem::create_directory(dsName, errCode)) + return ERR_FAILED_CREATING_FILE; + + char* streamPathName = new char[dsName.size() + 1]; + strcpy(streamPathName, dsName.c_str()); + ZarrStatus status = ZarrStreamSettings_set_store(settings, + streamPathName, + dsName.size() + 1, + nullptr); + if (status != ZarrStatus_Success) + { + LogMessage(getErrorMessage(status)); + ZarrStreamSettings_destroy(settings); + return ERR_ZARR_SETTINGS; + } + + // set data type and convert to zarr + int ztype = ConvertToZarrType(pixType); + if (ztype == -1) + { + LogMessage("Pixel data type is not supported by Zarr writer " + pixType); + ZarrStreamSettings_destroy(settings); + return ERR_ZARR_SETTINGS; + } + + status = ZarrStreamSettings_set_data_type(settings, (ZarrDataType)ztype); + if (status != ZarrStatus_Success) + { + LogMessage(getErrorMessage(status)); + ZarrStreamSettings_destroy(settings); + return ERR_ZARR_SETTINGS; + } + + status = ZarrStreamSettings_reserve_dimensions(settings, numberOfDimensions); + if (status != ZarrStatus_Success) + { + LogMessage(getErrorMessage(status)); + ZarrStreamSettings_destroy(settings); + return ERR_ZARR_SETTINGS; + } + + for (size_t i = 0; i < numberOfDimensions - 2; i++) + { + ZarrDimensionProperties dimProps; + ostringstream osd; + osd << "dim-" << i; + auto dimName(osd.str()); + dimProps.name = new char[dimName.size() + 1]; + strcpy(const_cast(dimProps.name), osd.str().c_str()); + dimProps.bytes_of_name = dimName.size() + 1; + dimProps.array_size_px = shape[i]; + dimProps.chunk_size_px = 1; + dimProps.shard_size_chunks = 1; + ZarrStatus status = ZarrStreamSettings_set_dimension(settings, i, &dimProps); + if (status != ZarrStatus_Success) + { + LogMessage(getErrorMessage(status)); + ZarrStreamSettings_destroy(settings); + return ERR_ZARR_SETTINGS; + } + } + + ZarrDimensionProperties dimPropsY; + string nameY("y"); + dimPropsY.name = new char[nameY.size() + 1]; + strcpy(const_cast(dimPropsY.name), nameY.c_str()); + dimPropsY.bytes_of_name = nameY.size() + 1; + dimPropsY.array_size_px = shape[numberOfDimensions - 2]; + dimPropsY.chunk_size_px = dimPropsY.array_size_px; + dimPropsY.shard_size_chunks = 1; + dimPropsY.kind = ZarrDimensionType_Space; + + status = ZarrStreamSettings_set_dimension(settings, numberOfDimensions - 2, &dimPropsY); + if (status != ZarrStatus_Success) + { + LogMessage(getErrorMessage(status)); + ZarrStreamSettings_destroy(settings); + return ERR_ZARR_SETTINGS; + } + + ZarrDimensionProperties dimPropsX; + string nameX("x"); + dimPropsX.name = new char[nameX.size() + 1]; + strcpy(const_cast(dimPropsX.name), nameX.c_str()); + dimPropsX.bytes_of_name = nameX.size() + 1; + dimPropsX.array_size_px = shape[numberOfDimensions - 1]; + dimPropsX.chunk_size_px = dimPropsX.array_size_px; + dimPropsX.shard_size_chunks = 1; + dimPropsX.kind = ZarrDimensionType_Space; + + status = ZarrStreamSettings_set_dimension(settings, numberOfDimensions - 1, &dimPropsX); + if (status != ZarrStatus_Success) + { + LogMessage(getErrorMessage(status)); + ZarrStreamSettings_destroy(settings); + return ERR_ZARR_SETTINGS; + } + + if (metaLength > 0) + { + status = ZarrStreamSettings_set_custom_metadata(settings, meta, metaLength); + if (status != ZarrStatus_Success) + { + LogMessage("Invalid summary metadata."); + ZarrStreamSettings_destroy(settings); + return ERR_ZARR_SETTINGS; + } + } + + zarrStream = ZarrStream_create(settings, ZarrVersion_2); + if (zarrStream == nullptr) + { + LogMessage("Failed creating Zarr stream: " + dsName); + ZarrStreamSettings_destroy(settings); + return ERR_ZARR_STREAM_CREATE; + } + + dataType = pixType; + streamDimensions.clear(); + for (int i = 0; i < numberOfDimensions; i++) streamDimensions.push_back(shape[i]); + // TODO: allow many streams + + currentImageNumber = 0; + + datasetIsOpen = true; + theHandle = handle; + + ZarrStreamSettings_destroy(settings); + + streamPath = dsName; + + return DEVICE_OK; +} + +int AcqZarrStorage::ConfigureDimension(int handle, int dimension, const char* name, const char* meaning) +{ + return DEVICE_OK; +} + +int AcqZarrStorage::ConfigureCoordinate(int handle, int dimension, int coordinate, const char* name) +{ + return DEVICE_OK; +} + +int AcqZarrStorage::Close(int handle) +{ + if (zarrStream == nullptr) + { + LogMessage("No stream is currently open."); + return ERR_ZARR_STREAM_CLOSE; + } + if (handle != theHandle) + { + LogMessage("Handle is not valid."); + return ERR_ZARR_STREAM_CLOSE; + } + + streamPath.clear(); + destroyStream(); + datasetIsOpen = false; + theHandle = -1; + + return DEVICE_OK; +} + +int AcqZarrStorage::Load(int handle, const char* path) +{ + return DEVICE_NOT_YET_IMPLEMENTED; +} + +int AcqZarrStorage::GetShape(int handle, int shape[]) +{ + if (zarrStream == nullptr) + { + LogMessage("No stream is currently open."); + return ERR_ZARR_STREAM_ACCESS; + } + if (theHandle != handle) + { + LogMessage("Handle is not valid."); + return ERR_ZARR_STREAM_ACCESS; + } + + int i(0); + for (auto d : streamDimensions) shape[i++] = d; + + return DEVICE_OK; +} + +int AcqZarrStorage::Delete(int handle) +{ + return DEVICE_NOT_YET_IMPLEMENTED; +} + +int AcqZarrStorage::List(const char* path, char** listOfDatasets, int maxItems, int maxItemLength) +{ + return DEVICE_NOT_YET_IMPLEMENTED; +} + +int AcqZarrStorage::AddImage(int handle, int sizeInBytes, unsigned char* pixels, int coordinates[], int numCoordinates, const char* imageMeta, int metaLength) +{ + // acquire-zarr supports append-only images + // TODO: check if the coordinates are coming in the right order and return run-time error if they dont + return AppendImage(handle, sizeInBytes, pixels, imageMeta, metaLength); +} + +int AcqZarrStorage::AppendImage(int handle, int sizeInBytes, unsigned char* pixels, const char* imageMeta, int metaLength) +{ + if (zarrStream == nullptr) + { + LogMessage("No stream is currently open."); + return ERR_ZARR_STREAM_ACCESS; + } + if (theHandle != handle) + { + LogMessage("Handle is not valid."); + return ERR_ZARR_STREAM_ACCESS; + } + + if (streamDimensions[streamDimensions.size() - 2] * streamDimensions[streamDimensions.size() - 1] * MM::GetPixelDataSizeInBytes(dataType) != sizeInBytes) + { + LogMessage("Stream dimensions do not match image size"); + return ERR_ZARR_STREAM_APPEND; + } + + size_t bytesIn(sizeInBytes); + size_t bytesOut(0); + ZarrStatus status = ZarrStream_append(zarrStream, pixels, bytesIn, &bytesOut); + if (status != ZarrStatus_Success) + { + LogMessage(getErrorMessage(status)); + return ERR_ZARR_STREAM_APPEND; + } + + if (bytesOut != bytesIn) + { + ostringstream os; + os << "Bytes in " << bytesIn << " does not match bytes out " << bytesOut; + LogMessage(os.str()); + return ERR_ZARR_STREAM_APPEND; + } + currentImageNumber++; + + return DEVICE_OK; +} + +int AcqZarrStorage::GetSummaryMeta(int handle, char** meta) +{ + if (zarrStream == nullptr) + { + LogMessage("No stream is currently open."); + return ERR_ZARR_STREAM_ACCESS; + } + if (theHandle != handle) + { + LogMessage("Handle is not valid."); + return ERR_ZARR_STREAM_ACCESS; + } + + // TODO: + *meta = new char[1]; + meta[0] = 0; + + return DEVICE_OK; +} + +int AcqZarrStorage::GetImageMeta(int handle, int coordinates[], int numCoordinates, char** meta) +{ + if (zarrStream == nullptr) + { + LogMessage("No stream is currently open."); + return ERR_ZARR_STREAM_ACCESS; + } + if (theHandle != handle) + { + LogMessage("Handle is not valid."); + return ERR_ZARR_STREAM_ACCESS; + } + + // TODO: + *meta = new char[1]; + meta[0] = 0; + + return DEVICE_OK; +} + +const unsigned char* AcqZarrStorage::GetImage(int handle, int coordinates[], int numCoordinates) +{ + if (zarrStream == nullptr) + { + LogMessage("No stream is currently open."); + return nullptr; + } + if (theHandle != handle) + { + LogMessage("Handle is not valid."); + return nullptr; + } + + return nullptr; +} + +int AcqZarrStorage::GetNumberOfDimensions(int handle, int& numDimensions) +{ + if (theHandle != handle) + { + LogMessage("Handle is not valid."); + return ERR_ZARR_STREAM_ACCESS; + } + return streamDimensions.size(); +} + +int AcqZarrStorage::GetDimension(int handle, int dimension, char* name, int nameLength, char* meaning, int meaningLength) +{ + return DEVICE_NOT_YET_IMPLEMENTED; +} + +int AcqZarrStorage::GetCoordinate(int handle, int dimension, int coordinate, char* name, int nameLength) +{ + return DEVICE_NOT_YET_IMPLEMENTED; +} + +int AcqZarrStorage::GetImageCount(int handle, int& imgcount) +{ + return DEVICE_NOT_YET_IMPLEMENTED; +} + +bool AcqZarrStorage::IsOpen(int handle) +{ + if (theHandle != handle) + { + return false; + } + return true; +} + +bool AcqZarrStorage::IsReadOnly(int handle) +{ + return false; +} + +int AcqZarrStorage::GetPath(int handle, char* path, int maxPathLength) +{ + return 0; +} + +std::string AcqZarrStorage::getErrorMessage(int code) +{ + return std::string(Zarr_get_error_message((ZarrStatus)code)); +} + +void AcqZarrStorage::destroyStream() +{ + if (zarrStream) + { + ZarrStream_destroy(zarrStream); + zarrStream = nullptr; + } +} + +int AcqZarrStorage::ConvertToZarrType(MM::StorageDataType type) +{ + ZarrDataType ztype; + switch (type) + { + case MM::StorageDataType_GRAY8: + ztype = ZarrDataType_uint8; + break; + + case MM::StorageDataType_GRAY16: + ztype = ZarrDataType_int16; // why is there no uint16? + break; + + default: + return -1; + } + return (int)ztype; +} diff --git a/DeviceAdapters/AcquireZarr/AcqZarrStorage.h b/DeviceAdapters/AcquireZarr/AcqZarrStorage.h new file mode 100644 index 000000000..a78df26d0 --- /dev/null +++ b/DeviceAdapters/AcquireZarr/AcqZarrStorage.h @@ -0,0 +1,93 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: AcqZarrStorage.h +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope devices. Includes the experimental StorageDevice +// +// AUTHOR: Nenad Amodaj +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// + +#pragma once +#include "MMDevice.h" +#include "DeviceBase.h" + +struct ZarrStream_s; + +class AcqZarrStorage : public CStorageBase +{ +public: + AcqZarrStorage(); + virtual ~AcqZarrStorage(); + + // Device API + // ---------- + int Initialize(); + int Shutdown(); + + void GetName(char* pszName) const; + bool Busy(); + + // Storage API + // ----------- + int Create(int handle, const char* path, const char* name, int numberOfDimensions, const int shape[], MM::StorageDataType pixType, const char* meta, int metaLength); + int ConfigureDimension(int handle, int dimension, const char* name, const char* meaning); + int ConfigureCoordinate(int handle, int dimension, int coordinate, const char* name); + int Close(int handle); + int Load(int handle, const char* path); + int GetShape(int handle, int shape[]); + int GetDataType(int handle, MM::StorageDataType& pixelDataType) { return dataType; } + + int Delete(int handle); + int List(const char* path, char** listOfDatasets, int maxItems, int maxItemLength); + int AddImage(int handle, int sizeInBytes, unsigned char* pixels, int coordinates[], int numCoordinates, const char* imageMeta, int metaLength); + int AppendImage(int handle, int sizeInBytes, unsigned char* pixels, const char* imageMeta, int metaLength); + int GetSummaryMeta(int handle, char** meta); + int GetImageMeta(int handle, int coordinates[], int numCoordinates, char** meta); + const unsigned char* GetImage(int handle, int coordinates[], int numCoordinates); + int GetNumberOfDimensions(int handle, int& numDimensions); + int GetDimension(int handle, int dimension, char* name, int nameLength, char* meaning, int meaningLength); + int GetCoordinate(int handle, int dimension, int coordinate, char* name, int nameLength); + int GetImageCount(int handle, int& imgcnt); + bool IsOpen(int handle); + bool IsReadOnly(int handle); + int GetPath(int handle, char* path, int maxPathLength); + int SetCustomMetadata(int handle, const char* key, const char* content, int contentLength) { return DEVICE_UNSUPPORTED_COMMAND; } + int GetCustomMetadata(int handle, const char* key, char** content) { return DEVICE_UNSUPPORTED_COMMAND; } + void ReleaseStringBuffer(char* buffer) { delete[] buffer; } + + // action interface + // ---------------- + +private: + bool initialized; + ZarrStream_s* zarrStream; + std::vector streamDimensions; + MM::StorageDataType dataType; + std::vector currentCoordinate; + int currentImageNumber; + bool datasetIsOpen = false; // May be redundant with zarrStream != nullptr + int theHandle = -1; // Only one dataset/handle supported at a time + std::string getErrorMessage(int code); + void destroyStream(); + int ConvertToZarrType(MM::StorageDataType type); + std::string streamPath; +}; + diff --git a/DeviceAdapters/AcquireZarr/AcquireZarr.vcxproj b/DeviceAdapters/AcquireZarr/AcquireZarr.vcxproj new file mode 100644 index 000000000..86d4a1c60 --- /dev/null +++ b/DeviceAdapters/AcquireZarr/AcquireZarr.vcxproj @@ -0,0 +1,100 @@ + + + + + Debug + x64 + + + Release + x64 + + + + + + + + + + + + + {b8c95f39-54bf-40a9-807b-598df2821d55} + + + + 17.0 + Win32Proj + {293b79ce-124b-4323-a21a-e2dcc8994d52} + AcquireZarr + 10.0 + + + + DynamicLibrary + true + v142 + Unicode + + + DynamicLibrary + false + v142 + true + Unicode + + + + + + + + + + + + + + + + + + + + true + NOMINMAX;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + true + $(MM_3RDPARTYPRIVATE)\czi\acquire-zarr\include;%(AdditionalIncludeDirectories) + stdcpp17 + + + Windows + true + false + $(MM_3RDPARTYPRIVATE)\CZI\acquire-zarr\lib;%(AdditionalLibraryDirectories) + acquire-zarr.lib;%(AdditionalDependencies) + + + + + true + true + true + NOMINMAX;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + true + $(MM_3RDPARTYPRIVATE)\czi\acquire-zarr\include;%(AdditionalIncludeDirectories) + stdcpp17 + + + Windows + true + false + $(MM_3RDPARTYPRIVATE)\CZI\acquire-zarr\lib;%(AdditionalLibraryDirectories) + acquire-zarr.lib;%(AdditionalDependencies) + + + + + + \ No newline at end of file diff --git a/DeviceAdapters/AcquireZarr/AcquireZarr.vcxproj.filters b/DeviceAdapters/AcquireZarr/AcquireZarr.vcxproj.filters new file mode 100644 index 000000000..3637fd50d --- /dev/null +++ b/DeviceAdapters/AcquireZarr/AcquireZarr.vcxproj.filters @@ -0,0 +1,33 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + diff --git a/DeviceAdapters/Go2Scope/CFileUtil.cpp b/DeviceAdapters/Go2Scope/CFileUtil.cpp new file mode 100644 index 000000000..c69e4e279 --- /dev/null +++ b/DeviceAdapters/Go2Scope/CFileUtil.cpp @@ -0,0 +1,113 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SFileUtil.cpp +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope devices. Includes the experimental StorageDevice +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#include "G2SFileUtil.h" + +/** + * Write integer value to a byte buffer + * @param buff Byte buffer + * @param len Value length (in bytes) + * @param val Integer value + * @author Miloš Jovanović + * @version 1.0 + */ +void writeInt(unsigned char* buff, std::uint8_t len, std::uint64_t val) noexcept +{ + if(buff == nullptr || len == 0) + return; + for(auto i = 0; i < len; i++) + buff[i] = (val >> (i * 8)) & 0xff; +} + +/** + * Read integer value from a byte buffer + * @param buff Byte buffer + * @param len Value length (in bytes) + * @return Integer value + * @author Miloš Jovanović + * @version 1.0 + */ +std::uint64_t readInt(const unsigned char* buff, std::uint8_t len) noexcept +{ + if(buff == nullptr || len == 0 || len > 8) + return 0; + std::uint64_t ret = 0; + for(std::uint8_t i = 0; i < len; i++) + { + auto shift = i * 8; + std::uint64_t xval = (std::uint64_t)buff[i] << shift; + ret |= xval; + } + return ret; +} + +/** + * Split CSV line into tokens + * @param line CSV line + * @return Tokens list + */ +std::vector splitLineCSV(const std::string& line) noexcept +{ + std::vector ret; + if(line.empty()) + return ret; + + std::string curr = ""; + bool qopen = false; + int qcnt = 0; + for(char c : line) + { + bool endswithQ = curr.size() >= 1 && curr[curr.size() - 1] == '\"'; + bool endswithS = curr.size() >= 1 && curr[curr.size() - 1] == ' '; + bool endswithEQ = curr.size() >= 2 && curr[curr.size() - 1] == '\"' && curr[curr.size() - 1] == '\\'; + if(c == ',' && (!qopen || (qcnt % 2 == 0 && (endswithQ || endswithS) && !endswithEQ))) + { + if(curr.size() >= 2 && curr[0] == '\"' && curr[curr.size() - 1] == '\"') + curr = curr.substr(1, curr.size() - 2); + ret.push_back(curr); + curr = ""; + qcnt = 0; + qopen = false; + } + else if(c == '"') + { + if(qcnt == 0) + qopen = true; + qcnt++; + //if(qcnt > 1 && qcnt % 2 == 1) + curr += "\""; + } + else + curr += c; + } + if(!curr.empty()) + { + if(curr.size() >= 2 && curr[0] == '\"' && curr[curr.size() - 1] == '\"') + curr = curr.substr(1, curr.size() - 2); + ret.push_back(curr); + } + return ret; +} diff --git a/DeviceAdapters/Go2Scope/G2SBigTiffDataset.cpp b/DeviceAdapters/Go2Scope/G2SBigTiffDataset.cpp new file mode 100644 index 000000000..116890756 --- /dev/null +++ b/DeviceAdapters/Go2Scope/G2SBigTiffDataset.cpp @@ -0,0 +1,1158 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2STiffFile.cpp +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope devices. Includes the experimental StorageDevice +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#define _LARGEFILE64_SOURCE +#include +#include +#include +#include +#include "G2SBigTiffDataset.h" +#include +#ifdef _WIN32 +#include +#else +#include +#include +#include +#include +#endif + +#define G2SFOLDER_EXT ".g2s" +#define G2SFILE_EXT ".g2s.tif" +#define G2SAXISINFO_FILE "axisinfo.txt" +#define G2SCUSTOMMETA_FOLDER "custommeta" + +/** + * Class constructor + * Constructor doesn't open the file, just creates an object set sets the configuration + * By convention G2S format files end with a .g2s.tif extension + * First data chunk doesn't have a index (e.g. SampleDataset.g2s.tif) + * Other data chunks contain an index (1-based, e.g. SampleDataset_1.g2s.tif, SampleDataset_2.g2s.tif..) + * Dataset files are contained within a directory. The name of the directory matches the dataset name with the .g2s sufix (e.g. SampleDataset.g2s) + */ +G2SBigTiffDataset::G2SBigTiffDataset() noexcept +{ + dspath = ""; + datasetuid = ""; + bitdepth = 8; + samples = 1; + imgcounter = 0; + flushcnt = 0; + chunksize = 0; + directIo = false; + bigTiff = true; + writemode = false; + datasetHandle = -1; +} + +/** + * Create a dataset + * All datasets are stored in separate folders, folder names have a .g2s suffix + * If the folder with the specified name already exists, a name with the index in the suffix will be used + * If the dataset is chunked files will be created only when the active chunk is filled + * @param path Dataset (folder) path + * @param dio Use direct I/O + * @param fbig Use BigTIFF format + * @param chunksz Chunk size + * @throws std::runtime_error + */ +void G2SBigTiffDataset::create(const std::string& path, bool dio, bool fbig, std::uint32_t chunksz) +{ + if(isOpen()) + throw std::runtime_error("Invalid operation. Dataset is already created"); + if(path.empty()) + throw std::runtime_error("Unable to create a file stream. Dataset path is undefined"); + directIo = dio; + writemode = true; + chunksize = chunksz; + bigTiff = fbig; + + // Extract dataset name + std::filesystem::path basepath = std::filesystem::u8path(path); + dsname = basepath.stem().u8string(); + if(dsname.find(".g2s") == dsname.size() - 4) + dsname = dsname.substr(0, dsname.size() - 4); + + // Determine dataset path + std::uint32_t counter = 1; + std::filesystem::path xpath = basepath.parent_path() / (dsname + G2SFOLDER_EXT); + while(std::filesystem::exists(xpath)) + { + // If the file path (path + name) exists, it should not be an error + // nor the file should be overwritten, first available suffix (index) will be appended to the file name + auto tmpname = dsname + "_" + std::to_string(counter++) + G2SFOLDER_EXT; + xpath = basepath.parent_path() / tmpname; + } + dspath = xpath.u8string(); + + + // Create a first file (data chunk) + std::error_code ec; + std::filesystem::path fp = xpath / (dsname + G2SFILE_EXT); + std::filesystem::create_directories(fp.parent_path(), ec); + if(ec.value() != 0) + throw std::runtime_error("Unable to create a file stream. Directory tree creation failed"); + activechunk = std::make_shared(fp.u8string(), directIo, bigTiff); + if(!activechunk) + throw std::runtime_error("Unable to create a file stream. Data chunk allocation failed"); + activechunk->open(true); + if(activechunk->getHeader().empty()) + throw std::runtime_error("Unable to create a file stream. File header creation failed"); + if(!datasetuid.empty()) + activechunk->writeDatasetUid(datasetuid); + if(!shape.empty()) + activechunk->writeShapeInfo(shape, chunksize); + datachunks.push_back(activechunk); +} + +/** + * Load a dataset + * If the dataset doesn't exist an exception will be thrown + * If the dataset exists dataset parameters and metadata will be parsed + * If the dataset is chunked all files will be enumerated, but only the first file will be loaded + * @param path Dataset (folder) path or File path of the first data chunk + * @param dio Use direct I/O + * @throws std::runtime_error + */ +void G2SBigTiffDataset::load(const std::string& path, bool dio) +{ + if(isOpen()) + throw std::runtime_error("Invalid operation. Dataset is already loaded"); + if(path.empty()) + throw std::runtime_error("Unable to load a dataset. Dataset path is undefined"); + directIo = dio; + writemode = false; + + // Check dataset / file path + auto xp = std::filesystem::u8path(path); + if(!std::filesystem::exists(xp)) + { + // Check if the dataset path has a .g2s extension + std::string fpath(path); + if(fpath.find(".g2s") != fpath.size() - 4) + fpath += ".g2s"; + xp = std::filesystem::u8path(path); + if(!std::filesystem::exists(xp)) + throw std::runtime_error("Unable to load a dataset. Specified path doesn't exist"); + } + + // If the first data chunk (file) path is specified -> use parent folder path + if(std::filesystem::is_regular_file(xp)) + xp = xp.parent_path(); + dspath = xp.u8string(); + dsname = xp.stem().u8string(); + if(dsname.find(".g2s") == dsname.size() - 4) + dsname = dsname.substr(0, dsname.size() - 4); + + // Determine file name prefix + std::string fprefix = ""; + for(const auto& entry : std::filesystem::directory_iterator(xp)) + { + // Skip auto folder paths + auto fname = entry.path().filename().u8string(); + if(fname == "." || fname == "..") + continue; + + // Skip folders + if(std::filesystem::is_directory(entry)) + continue; + + // Skip unsupported file formats + auto fext = entry.path().extension().u8string(); + if(fext.size() == 0) + continue; + if(fext[0] == '.') + fext = fext.substr(1); + std::transform(fext.begin(), fext.end(), fext.begin(), [](char c) { return (char)tolower(c); }); + if(fext != "tiff" && fext != "tif" && fext != "g2s.tiff" && fext != "g2s.tif") + continue; + + auto fx = fname.substr(0, fname.length() - fext.size() - 1); + if(fx.find(".g2s") != std::string::npos) + fx = fx.substr(0, fname.find(".g2s")); + + if(fprefix.empty()) + // First file -> use the entire file name + fprefix = fx; + else if(fx.find(fprefix) == 0) + // File name starts with the file prefix + continue; + else + { + // File prefix is invalid -> Try shorter file prefix + auto lind = fprefix.find_last_of('_'); + while(lind != std::string::npos) + { + fprefix = fprefix.substr(0, lind); + if(fx.find(fprefix) == 0) + break; + lind = fprefix.find_last_of('_'); + } + if(fx.find(fprefix) != 0) + throw std::runtime_error("Unable to load a dataset. Data chunk file name missmatch"); + } + } + + // Enumerate files + std::vector dcindex; + for(const auto& entry : std::filesystem::directory_iterator(xp)) + { + // Skip auto folder paths + auto fname = entry.path().filename().u8string(); + if(fname == "." || fname == "..") + continue; + + // Skip folders + if(std::filesystem::is_directory(entry)) + continue; + + // Skip unsupported file formats + auto fext = entry.path().extension().u8string(); + if(fext.size() == 0) + continue; + if(fext[0] == '.') + fext = fext.substr(1); + std::transform(fext.begin(), fext.end(), fext.begin(), [](char c) { return (char)tolower(c); }); + if(fext != "tiff" && fext != "tif" && fext != "g2s.tiff" && fext != "g2s.tif") + continue; + + // Determine absolute path and create chunk descriptor + auto abspath = std::filesystem::absolute(entry).u8string(); + auto dchunk = std::make_shared(abspath, directIo); + + // Determine chunk index from a file name + std::uint32_t cind = 0; + auto findtoken = fname.substr(0, fname.length() - fext.size() - 1); + if(findtoken.find(".g2s") != std::string::npos) + findtoken = findtoken.substr(0, fname.find(".g2s")); + findtoken = findtoken.substr(fprefix.length()); + if(!findtoken.empty()) + { + if(findtoken[0] == '_') + findtoken = findtoken.substr(1); + try { cind = std::stoul(findtoken); } catch(...) { } + } + + // Check if chunk index is valid and determine insert index + std::int64_t iind = -1; + for(std::size_t i = 0; i < dcindex.size(); i++) + { + if(dcindex[i] >= cind) + { + iind = (std::int64_t)i; + break; + } + } + if(iind < 0) + { + datachunks.push_back(dchunk); + dcindex.push_back(cind); + } + else + { + auto cit = datachunks.begin(); + std::advance(cit, iind); + auto iit = dcindex.begin(); + std::advance(iit, iind); + + datachunks.insert(cit, dchunk); + dcindex.insert(iit, cind); + } + } + if(datachunks.empty()) + throw std::runtime_error("Unable to load a dataset. No files found"); + + // Load first data chunk + try + { + samples = 1; + imgcounter = 0; + metadata.clear(); + custommeta.clear(); + activechunk = datachunks.front(); + activechunk->open(false); + activechunk->parse(datasetuid, shape, chunksize, metadata, bitdepth); + imgcounter += activechunk->getImageCount(); + resetAxisInfo(); + parseAxisInfo(); + parseCustomMetadata(); + } + catch(std::exception&) + { + close(); + throw; + } + + // Validate dataset parameters + if(activechunk->getChunkIndex() != 0) + { + close(); + throw std::runtime_error("Unable to load a dataset. First data chunk is missing"); + } + if(datasetuid.empty()) + { + close(); + throw std::runtime_error("Unable to load a dataset. Invalid dataset UID"); + } + if(shape.size() < 3) + { + close(); + throw std::runtime_error("Unable to load a dataset. Invalid dataset shape"); + } + if(bitdepth < 8 || bitdepth > 16) + { + close(); + throw std::runtime_error("Unable to load a dataset. Unsupported pixel format"); + } + + // Parse headers for other data chunks + try + { + for(std::size_t i = 1; i < datachunks.size(); i++) + { + validateDataChunk((std::uint32_t)i, false); + imgcounter += datachunks[i]->getImageCount(); + datachunks[i]->close(); + } + } + catch(std::exception&) + { + close(); + throw; + } +} + +/** + * Close the dataset + * If a dataset hasn't been created / loaded this method will have no effect + * File handles will be released / closed + * In the create mode during closing final section (dataset metadata) is commited to the first data chunk (file) + */ +void G2SBigTiffDataset::close() noexcept +{ + if(writemode && datachunks.size() == 1 && datachunks[0]->isOpen()) + datachunks[0]->appendMetadata(metadata); + writeAxisInfo(); + writeCustomMetadata(); + for(const auto& fx : datachunks) + fx->close(); + imgcounter = 0; + bitdepth = 8; + samples = 1; + metadata.clear(); + shape.clear(); + datachunks.clear(); + activechunk.reset(); + axisinfo.clear(); + custommeta.clear(); +} + +/** + * Set dataset shape / dimension & axis sizes + * First two axis are always width and height + * If the shape info is invalid this method will take no effect + * Shape can only be set in the write mode, before adding any images + * @param dims Axis sizes list + * @throws std::runtime_error + */ +void G2SBigTiffDataset::setShape(const std::vector& dims) +{ + if(dims.size() < 2) + throw std::runtime_error("Unable to set dataset shape. Invalid shape info"); + if(!writemode) + throw std::runtime_error("Unable to set dataset shape in read mode"); + if(datachunks.size() > 1) + throw std::runtime_error("Unable to set dataset shape. Dataset configuration is already set"); + if(imgcounter > 0 && shape.size() >= 2) + { + if(dims.size() != shape.size()) + throw std::runtime_error("Unable to set dataset shape. Invalid axis count"); + if(dims[dims.size() - 2] != shape[shape.size() - 2] || dims[dims.size() - 1] != shape[shape.size() - 1]) + throw std::runtime_error("Unable to set dataset shape. Image dimensions don't match the existing image dimensions"); + return; + } + shape = dims; + + // Resize axis descriptors vector + resetAxisInfo(); + + // Write shape info + if(activechunk) + activechunk->writeShapeInfo(shape, chunksize); +} + +/** + * Set dataset shape / dimension & axis sizes + * Last two axis are always width and height + * If the shape info is invalid this method will take no effect + * Shape can only be set in the write mode, before adding any images + * @param dims Axis sizes list + * @throws std::runtime_error + */ +void G2SBigTiffDataset::setShape(std::initializer_list dims) +{ + if(dims.size() < 2) + throw std::runtime_error("Unable to set dataset shape. Invalid shape info"); + if(!writemode) + throw std::runtime_error("Unable to set dataset shape in read mode"); + if(datachunks.size() > 1) + throw std::runtime_error("Unable to set dataset shape. Dataset configuration is already set"); + if(imgcounter > 0 && shape.size() >= 2) + { + if(dims.size() != shape.size()) + throw std::runtime_error("Unable to set dataset shape. Invalid axis count"); + if(*(dims.end() - 2) != shape[shape.size() - 2] || *(dims.end() - 1) != shape[shape.size() - 1]) + throw std::runtime_error("Unable to set dataset shape. Image dimensions don't match the existing image dimensions"); + return; + } + shape = dims; + + // Resize axis descriptors vector + resetAxisInfo(); + + // Write shape info + if(activechunk) + activechunk->writeShapeInfo(shape, chunksize); +} + +/** + * Get actual dataset shape / dimension & axis sizes (based on the actual image count) + * Last two axis are always width and height + * @return Dataset shape + */ +std::vector G2SBigTiffDataset::getActualShape() const noexcept +{ + std::vector ret = shape; + ret[0] = (int)std::ceil((double)imgcounter / getFixBlockImageCount()); + return ret; +} + +/** + * Get actual axis size (based on the actual image count) + * @param ind Axis index + * @return Axis size + */ +std::uint32_t G2SBigTiffDataset::getAxisSize(std::size_t ind) const noexcept +{ + if(ind >= shape.size()) + return 0; + if(ind == 0) + return (int)std::ceil((double)imgcounter / getFixBlockImageCount()); + return shape[ind]; +} + +/** + * Set pixel format + * If the pixel format is invalid this method will take no effect + * Pixel format can only be set in the write mode, before adding any images + * @param depth Bit depth (bits per sample) + * @parma vsamples Samples per pixel + * @throws std::runtime_error + */ +void G2SBigTiffDataset::setPixelFormat(std::uint8_t depth, std::uint8_t vsamples) +{ + if(!writemode) + throw std::runtime_error("Unable to set pixel format in read mode"); + if(datachunks.size() > 1) + throw std::runtime_error("Unable to set pixel format. Dataset configuration is already set"); + if(imgcounter > 0) + { + if(bitdepth != depth || samples != vsamples) + throw std::runtime_error("Unable to set pixel format. Specified pixel format doesn't match current pixel format"); + return; + } + bitdepth = depth; + samples = vsamples; +} + +/** + * Set dataset metadata + * Metadata will be stored in byte buffer whose size is 1 byte larger than the metadata string length + * @param meta Metadata string + */ +void G2SBigTiffDataset::setMetadata(const std::string& meta) +{ + if(!writemode) + throw std::runtime_error("Unable to set dataset metadata in read mode"); + + metadata.clear(); + if(meta.empty()) + return; + metadata.resize(meta.size() + 1); + std::copy(meta.begin(), meta.end(), metadata.begin()); +} + +/** + * Set dataset UID + * UID must be in a standard UUID format, 16-bytes long hex string with or without the dash delimiters: + * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX + * @param val Dataset UID + * @throws std::runtime_error + */ +void G2SBigTiffDataset::setUID(const std::string& val) +{ + if(!writemode) + throw std::runtime_error("Unable to set dataset UID in read mode"); + if(datachunks.size() > 1) + throw std::runtime_error("Unable to set dataset UID. Dataset configuration is already set"); + + if(val.empty()) + datasetuid = val; + else + { + if(val.size() != 32 && val.size() != 36) + throw std::runtime_error("Unable to set the dataset UID. Invalid UID format"); + auto hasdashes = val.size() == 36; + if(hasdashes && (val[8] != '-' || val[13] != '-' || val[18] != '-' || val[23] != '-')) + throw std::runtime_error("Unable to set the dataset UID. Invalid UID format"); + for(std::size_t i = 0; i < val.size(); i++) + { + if(hasdashes && (i == 8 || i == 13 || i == 18 || i == 23)) + continue; + if(val[i] < 48 || val[i] > 102 || (val[i] > 57 && val[i] < 65) || (val[i] > 70 && val[i] < 97)) + throw std::runtime_error("Unable to set the dataset UID. Invalid UID format"); + } + datasetuid = hasdashes ? val : val.substr(0, 8) + "-" + val.substr(8, 4) + "-" + val.substr(12, 4) + "-" + val.substr(16, 4) + "-" + val.substr(20); + } + + // Update file header + if(activechunk) + activechunk->writeDatasetUid(datasetuid); +} + +void G2SBigTiffDataset::setHandle(int val) +{ + datasetHandle = val; +} + +/** + * Configure axis info + * If axis index is invalid this method will have no effect + * @param dim Axis index + * @param name Axis name + * @param desc Axis description + */ +void G2SBigTiffDataset::configureAxis(int dim, const std::string& name, const std::string& desc) noexcept +{ + if(!writemode) + return; + if(dim < 0 || (std::size_t)dim >= axisinfo.size()) + return; + axisinfo[dim].Name = name; + axisinfo[dim].Description = desc; +} + +/** + * Configure axis coordinate info + * If axis / coordinate index is invalid this method will have no effect + * @param dim Axis index + * @param coord Axis coordinate index + * @param desc Coordinate description + */ +void G2SBigTiffDataset::configureCoordinate(int dim, int coord, const std::string& desc) noexcept +{ + if(!writemode) + return; + if(dim < 0 || coord < 0 || (std::size_t)dim >= axisinfo.size() - 2) + return; + if((std::size_t)coord >= axisinfo[dim].Coordinates.size()) + { + if(dim == 0) + axisinfo[dim].Coordinates.resize((std::size_t)coord + 1); + else + return; + } + axisinfo[dim].Coordinates[coord] = desc; +} + +/** + * Get dataset metadata + * If metadata is specified value will be returned from cache, otherwise it will be read from a file stream + * @return Metadata string + */ +std::string G2SBigTiffDataset::getMetadata() const noexcept +{ + // Check metadata cache + if(metadata.empty()) + return ""; + std::string str(metadata.begin(), metadata.end() - 1); + return str; +} + +/** + * Get image metadata + * If the coordinates are not specified images are read sequentially, metadata for the current image + * will be returned, in which case the current image won't be changed + * If no metadata is defined this method will return an empty string + * If no images are defined this method will return an empty string + * In the sequential mode the image IFD will be loaded if this method is called before getImage() (only for the first image) + * For other images getImage() should always be called prior to calling getImageMetadata() + * @param coord Image coordinates + * @return Image metadata + * @throws std::runtime_error + */ +std::string G2SBigTiffDataset::getImageMetadata(const std::vector& coord) +{ + if(!isOpen()) + throw std::runtime_error("Invalid operation. No open file stream available"); + if(imgcounter == 0) + throw std::runtime_error("Invalid operation. No images available"); + + // Select current image (IFD) + if(!coord.empty()) + selectImage(coord); + else if(activechunk->getCurrentIFD().empty()) + // Load IFD + activechunk->loadIFD(activechunk->getCurrentIFDOffset()); + + return activechunk->getImageMetadata(); +} + +/** + * Add image / write image to the file + * Images are added sequentially + * Image data is stored uncompressed + * Metadata is stored in plain text, after the pixel data + * Image IFD is stored before pixel data + * If the new image doesn't belong to the current chunk, a new file will be created automatically, and the current one will be closed + * @param buff Image buffer + * @param len Image buffer length + * @param meta Image metadata (optional) + * @throws std::runtime_error + */ +void G2SBigTiffDataset::addImage(const unsigned char* buff, std::size_t len, const std::string& meta) +{ + if(!isOpen()) + throw std::runtime_error("Invalid operation. No open file stream available"); + if(!writemode) + throw std::runtime_error("Invalid operation. Unable to add images in read mode"); + if(shape.size() < 2) + throw std::runtime_error("Invalid operation. Dataset shape is not defined"); + if(!bigTiff && len > TIFF_MAX_BUFFER_SIZE) + throw std::runtime_error("Invalid operation. Image data is too long"); + if(!bigTiff && meta.size() > TIFF_MAX_BUFFER_SIZE) + throw std::runtime_error("Invalid operation. Metadata string is too large"); + + // Check active data chunk + if(chunksize > 0 && imgcounter > 0 && imgcounter % getChunkImageCount() == 0) + { + // Close current data chunk + // Only the first data chunk should contain dataset metadata + if(datachunks.size() == 1) + activechunk->appendMetadata(metadata); + activechunk->close(); + + // Create new data chunk + std::filesystem::path fp = std::filesystem::path(dspath) / (dsname + "_" + std::to_string(datachunks.size()) + G2SFILE_EXT); + activechunk = std::make_shared(fp.u8string(), directIo, bigTiff, (std::uint32_t)datachunks.size()); + if(!activechunk) + throw std::runtime_error("Unable to add an image. Data chunk allocation failed"); + activechunk->open(true); + if(activechunk->getHeader().empty()) + throw std::runtime_error("Unable to add an image. File header creation failed"); + activechunk->writeDatasetUid(datasetuid); + activechunk->writeShapeInfo(shape, chunksize); + datachunks.push_back(activechunk); + } + + // Check file size limits + activechunk->addImage(buff, len, getWidth(), getHeight(), bitdepth, meta); + imgcounter++; + + // Flush pending data + if(flushcnt > 0 && activechunk->getImageCount() % flushcnt == 0) + activechunk->flush(); +} + +/** + * Get image data (pixel buffer) + * If the coordinates are not specified images are read sequentially + * This method will change (advance) the current image + * If this method is called after the last available image (in sequential mode), or with invalid coordinates an exception will be thrown + * @param coord Image coordinates + * @return Image data + * @throws std::runtime_error + */ +std::vector G2SBigTiffDataset::getImage(const std::vector& coord) +{ + if(!isOpen()) + throw std::runtime_error("Invalid operation. No open file stream available"); + if(imgcounter == 0) + throw std::runtime_error("Invalid operation. No images available"); + + // Select current image (IFD) + if(!coord.empty()) + selectImage(coord); + else + advanceImage(); + return activechunk->getImage(); +} + +/** + * Set custom metadata entry + * This method will have no effect in READ mode + * @param key Metadata entry key + * @param value Metadata entry value + */ +void G2SBigTiffDataset::setCustomMetadata(const std::string& key, const std::string& value) noexcept +{ + if(!writemode || key.empty()) + return; + custommeta.insert(std::make_pair(key, value)); +} + +/** + * Get custom metadata entry + * If entry is not defined an exception will be thrown + * @param key Metadata entry key + * @return Metadata entry value + * @throws std::runtime_error + */ +std::string G2SBigTiffDataset::getCustomMetadata(const std::string& key) const +{ + auto it = custommeta.find(key); + if(it == custommeta.end()) + throw std::runtime_error("Invalid custom metadata key: " + key); + return it->second; +} + +/** + * Check if custom metadata entry is defined + * @param key Metadata entry key + * @return Metadata entry is defined + */ +bool G2SBigTiffDataset::hasCustomMetadata(const std::string& key) const noexcept +{ + auto it = custommeta.find(key); + return it != custommeta.end(); +} + +/** + * Check if image for the specified coordinates is already set + * @param coordinates Coordinates list + * @param numCoordinates Coordinates count + * @return Does image at the specified coordinates exists + */ +bool G2SBigTiffDataset::isCoordinateSet(int coordinates[], int numCoordinates) const noexcept +{ + std::uint32_t imgind = 0; + for(int i = 0; i < numCoordinates; i++) + { + if(i >= shape.size() - 2) + break; + std::uint32_t sum = 1; + for(int j = i + 1; j < shape.size() - 2; j++) + sum *= shape[j]; + imgind += sum * coordinates[i]; + } + return imgind < imgcounter; +} + +/** + * Change active data chunk + * This method is used only for reading data + * Dataset properties from the new data chunk will be validated + * @param chunkind Data chunk index + * @throws std::runtime_error + */ +void G2SBigTiffDataset::switchDataChunk(std::uint32_t chunkind) +{ + // Validate next data chunk + validateDataChunk(chunkind, true); + + // Change active data chunk + activechunk->close(); + activechunk = datachunks[chunkind]; +} + +/** + * Validate data chunk + * Data chunk (file stream) will be opened in order to parse the header + * File stream won't be closed unless validation fails + * @param chunkind Data chunk index + * @param index Index data chunk IFDs + * @throws std::runtime_error + */ +void G2SBigTiffDataset::validateDataChunk(std::uint32_t chunkind, bool index) +{ + std::string ldataseuid = ""; + std::vector lshape; + std::uint32_t lchunksz = 0; + std::vector lmetadata; + std::uint8_t lbitdepth = 0; + + // Open & parse data chunk (file) + datachunks[chunkind]->open(false); + datachunks[chunkind]->parse(ldataseuid, lshape, lchunksz, lmetadata, lbitdepth, index); + + // Validate dataset properties + if(datasetuid != ldataseuid) + { + datachunks[chunkind]->close(); + throw std::runtime_error("Invalid data chunk. Dataset UID missmatch"); + } + if(shape.size() != lshape.size()) + { + datachunks[chunkind]->close(); + throw std::runtime_error("Invalid data chunk. Dataset shape missmatch"); + } + for(std::size_t i = 0; i < shape.size(); i++) + { + if(shape[i] != lshape[i]) + { + datachunks[chunkind]->close(); + throw std::runtime_error("Invalid data chunk. Axis " + std::to_string(i) + " size missmatch"); + } + } + if(chunksize != lchunksz) + { + datachunks[chunkind]->close(); + throw std::runtime_error("Invalid data chunk. Chunk size missmatch"); + } + if(index && bitdepth != lbitdepth) + { + datachunks[chunkind]->close(); + throw std::runtime_error("Invalid data chunk. Pixel format missmatch"); + } + if(datachunks[chunkind]->getChunkIndex() > 0 && datachunks[chunkind]->getChunkIndex() != chunkind) + { + datachunks[chunkind]->close(); + throw std::runtime_error("Invalid data chunk. Chunk index missmatch"); + } +} + +/** + * Select image + * This method will automatically switch active data chunk + * @param coord Image coordinates + * @throws std::runtime_error + */ +void G2SBigTiffDataset::selectImage(const std::vector& coord) +{ + if(coord.empty()) + return; + if(!activechunk) + throw std::runtime_error("Invalid operation. Invalid data chunk"); + + // Calculate data chunk & image index + std::uint32_t chunkind = 0, imgind = 0; + calcImageIndex(coord, chunkind, imgind); + if(chunkind != activechunk->getChunkIndex()) + switchDataChunk(chunkind); + if(imgind >= activechunk->getIFDOffsets().size()) + throw std::runtime_error("Invalid operation. Invalid image coordinates"); + + // Change current image & load IFD + activechunk->setCurrentImage(imgind); + activechunk->loadIFD(activechunk->getIFDOffsets()[imgind]); +} + +/** + * Advance current image + * This method will automatically switch active data chunk + * @throws std::runtime_error + */ +void G2SBigTiffDataset::advanceImage() +{ + if(!activechunk) + throw std::runtime_error("Invalid operation. Invalid data chunk"); + if(activechunk == datachunks.back() && (activechunk->getCurrentImage() + 1 > activechunk->getImageCount() || activechunk->getNextIFDOffset() == 0)) + throw std::runtime_error("Invalid operation. No more images available"); + + // Check if need to switch the data chunk + if(activechunk->getCurrentImage() + 1 == activechunk->getImageCount()) + switchDataChunk(activechunk->getChunkIndex() + 1); + + // Clear current IFD before advancing + // In a case where getImageMetadata() is called before any getImage() call + // we should skip clearing the current IFD; this works only for the first image + if(activechunk->getCurrentImage() > 0) + activechunk->advanceIFD(); + + // Advance current image + activechunk->setCurrentImage(activechunk->getCurrentImage() + 1); + + // Load IFD (skip if already loaded by the getImageMetadata()) + if(activechunk->getCurrentIFD().empty()) + activechunk->loadNextIFD(); +} + +/** + * Get number of images in a data chunk + * Data chunk represents a dataset subset with one or more slowest changing dimension coordinates + * If chunking is turned OFF this method will return 0 + * @return Image count + */ +std::uint32_t G2SBigTiffDataset::getChunkImageCount() const noexcept +{ + if(chunksize == 0) + return 0; + std::uint32_t ret = 1; + for(std::size_t i = 1; i < shape.size() - 2; i++) + ret *= shape[i]; + return chunksize * ret; +} + +/** + * Get number of images in a fix block (dataset subset without the slowest changing axis) + * @return Image count + */ +std::uint32_t G2SBigTiffDataset::getFixBlockImageCount() const noexcept +{ + std::uint32_t ret = 1; + for(std::size_t i = 1; i < shape.size() - 2; i++) + ret *= shape[i]; + return ret; +} + +/** + * Calculate image index from image coordinates + * Image coordiantes should not contain indices for the last two dimensions (width & height) + * By convention image acquisitions loops through the coordinates in the descending order (higher coordinates are looped first) + * E.g. ZTC order means that all channels are acquired before changing the time point, and all specified time points + * are acquired before moving the Z-stage, in which case dataset with the shape 2-4-3 for coordinates 1-2-1 will return 19 (=1*12 + 2*3 + 1*1) + * First image coordinate can go beyond the specified shape size + * @param coord Image coordinates + * @param chunkind Data chunk index [out] + * @param imgind Image index (in the data chunk) [out] + * @throws std::runtime_error + */ +void G2SBigTiffDataset::calcImageIndex(const std::vector& coord, std::uint32_t& chunkind, std::uint32_t& imgind) const +{ + // Validate coordinates count + if(coord.size() > shape.size() - 2) + throw std::runtime_error("Invalid number of coordinates"); + if(chunkind >= datachunks.size() || (chunkind > 0 && chunksize == 0)) + throw std::runtime_error("Invalid data chunk index"); + if(coord.empty()) + { + chunkind = 0; + imgind = 0; + return; + } + + // Validate ranges for all axis (except the first) + for(std::size_t i = 1; i < coord.size(); i++) + { + if(coord[i] >= shape[i]) + throw std::runtime_error("Invalid coordinate for dimension " + std::to_string(i + 2)); + } + + // Determine chunk index + if(chunksize == 0) + chunkind = 0; + else + chunkind = coord[0] / chunksize; + + // Adjust slowest changing dimension index to set the base for image index calculation + std::vector lcoord = coord; + std::uint32_t baseind = chunkind * chunksize; + lcoord[0] -= baseind; + + // Calculate image index + std::uint32_t ind = 0; + for(int i = 0; i < lcoord.size(); i++) + { + if(lcoord[i] == 0) + continue; + std::uint32_t sum = 1; + for(int j = i + 1; j < shape.size() - 2; j++) + sum *= shape[j]; + ind += sum * lcoord[i]; + } + imgind = ind; +} + +/** + * Reset axis info structure + */ +void G2SBigTiffDataset::resetAxisInfo() noexcept +{ + axisinfo.clear(); + axisinfo.resize(shape.size()); + for(std::size_t i = 0; i < shape.size() - 2; i++) + axisinfo[i].setSize((std::size_t)shape[i]); +} + +/** + * Parse axis info + * Axis info is expected to be stored in a file: 'axisinfo.txt' + * @throws std::runtime_error + */ +void G2SBigTiffDataset::parseAxisInfo() +{ + auto fpath = std::filesystem::u8path(dspath) / G2SAXISINFO_FILE; + if(!std::filesystem::exists(fpath) || axisinfo.empty()) + return; + + // Load file content + std::fstream fs(fpath.u8string(), std::ios::in); + if(!fs.is_open()) + throw std::runtime_error("Unable to load axis info. Opening axis info file failed"); + + int ind = 0; + std::string line = ""; + while(std::getline(fs, line)) + { + if(line.empty()) + continue; + if((std::size_t)ind >= axisinfo.size()) + throw std::runtime_error("Unable to load axis info. Invalid axis info data"); + std::vector tokens = splitLineCSV(line); + if(tokens.size() < 3) + throw std::runtime_error("Unable to load axis info. Corrupted axis info, axis: " + std::to_string(ind)); + std::uint32_t axisdim = 0; + try + { + axisdim = std::stoul(tokens[2]); + } + catch(std::exception& e) + { + throw std::runtime_error("Unable to load axis info. " + std::string(e.what()) + ", axis: " + std::to_string(ind)); + } + if(tokens.size() != (std::size_t)(3 + axisdim)) + throw std::runtime_error("Unable to load axis info. Axis info corrupted, axis: " + std::to_string(ind)); + if(axisinfo[ind].Coordinates.size() != axisdim) + { + if(ind > 0) + throw std::runtime_error("Unable to load axis info. Axis size missmatch, axis: " + std::to_string(ind)); + axisinfo[ind].Coordinates.resize(axisdim); + } + axisinfo[ind].Name = tokens[0]; + axisinfo[ind].Description = tokens[1]; + for(std::uint32_t i = 0; i < axisdim; i++) + axisinfo[ind].Coordinates[i] = tokens[3 + i]; + ind++; + } +} + +/** + * Write axis info + * Axis info will be stored in a separate file: 'axisinfo.txt' + * If no axis info is defined file won't be created + * Axis info will be stored in plain text, CSV-like format + */ +void G2SBigTiffDataset::writeAxisInfo() const noexcept +{ + // Check if axis info is set, and that we are in WRITE mode + if(axisinfo.empty() || !writemode) + return; + + bool hasinfo = false; + for(const auto& dinf : axisinfo) + { + if(!dinf.Name.empty() || !dinf.Description.empty()) + { + hasinfo = true; + break; + } + for(const auto& cinf : dinf.Coordinates) + { + if(!cinf.empty()) + { + hasinfo = true; + break; + } + } + if(hasinfo) + break; + } + + // If axis info is empty, but the file exists -> delete it before exiting + auto fpath = std::filesystem::u8path(dspath) / G2SAXISINFO_FILE; + if(!hasinfo) + { + std::error_code ec; + auto ex = std::filesystem::exists(fpath, ec); + if(!ec && ex) + std::filesystem::remove(fpath, ec); + return; + } + + // Write data to a file + std::fstream fs(fpath.u8string(), std::ios::out | std::ios::trunc); + if(!fs.is_open()) + return; + for(const auto& dinf : axisinfo) + { + fs << "\"" << dinf.Name << "\",\"" << dinf.Description << "\"," << dinf.Coordinates.size(); + for(const auto& cinf : dinf.Coordinates) + fs << ",\"" << cinf << "\""; + fs << std::endl; + } + fs.close(); +} + +/** + * Parse custom metadata + * Custome metadata files are stored in "custommeta" folder + * @throws std::runtime_error + */ +void G2SBigTiffDataset::parseCustomMetadata() +{ + auto fpath = std::filesystem::u8path(dspath) / G2SCUSTOMMETA_FOLDER; + if(!std::filesystem::exists(fpath) || !std::filesystem::is_directory(fpath)) + return; + + for (const auto& entry : std::filesystem::directory_iterator(fpath)) { + // Load file content + std::fstream fs(entry.path().u8string(), std::ios::in); + if (!fs.is_open()) + throw std::runtime_error("Unable to load custom metadata from: " + entry.path().string()); + + std::string key = entry.path().filename().string(); + std::stringstream buffer; + buffer << fs.rdbuf(); + std::string content = buffer.str(); + custommeta.insert(std::make_pair(key, content)); + } +} + +/** + * Write custom metadata + * Custom metadata will be stored in a separate folder: 'custommeta' + * If no metadata is defined the folder won't be created. + * Custome metadata is mutable. + */ +void G2SBigTiffDataset::writeCustomMetadata() const noexcept +{ + auto customMetaFolder = std::filesystem::u8path(dspath) / G2SCUSTOMMETA_FOLDER; + if (!custommeta.empty() && !std::filesystem::exists(customMetaFolder)) + std::filesystem::create_directories(customMetaFolder); + + // Write each metadata key to a separate file + for (const auto& keyval : custommeta) + { + auto fpath = customMetaFolder / keyval.first; + std::fstream fs(fpath.u8string(), std::ios::out | std::ios::trunc); + assert(fs.is_open()); + //if(!fs.is_open()) + // TODO: remove noexcept +// throw std::runtime_error("Failed writing custom metadata to: " + fpath.string()); + fs << keyval.second; + fs.close(); + } +} diff --git a/DeviceAdapters/Go2Scope/G2SBigTiffDataset.h b/DeviceAdapters/Go2Scope/G2SBigTiffDataset.h new file mode 100644 index 000000000..112f79f63 --- /dev/null +++ b/DeviceAdapters/Go2Scope/G2SBigTiffDataset.h @@ -0,0 +1,177 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2STiffFile.h +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: BIGTIFF storage device driver +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Luminous Point LLC, Lumencor Inc. 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +//// +/////////////////////////////////////////////////////////////////////////////// +#pragma once +#include +#include +#include +#include +#include "G2SBigTiffStream.h" + +/** + * Go2Scope BigTiff File Parser + * Go2Scope BigTiff format is the extension of the BIgTIFF format (v6) + * Supports both storing the entire datasets (image acquisitions) in a single file + * or + * Chunking the dataset in multiple files, + * Chunk contains a subset of images that have a common slowest changing dimension + * Support for dataset and per-image metadata + * Support for large files (>2GB) + * Support for Direct and Cached I/O + * Support for sequential & random image access (read / write) + * @author Miloš Jovanović + * @version 1.0 + */ +class G2SBigTiffDataset +{ +public: + //============================================================================================================================ + // Constructors & Destructors + //============================================================================================================================ + G2SBigTiffDataset() noexcept; + G2SBigTiffDataset(const G2SBigTiffDataset& src) noexcept = default; + ~G2SBigTiffDataset() noexcept { close(); } + +public: + //============================================================================================================================ + // Internal data types + //============================================================================================================================ + /** + * Dataset dimension descriptor + * @author Miloš Jovanović + * @version 1.0 + */ + struct G2SDimensionInfo + { + /** + * Default initializer + * @param vname Axis name + * @param ndim Axis size + */ + G2SDimensionInfo(int ndim = 0) noexcept : Name(""), Description(""), Coordinates(ndim) { } + + /** + * Set dimensions size + * @param sz Number of axis coordinates + */ + void setSize(std::size_t sz) noexcept { Coordinates.resize(sz); } + /** + * Get dimension size + * @return Number of axis coordinates + */ + std::size_t getSize() const noexcept { return Coordinates.size(); } + + std::string Name; ///< Axis name + std::string Description; ///< Axis description + std::vector Coordinates; ///< Axis coordinates + }; + +public: + //============================================================================================================================ + // Public interface + //============================================================================================================================ + void create(const std::string& path, bool dio = DEFAULT_DIRECT_IO, bool fbig = DEFAULT_BIGTIFF, std::uint32_t chunksz = 0); + void load(const std::string& path, bool dio = DEFAULT_DIRECT_IO); + void close() noexcept; + void setFlushCycles(std::uint32_t val) noexcept { flushcnt = val; } + std::uint32_t getChunkSize() const noexcept { return chunksize; } + void setShape(const std::vector& dims); + void setShape(std::initializer_list dims); + std::vector getShape() const noexcept { return shape; } + std::vector getActualShape() const noexcept; + std::size_t getDimension() const noexcept { return shape.size(); } + std::uint32_t getAxisSize(std::size_t ind) const noexcept; + std::uint32_t getWidth() const noexcept { return shape.size() < 2 ? 0 : shape[shape.size() - 1]; } + std::uint32_t getHeight() const noexcept { return shape.size() < 2 ? 0 : shape[shape.size() - 2]; } + void setPixelFormat(std::uint8_t depth, std::uint8_t vsamples = 1); + int getBitDepth() const noexcept { return (int)bitdepth; } + int getBpp() const noexcept { return (int)std::ceil(bitdepth / 8.0); } + int getSamples() const noexcept { return (int)samples; } + void setMetadata(const std::string& meta); + std::string getMetadata() const noexcept; + void setUID(const std::string& val); + std::string getUID() const noexcept { return datasetuid; } + void setHandle(int val); + int getHandle() const noexcept { return datasetHandle; } + void configureAxis(int dim, const std::string& name, const std::string& desc) noexcept; + void configureCoordinate(int dim, int coord, const std::string& desc) noexcept; + const G2SDimensionInfo& getAxisInfo(std::uint32_t dim) const { if(dim >= axisinfo.size()) throw std::runtime_error("Unable to obtain axis info. Invalid axis index"); return axisinfo[dim]; } + std::string getImageMetadata(const std::vector& coord = {}); + void addImage(const std::vector& buff, const std::string& meta = "") { addImage(&buff[0], buff.size(), meta); } + void addImage(const unsigned char* buff, std::size_t len, const std::string& meta = ""); + std::vector getImage(const std::vector& coord = {}); + std::uint32_t getDatasetImageCount() const noexcept { std::uint32_t ret = 1; for(std::size_t i = 0; i < shape.size() - 2; i++) ret *= shape[i]; return ret; } + std::uint32_t getImageCount() const noexcept { return imgcounter; } + std::string getPath() const noexcept { return dspath; } + std::string getName() const noexcept { return dsname; } + void setCustomMetadata(const std::string& key, const std::string& value) noexcept; + std::string getCustomMetadata(const std::string& key) const; + bool hasCustomMetadata(const std::string& key) const noexcept; + bool isDirectIO() const noexcept { return directIo; } + bool isBigTIFF() const noexcept { return bigTiff; } + bool isInWriteMode() const noexcept { return writemode; } + bool isInReadMode() const noexcept { return !writemode; } + bool isCoordinateSet(int coordinates[], int numCoordinates) const noexcept; + bool isOpen() const noexcept { return !datachunks.empty() && activechunk; } + +private: + //============================================================================================================================ + // Internal methods + //============================================================================================================================ + void switchDataChunk(std::uint32_t chunkind); + void validateDataChunk(std::uint32_t chunkind, bool index); + void selectImage(const std::vector& coord); + void advanceImage(); + std::uint32_t getChunkImageCount() const noexcept; + std::uint32_t getFixBlockImageCount() const noexcept; + void calcImageIndex(const std::vector& coord, std::uint32_t& chunkind, std::uint32_t& imgind) const; + void resetAxisInfo() noexcept; + void parseAxisInfo(); + void writeAxisInfo() const noexcept; + void parseCustomMetadata(); + void writeCustomMetadata() const noexcept; + +private: + //============================================================================================================================ + // Data members - Dataset properties + //============================================================================================================================ + std::string dspath; ///< Dataset (directory) path + std::string dsname; ///< Dataset name + std::string datasetuid; ///< Dataset UID + std::vector shape; ///< Dataset shape (dimension / axis sizes) + std::vector datachunks; ///< Data chunks / File stream descriptors + G2SFileStreamHandle activechunk; ///< Active data chunk + std::vector metadata; ///< Dataset (summary) metdata (cache) + std::vector axisinfo; ///< Dataset axis descriptors + std::map custommeta; ///< Custom metadata map + std::uint32_t imgcounter; ///< Image counter + std::uint32_t flushcnt; ///< Image flush cycles + std::uint32_t chunksize; ///< Chunk size + std::uint8_t bitdepth; ///< Bit depth + std::uint8_t samples; ///< Samples per pixel + bool directIo; ///< Use direct I/O for file operations + bool bigTiff; ///< Use big TIFF format + bool writemode; ///< Is dataset opened for writing + int datasetHandle; ///< dataset handle +}; + diff --git a/DeviceAdapters/Go2Scope/G2SBigTiffStorage.cpp b/DeviceAdapters/Go2Scope/G2SBigTiffStorage.cpp new file mode 100644 index 000000000..cf6726afc --- /dev/null +++ b/DeviceAdapters/Go2Scope/G2SBigTiffStorage.cpp @@ -0,0 +1,1329 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SBigTiffStorage.cpp +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: BIGTIFF storage device driver +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Luminous Point LLC, Lumencor Inc., 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +/////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include +#include +#include "G2SBigTiffStorage.h" + +#define MAX_FILE_SEARCH_INDEX 128 +#define PROP_DIRECTIO "DirectIO" +#define PROP_FLUSHCYCLE "FlushCycle" +#define PROP_CHUNKSIZE "ChunkSize" + +/** + * Default class constructor + */ +G2SBigTiffStorage::G2SBigTiffStorage() +{ + InitializeDefaultErrorMessages(); + + // set device specific error messages + SetErrorText(ERR_INTERNAL, "Internal driver error, see log file for details"); + SetErrorText(ERR_TIFF, "Generic TIFF error. See log for more info."); + SetErrorText(ERR_TIFF_STREAM_UNAVAILABLE, "BigTIFF storage error. File stream is not available."); + SetErrorText(ERR_TIFF_INVALID_PATH, "Invalid path or name."); + SetErrorText(ERR_TIFF_INVALID_DIMENSIONS, "Invalid number of dimensions. Minimum is 3."); + SetErrorText(ERR_TIFF_INVALID_PIXEL_TYPE, "Invalid or unsupported pixel type."); + SetErrorText(ERR_TIFF_OPEN_FAILED, "Failed opening TIF file."); + SetErrorText(ERR_TIFF_HANDLE_INVALID, "Dataset handle is not valid."); + SetErrorText(ERR_TIFF_STRING_TOO_LONG, "Requested string is too long for the provided buffer."); + SetErrorText(ERR_TIFF_INVALID_COORDINATE, "Dataset coordinates are valid."); + SetErrorText(ERR_TIFF_DATASET_CLOSED, "Operation unavailable - dataset is closed."); + SetErrorText(ERR_TIFF_DATASET_READONLY, "Operation unavailable - dataset is read-only."); + SetErrorText(ERR_TIFF_DELETE_FAILED, "File / folder delete failed."); + SetErrorText(ERR_TIFF_ALLOCATION_FAILED, "Dataset memory allocation failed."); + SetErrorText(ERR_TIFF_CORRUPTED_METADATA, "Metadata corrupted / invalid"); + SetErrorText(ERR_TIFF_UPDATE_FAIL, "Dataset structure update failed"); + SetErrorText(ERR_TIFF_FILESYSTEM_ERROR, "Filesystem error"); + SetErrorText(ERR_TIFF_INVALID_META_KEY, "Invalid metadata key"); + + // create pre-initialization properties + // ------------------------------------ + // + + // Name + CreateProperty(MM::g_Keyword_Name, g_BigTiffStorage, MM::String, true); + // + // Description + std::ostringstream os; + os << "BigTIFF Storage " << G2STIFF_VERSION; + CreateProperty(MM::g_Keyword_Description, os.str().c_str(), MM::String, true); +} + +/** + * Get device name + * @param Name String buffer [out] + */ +void G2SBigTiffStorage::GetName(char* Name) const +{ + CDeviceUtils::CopyLimitedString(Name, g_BigTiffStorage); +} + +/** + * Device driver initialization routine + */ +int G2SBigTiffStorage::Initialize() +{ + if(initialized) + return DEVICE_OK; + + // Add DirectIO property + int nRet = CreateIntegerProperty(PROP_DIRECTIO, 0, false); + assert(nRet == DEVICE_OK); + AddAllowedValue(PROP_DIRECTIO,"0"); + AddAllowedValue(PROP_DIRECTIO,"1"); + + // Add flush counter property + nRet = CreateIntegerProperty(PROP_FLUSHCYCLE, 0, false); + assert(nRet == DEVICE_OK); + + // Add chunk size property + nRet = CreateIntegerProperty(PROP_CHUNKSIZE, 0, false); + assert(nRet == DEVICE_OK); + + UpdateStatus(); + + initialized = true; + return DEVICE_OK; +} + +/** + * Device driver shutdown routine + * During device shutdown cache will be emptied, + * and all open file handles will be closed + */ +int G2SBigTiffStorage::Shutdown() noexcept +{ + initialized = false; + for(auto it = cache.begin(); it != cache.end(); it++) + { + if(it->second.isOpen()) + { + auto fs = reinterpret_cast(it->second.FileHandle); + fs->close(); + it->second.close(); + delete fs; + } + } + cache.clear(); + return DEVICE_OK; +} + +/** + * Create storage entry + * Dataset storage descriptor will open a file handle, to close a file handle call Close() + * Dataset storage descriptor will reside in device driver cache + * If the file already exists this method will fail with 'DEVICE_DUPLICATE_PROPERTY' status code + * @param handle + * @param path Absolute file path (TIFF file) + * @param name Dataset name + * @param numberOfDimensions Number of dimensions + * @param shape Axis sizes + * @param pixType Pixel format + * @param meta Metadata + * @param metaLength length of the metadata string + * @return Status code + */ +int G2SBigTiffStorage::Create(int handle, const char* path, const char* name, int numberOfDimensions, const int shape[], MM::StorageDataType pixType, const char* meta, int metaLength) noexcept +{ + if(path == nullptr) + return ERR_TIFF_INVALID_PATH; + if(numberOfDimensions < 3) + return ERR_TIFF_INVALID_DIMENSIONS; + if(!(pixType == MM::StorageDataType::StorageDataType_GRAY16 || pixType == MM::StorageDataType::StorageDataType_GRAY8 || pixType == MM::StorageDataType::StorageDataType_RGB32)) + return ERR_TIFF_INVALID_PIXEL_TYPE; + if (shape == nullptr) + return DEVICE_INVALID_INPUT_PARAM; + + try + { + // Check cache size limits + if(cache.size() >= MAX_CACHE_SIZE) + { + cacheReduce(); + if(CACHE_HARD_LIMIT && cache.size() >= MAX_CACHE_SIZE) + return ERR_TIFF_CACHE_OVERFLOW; + } + + // Compose dataset path + std::string dsname(name); + if(dsname.find(".tiff") == dsname.size() - 5) + dsname = dsname.substr(0, dsname.size() - 5); + else if(dsname.find(".tif") == dsname.size() - 4 || dsname.find(".tf8") == dsname.size() - 4) + dsname = dsname.substr(0, dsname.size() - 4); + if(dsname.find(".g2s") != dsname.size() - 4) + dsname += ".g2s"; + std::filesystem::path saveRoot = std::filesystem::u8path(path); + std::filesystem::path dsName = saveRoot / dsname; + + // Create a file on disk and store the file handle + auto fhandle = new G2SBigTiffDataset(); + if(fhandle == nullptr) + { + LogMessage("Error obtaining file handle for: " + dsName.u8string()); + return ERR_TIFF_STREAM_UNAVAILABLE; + } + + try + { + fhandle->create(dsName.u8string(), getDirectIO(), true, (std::uint32_t)getChunkSize()); + if(!fhandle->isOpen()) + { + LogMessage("Failed to open file: " + dsName.u8string()); + return ERR_TIFF_OPEN_FAILED; + } + fhandle->setFlushCycles((std::uint32_t)getFlushCycle()); + } + catch(std::exception& err) + { + delete fhandle; + LogMessage(std::string(err.what()) + " for " + dsName.u8string()); + return ERR_TIFF_OPEN_FAILED; + } + + // Create dataset storage descriptor + std::string guid = boost::lexical_cast(boost::uuids::random_generator()()); // Entry UUID + if (guid.size() > MM::MaxStrLength) + { +#ifdef _NDEBUG + guid = guid.substr(0, MM::MaxStrLength); +#else + assert("Dataset handle size is too long"); +#endif + } + + + G2SStorageEntry sdesc(fhandle->getPath()); + sdesc.FileHandle = fhandle; + try + { + + // Set dataset UUID / shape / metadata + std::vector vshape; + vshape.assign(shape, shape + numberOfDimensions); + fhandle->setUID(guid); + fhandle->setHandle(handle); + fhandle->setShape(vshape); + std::string metadataStr(meta, metaLength); + fhandle->setMetadata(metadataStr); + + // Set pixel format + if(pixType == MM::StorageDataType::StorageDataType_GRAY8) + fhandle->setPixelFormat(8, 1); + else if(pixType == MM::StorageDataType::StorageDataType_GRAY16) + fhandle->setPixelFormat(16, 1); + else if(pixType == MM::StorageDataType::StorageDataType_RGB32) + fhandle->setPixelFormat(8, 4); + } + catch(std::exception& err) + { + delete fhandle; + LogMessage(std::string(err.what()) + " for " + dsName.u8string()); + return ERR_TIFF_UPDATE_FAIL; + } + + // Append dataset storage descriptor to cache + auto it = cache.insert({ handle, sdesc }); + if(it.first == cache.end()) + { + delete fhandle; + LogMessage("Adding BigTIFF dataset to cache failed. Path: " + dsName.u8string() + ", handle: " + std::to_string(handle)); + return ERR_TIFF_CACHE_INSERT; + } + if(!it.second) + { + // Dataset already exists + delete fhandle; + LogMessage("Adding BigTIFF dataset to cache failed. Path: " + dsName.u8string() + ", handle: " + std::to_string(handle)); + return ERR_TIFF_CACHE_INSERT; + } + + return DEVICE_OK; + } + catch(std::exception& e) + { + LogMessage("Create error: " +std::string(e.what())); + return ERR_INTERNAL; + } +} + +/** + * Load dataset from disk + * Dataset storage descriptor will be read from file + * Dataset storage descriptor will open a file handle, to close a file handle call Close() + * Dataset storage descriptor will reside in device driver cache + * @param path Absolute file path (TIFF file) / Absolute dataset folder path + * @param handle + * @param name Dataset name + * @return Status code + */ +int G2SBigTiffStorage::Load(int handle, const char* path) noexcept +{ + if(path == nullptr) + return DEVICE_INVALID_INPUT_PARAM; + + try + { + // Check if the file exists + std::error_code ec; + std::filesystem::path actpath = std::filesystem::u8path(path); + auto ex = std::filesystem::exists(actpath, ec); + if(ec) + { + LogMessage("Load filesystem error " + std::to_string(ec.value()) + ". " + ec.message()); + return ERR_TIFF_FILESYSTEM_ERROR; + } + auto rf = ex ? std::filesystem::is_regular_file(actpath, ec) : false; + if(ec) + { + LogMessage("Load filesystem error " + std::to_string(ec.value()) + ". " + ec.message()); + return ERR_TIFF_FILESYSTEM_ERROR; + } + + if(!ex) + { + // Try finding the folder by adding the index (number suffix) + bool fnd = false; + auto dir = actpath.parent_path(); + auto fname = actpath.stem().u8string(); + auto ext = actpath.extension().u8string(); + for(int i = 0; i < MAX_FILE_SEARCH_INDEX; i++) + { + actpath = dir / (fname + "_" + std::to_string(i) + ext); + if(std::filesystem::exists(actpath)) + { + fnd = true; + break; + } + } + if(!fnd) + return ERR_TIFF_INVALID_PATH; + } + else if(rf) + { + // File path of the first data chunk is specified -> use parent folder path + actpath = actpath.parent_path(); + } + + // Check if file is already in cache + auto cit = cache.begin(); + while(cit != cache.end()) + { + if(std::filesystem::u8path(cit->second.Path) == actpath) + break; + cit++; + } + + G2SBigTiffDataset* fhandle = nullptr; + if(cit == cache.end()) + // Open a file on disk and store the file handle + fhandle = new G2SBigTiffDataset(); + else if(cit->second.FileHandle == nullptr) + { + // Open a file on disk and update the cache file handle + fhandle = new G2SBigTiffDataset(); + cit->second.FileHandle = fhandle; + } + else + // Use existing object descriptor + fhandle = (G2SBigTiffDataset*)cit->second.FileHandle; + if(fhandle == nullptr) + { + LogMessage("Loading BigTIFF dataset failed (" + actpath.u8string() + "). Dataset allocation failed"); + return ERR_TIFF_ALLOCATION_FAILED; + } + + try + { + if(!fhandle->isOpen()) + { + fhandle->load(std::filesystem::absolute(actpath).u8string(), getDirectIO()); + if(!fhandle->isOpen()) + return ERR_TIFF_OPEN_FAILED; + } + fhandle->setFlushCycles((std::uint32_t)getFlushCycle()); + } + catch(std::exception& e) + { + delete fhandle; + LogMessage("Loading BigTIFF dataset failed (" + actpath.u8string() + "). " + std::string(e.what())); + return ERR_TIFF_OPEN_FAILED; + } + + // Obtain / generate dataset UID + std::string guid = fhandle->getUID().empty() ? boost::lexical_cast(boost::uuids::random_generator()()) : fhandle->getUID(); + if(guid.size() > MM::MaxStrLength) + { + delete fhandle; + return ERR_TIFF_STRING_TOO_LONG; + } + + // Append dataset storage descriptor to cache + if(cit == cache.end()) + { + // Create dataset storage descriptor + G2SStorageEntry sdesc(std::filesystem::absolute(actpath).u8string()); + sdesc.FileHandle = fhandle; + + auto it = cache.insert({ handle, sdesc }); + if(it.first == cache.end()) + { + delete fhandle; + LogMessage("Loading BigTIFF dataset failed (" + actpath.u8string() + "). Dataset cache is full"); + return DEVICE_OUT_OF_MEMORY; + } + } + return DEVICE_OK; + } + catch(std::exception& e) + { + LogMessage("Load error: " +std::string(e.what())); + return ERR_INTERNAL; + } +} + +/** + * Get actual dataset shape + * Shape contains image width and height as first two dimensions + * @param handle Entry GUID + * @param shape Dataset shape [out] + * @return Status code + */ +int G2SBigTiffStorage::GetShape(int handle, int shape[]) noexcept +{ + if(shape == nullptr) + return DEVICE_INVALID_INPUT_PARAM; + + // Obtain dataset descriptor from cache + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + + auto fs = reinterpret_cast(it->second.FileHandle); + for(std::size_t i = 0; i < fs->getDimension(); i++) + shape[i] = fs->getActualShape()[i]; + return DEVICE_OK; +} + +/** + * Get dataset pixel format + * @param handle Entry GUID + * @param pixelDataType Pixel format [out] + * @return Status code + */ +int G2SBigTiffStorage::GetDataType(int handle, MM::StorageDataType& pixelDataType) noexcept +{ + // Obtain dataset descriptor from cache + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + + // Get pixel format + if(!it->second.isOpen()) + pixelDataType = MM::StorageDataType_UNKNOWN; + else + { + auto fs = reinterpret_cast(it->second.FileHandle); + switch(fs->getBpp()) + { + case 1: + pixelDataType = MM::StorageDataType_GRAY8; + break; + case 2: + pixelDataType = MM::StorageDataType_GRAY16; + break; + case 3: + case 4: + pixelDataType = MM::StorageDataType_RGB32; + break; + default: + pixelDataType = MM::StorageDataType_UNKNOWN; + break; + } + } + return DEVICE_OK; +} + +/** + * Close the dataset + * File handle will be closed + * Metadata will be discarded + * Storage entry descriptor will remain in cache + * @param handle Entry GUID + * @return Status code + */ +int G2SBigTiffStorage::Close(int handle) noexcept +{ + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(it->second.isOpen()) + { + auto fs = reinterpret_cast(it->second.FileHandle); + fs->close(); + it->second.close(); + delete fs; + } + return DEVICE_OK; +} + +/** + * Delete existing dataset (file on disk) + * If the file doesn't exist this method will return 'DEVICE_NO_PROPERTY_DATA' status code + * Dataset storage descriptor will be removed from cache + * @param handle Entry GUID + * @return Status code + */ +int G2SBigTiffStorage::Delete(int handle) noexcept +{ + + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + + try + { + // Check if the file exists + std::error_code ec; + auto fp = std::filesystem::u8path(it->second.Path); + auto ex = std::filesystem::exists(fp, ec); + if(ec) + { + LogMessage("Delete filesystem error " + std::to_string(ec.value()) + ". " + ec.message()); + return ERR_TIFF_FILESYSTEM_ERROR; + } + if(!ex) + return ERR_TIFF_INVALID_PATH; + + // Close the file handle + if(it->second.isOpen()) + { + auto fs = reinterpret_cast(it->second.FileHandle); + fs->close(); + it->second.close(); + delete fs; + } + + // Delete the file + bool succ = std::filesystem::remove(fp, ec); + if(ec) + { + LogMessage("Delete filesystem error " + std::to_string(ec.value()) + ". " + ec.message()); + return ERR_TIFF_FILESYSTEM_ERROR; + } + if(!succ) + return ERR_TIFF_DELETE_FAILED; + + // Discard the cache entry + cache.erase(it); + return DEVICE_OK; + } + catch(std::exception& e) + { + LogMessage("Delete error: " +std::string(e.what())); + return ERR_INTERNAL; + } +} + +/** + * List datasets in the specified folder / path + * If the list of found datasets is longer than 'maxItems' only first [maxItems] will be + * returned and 'DEVICE_SEQUENCE_TOO_LARGE' status code will be returned + * If the dataset path is longer than 'maxItemLength' dataset path will be truncated + * If the specified path doesn't exist, or it's not a valid folder path 'DEVICE_INVALID_INPUT_PARAM' status code will be returned + * @param path Folder path + * @param listOfDatasets Dataset path list [out] + * @param maxItems Max dataset count + * @param maxItemLength Max dataset path length + * @return Status code + */ +int G2SBigTiffStorage::List(const char* path, char** listOfDatasets, int maxItems, int maxItemLength) noexcept +{ + if(path == nullptr || listOfDatasets == nullptr || maxItems <= 0 || maxItemLength <= 0) + return DEVICE_INVALID_INPUT_PARAM; + + try + { + std::error_code ec; + auto dp = std::filesystem::u8path(path); + auto exs = std::filesystem::exists(dp, ec); + if(ec) + { + LogMessage("List filesystem error " + std::to_string(ec.value()) + ". " + ec.message()); + return ERR_TIFF_FILESYSTEM_ERROR; + } + auto isdir = std::filesystem::is_directory(dp, ec); + if(ec) + { + LogMessage("List filesystem error " + std::to_string(ec.value()) + ". " + ec.message()); + return ERR_TIFF_FILESYSTEM_ERROR; + } + if(!exs || !isdir) + return ERR_TIFF_INVALID_PATH; + auto allfnd = scanDir(path, listOfDatasets, maxItems, maxItemLength, 0); + + // TODO: review memory allocation and whether the limit can be removed + return allfnd ? DEVICE_OK : ERR_TIFF_STRING_TOO_LONG; + } + catch(std::exception& e) + { + LogMessage("List error: " +std::string(e.what())); + return ERR_INTERNAL; + } +} + +/** + * Add image / write image to file + * Image metadata will be stored in cache + * @param handle Entry GUID + * @param pixels Pixel data buffer + * @param sizeInBytes pixel array size + * @param coordinates Image coordinates + * @param numCoordinates Coordinate count + * @param imageMeta Image metadata + * @param metaLength metadata length + * @return Status code + */ +int G2SBigTiffStorage::AddImage(int handle, int sizeInBytes, unsigned char* pixels, int coordinates[], int numCoordinates, const char* imageMeta, int metaLength) noexcept +{ + if(pixels == nullptr || sizeInBytes <= 0 || numCoordinates <= 0) + return DEVICE_INVALID_INPUT_PARAM; + + // Obtain dataset descriptor from cache + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + + // Validate image dimensions / coordinates + auto fs = reinterpret_cast(it->second.FileHandle); + if(fs->isInReadMode()) + return ERR_TIFF_DATASET_READONLY; + if(!validateCoordinates(fs, coordinates, numCoordinates, true)) + return ERR_TIFF_INVALID_COORDINATE; + if(fs->isCoordinateSet(coordinates, numCoordinates)) + return ERR_TIFF_INVALID_COORDINATE; + + try + { + // Add image + std::string imageMetaStr(imageMeta, metaLength); + fs->addImage(pixels, sizeInBytes, imageMetaStr); + return DEVICE_OK; + } + catch(std::exception& e) + { + LogMessage("AddImage error: " +std::string(e.what())); + return ERR_TIFF_UPDATE_FAIL; + } +} + +/** + * Append image / write image to file + * Image metadata will be stored in cache + * @param handle Entry GUID + * @param pixels Pixel data buffer + * @param sizeInBytes pixel array size + * @param imageMeta Image metadata + * @param metaLength length of the metadata + * @return Status code + */ +int G2SBigTiffStorage::AppendImage(int handle, int sizeInBytes, unsigned char* pixels, const char* imageMeta, int metaLength) noexcept +{ + if(pixels == nullptr || sizeInBytes <= 0) + return DEVICE_INVALID_INPUT_PARAM; + + // Obtain dataset descriptor from cache + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + + try + { + // Append image + auto fs = reinterpret_cast(it->second.FileHandle); + if(fs->isInReadMode()) + return ERR_TIFF_DATASET_READONLY; + std::string imageMetaStr(imageMeta, metaLength); + fs->addImage(pixels, sizeInBytes, imageMetaStr); + return DEVICE_OK; + } + catch(std::exception& e) + { + LogMessage("AppendImage error: " +std::string(e.what())); + return ERR_TIFF_UPDATE_FAIL; + } +} + +/** + * Get dataset summary metadata + * If the netadata size is longer than the provided buffer, only the first [bufSize] bytes will be + * copied, and the status code 'DEVICE_SEQUENCE_TOO_LARGE' will be returned + * @param handle Entry GUID + * @param meta Metadata buffer [out] + * @return Status code + */ +int G2SBigTiffStorage::GetSummaryMeta(int handle, char** meta) noexcept +{ + // Obtain dataset descriptor from cache + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + + try + { + // Copy metadata string + auto fs = reinterpret_cast(it->second.FileHandle); + *meta = new char[fs->getMetadata().size() + 1]; + strncpy(*meta, fs->getMetadata().c_str(), fs->getMetadata().size() + 1); + return DEVICE_OK; + } + catch(std::exception& e) + { + LogMessage("GetSummaryMeta error: " + std::string(e.what())); + return ERR_TIFF_CORRUPTED_METADATA; + } +} + +/** + * Get dataset image metadata + * If the netadata size is longer than the provided buffer, only the first [bufSize] bytes will be + * copied, and the status code 'DEVICE_SEQUENCE_TOO_LARGE' will be returned + * @param handle Entry GUID + * @param coordinates Image coordinates + * @param numCoordinates Coordinate count + * @param meta Metadata buffer [out] + * @param bufSize Buffer size + * @return Status code + */ +int G2SBigTiffStorage::GetImageMeta(int handle, int coordinates[], int numCoordinates, char** meta) noexcept +{ + if(coordinates == nullptr || numCoordinates == 0) + return DEVICE_INVALID_INPUT_PARAM; + + // Obtain dataset descriptor from cache + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + + auto fs = reinterpret_cast(it->second.FileHandle); + if(!validateCoordinates(fs, coordinates, numCoordinates)) + return ERR_TIFF_INVALID_COORDINATE; + + // Obtain metadata from the file stream + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + + // Copy coordinates without including the width and height + std::vector coords(fs->getDimension() - 2); + for(int i = 0; i < coords.size(); i++) + { + if(i >= numCoordinates) + break; + coords[i] = coordinates[i]; + } + + try + { + auto fmeta = fs->getImageMetadata(coords); + *meta = new char[fmeta.size() + 1]; + strncpy(*meta, fmeta.c_str(), fmeta.size() + 1); + return DEVICE_OK; + } + catch(std::exception& e) + { + LogMessage("GetImageMeta error: " + std::string(e.what())); + return ERR_TIFF_CORRUPTED_METADATA; + } +} + +/** + * Get image / pixel data + * Image buffer will be created inside this method, so + * object (buffer) destruction becomes callers responsibility + * @param handle Entry GUID + * @param coordinates Image coordinates + * @param numCoordinates Coordinate count + * @return Pixel buffer pointer + */ +const unsigned char* G2SBigTiffStorage::GetImage(int handle, int coordinates[], int numCoordinates) noexcept +{ + if(numCoordinates <= 0) + return nullptr; + try + { + // Obtain dataset descriptor from cache + auto it = cache.find(handle); + if(it == cache.end()) + return nullptr; + + auto fs = reinterpret_cast(it->second.FileHandle); + if(!validateCoordinates(fs, coordinates, numCoordinates)) + return nullptr; + + if(!it->second.isOpen()) + return nullptr; + + // Copy coordinates without including the width and height + std::vector coords(fs->getDimension() - 2); + for(int i = 0; i < coords.size(); i++) + { + if (i >= numCoordinates) + break; + coords[i] = coordinates[i]; + } + + it->second.ImageData = fs->getImage(coords); + return &it->second.ImageData[0]; + } + catch(std::runtime_error& e) + { + LogMessage("GetImage error: " +std::string(e.what())); + return nullptr; + } +} + +/** + * Configure metadata for a given dimension + * @param handle Entry GUID + * @param dimension Dimension index + * @param name Name of the dimension + * @param meaning Z,T,C, etc. (physical meaning) + * @return Status code + */ +int G2SBigTiffStorage::ConfigureDimension(int handle, int dimension, const char* name, const char* meaning) noexcept +{ + if(dimension < 0 || name == nullptr || meaning == nullptr) + return DEVICE_INVALID_INPUT_PARAM; + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + if(fs->isInReadMode()) + return ERR_TIFF_DATASET_READONLY; + + if((std::size_t)dimension >= fs->getDimension()) + return ERR_TIFF_INVALID_DIMENSIONS; + fs->configureAxis(dimension, std::string(name), std::string(meaning)); + return DEVICE_OK; +} + +/** + * Configure a particular coordinate name. e.g. channel name / position name ... + * @param handle Entry GUID + * @param dimension Dimension index + * @param coordinate Coordinate index + * @param name Coordinate name + * @return Status code + */ +int G2SBigTiffStorage::ConfigureCoordinate(int handle, int dimension, int coordinate, const char* name) noexcept +{ + if(dimension < 0 || coordinate < 0 || name == nullptr) + return DEVICE_INVALID_INPUT_PARAM; + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + if(fs->isInReadMode()) + return ERR_TIFF_DATASET_READONLY; + + if(dimension < 0 || (std::size_t)dimension >= fs->getDimension()) + return ERR_TIFF_INVALID_DIMENSIONS; + if(coordinate < 0 || ((std::size_t)coordinate >= fs->getShape()[dimension] && dimension > 0)) + return ERR_TIFF_INVALID_COORDINATE; + fs->configureCoordinate(dimension, coordinate, std::string(name)); + return DEVICE_OK; +} + +/** + * Get number of dimensions + * @param handle Entry GUID + * @param numDimensions Number of dimensions [out] + * @return Status code + */ +int G2SBigTiffStorage::GetNumberOfDimensions(int handle, int& numDimensions) noexcept +{ + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + + auto fs = reinterpret_cast(it->second.FileHandle); + numDimensions = (int)fs->getDimension(); + return DEVICE_OK; +} + +/** + * Get number of dimensions + * @param handle Entry GUID + * @return Status code + */ +int G2SBigTiffStorage::GetDimension(int handle, int dimension, char* name, int nameLength, char* meaning, int meaningLength) noexcept +{ + if(dimension < 0 || meaningLength <= 0 || name == nullptr || meaning == nullptr) + return DEVICE_INVALID_INPUT_PARAM; + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + + if(dimension < 0 || (std::size_t)dimension >= fs->getDimension()) + return ERR_TIFF_INVALID_DIMENSIONS; + + try + { + const auto& axinf = fs->getAxisInfo((std::uint32_t)dimension); + if(axinf.Name.size() > (std::size_t)nameLength) + return ERR_TIFF_STRING_TOO_LONG; + if(axinf.Description.size() > (std::size_t)meaningLength) + return ERR_TIFF_STRING_TOO_LONG; + strncpy(name, axinf.Name.c_str(), nameLength); + strncpy(meaning, axinf.Description.c_str(), meaningLength); + return DEVICE_OK; + } + catch(std::exception& e) + { + LogMessage("GetDimension error: " + std::string(e.what())); + return ERR_TIFF_INVALID_COORDINATE; + } +} + +/** + * Get number of dimensions + * @param handle Entry GUID + * @return Status code + */ +int G2SBigTiffStorage::GetCoordinate(int handle, int dimension, int coordinate, char* name, int nameLength) noexcept +{ + if(dimension < 0 || coordinate < 0 || nameLength <= 0 || name == nullptr) + return DEVICE_INVALID_INPUT_PARAM; + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + + if(dimension < 0 || (std::size_t)dimension >= fs->getDimension()) + return ERR_TIFF_INVALID_DIMENSIONS; + if(coordinate < 0) + return ERR_TIFF_INVALID_COORDINATE; + if((std::size_t)coordinate >= fs->getShape()[dimension]) + { + if(dimension > 0) + return ERR_TIFF_INVALID_COORDINATE; + if((std::size_t)coordinate >= fs->getActualShape()[dimension]) + return ERR_TIFF_INVALID_COORDINATE; + } + + try + { + const auto& axinf = fs->getAxisInfo((std::uint32_t)dimension); + if((std::size_t)coordinate < axinf.Coordinates.size()) + { + if(axinf.Coordinates[coordinate].size() > (std::size_t)nameLength) + return ERR_TIFF_STRING_TOO_LONG; + strncpy(name, axinf.Coordinates[coordinate].c_str(), nameLength); + } + else if(dimension > 0) + return ERR_TIFF_INVALID_COORDINATE; + return DEVICE_OK; + } + catch(std::exception& e) + { + LogMessage("GetCoordinate error: " + std::string(e.what())); + return ERR_TIFF_INVALID_COORDINATE; + } +} + +/** + * Get number of available images + * @param handle Entry GUID + * @param imgcount Image count [out] + * @return Status code + */ +int G2SBigTiffStorage::GetImageCount(int handle, int& imgcount) noexcept +{ + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + imgcount = (int)fs->getImageCount(); + return DEVICE_OK; +} + +/** + * Set custom metadata (key-value pair) + * @param handle Entry GUID + * @param key Metadata entry key + * @param content Metadata entry value / content + * @return Status code + */ +int G2SBigTiffStorage::SetCustomMetadata(int handle, const char* key, const char* content, int contentLength) noexcept +{ + if (key == nullptr || content == nullptr) + return DEVICE_INVALID_INPUT_PARAM; + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + if(fs->isInReadMode()) + return ERR_TIFF_DATASET_READONLY; + + std::string contentStr(content, contentLength); + fs->setCustomMetadata(key, content); + return DEVICE_OK; +} + +/** + * Get custom metadata (key-value pair) + * @param handle Entry GUID + * @param key Metadata entry key + * @param content Metadata entry value / content [out] + * @return Status code + */ +int G2SBigTiffStorage::GetCustomMetadata(int handle, const char* key, char** content) noexcept +{ + if(key == nullptr) + return DEVICE_INVALID_INPUT_PARAM; + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + if(!fs->hasCustomMetadata(key)) + return ERR_TIFF_INVALID_META_KEY; + + try + { + auto mval = fs->getCustomMetadata(key); + *content = new char[mval.size() + 1]; + strncpy(*content, mval.c_str(), mval.size() + 1); + return DEVICE_OK; + } + catch(std::exception& e) + { + LogMessage("GetCustomMetadata error: " + std::string(e.what())); + return ERR_TIFF_INVALID_META_KEY; + } +} + +/** + * Check if dataset is open + * If the dataset doesn't exist, or the GUID is invalid this method will return false + * @param handle Entry GUID + * @return true if dataset is open + */ +bool G2SBigTiffStorage::IsOpen(int handle) noexcept +{ + auto it = cache.find(handle); + if(it == cache.end()) + return false; + return it->second.isOpen(); +} + +/** + * Check if dataset is read-only + * If the dataset doesn't exist, or the GUID is invalid this method will return true + * @param handle Entry GUID + * @return true if images can't be added to the dataset + */ +bool G2SBigTiffStorage::IsReadOnly(int handle) noexcept +{ + auto it = cache.find(handle); + if(it == cache.end()) + return true; + if(!it->second.isOpen()) + return true; + auto fs = reinterpret_cast(it->second.FileHandle); + return fs->isInReadMode(); +} + +/** + * Get dataset path + * @param handle Entry GUID + * @param path Dataset path [out] + * @param maxPathLength Max path length + * @return Status code + */ +int G2SBigTiffStorage::GetPath(int handle, char* path, int maxPathLength) noexcept +{ + if(maxPathLength <= 0 || path == nullptr) + return DEVICE_INVALID_INPUT_PARAM; + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(it->second.Path.size() > (std::size_t)maxPathLength) + return ERR_TIFF_STRING_TOO_LONG; + strncpy(path, it->second.Path.c_str(), it->second.Path.size()); + return DEVICE_OK; +} + +/** + * Check if there is a valid dataset on the selected path + * @param path Dataset path + * @return Path is a valid dataset + */ +bool G2SBigTiffStorage::CanLoad(const char* path) noexcept +{ + if(path == nullptr) + return false; + try + { + std::error_code ec; + std::filesystem::path xpath = std::filesystem::u8path(path); + if(!std::filesystem::exists(xpath, ec)) + return false; + if(ec) + return false; + + bool isdir = std::filesystem::is_directory(xpath, ec); + if(ec) + return false; + if(isdir) + { + // If directory is selected check if it's not empty and if the name ends with .g2s + auto dname = xpath.filename().u8string(); + if(dname.find(".g2s") != dname.size() - 4) + return false; + + // Check for valid files + int validfiles = 0; + for(const auto& entry : std::filesystem::directory_iterator(xpath)) + { + // Skip auto folder paths + auto fname = entry.path().filename().u8string(); + if(fname == "." || fname == "..") + continue; + + // Skip folders + bool issubdir = std::filesystem::is_directory(entry, ec); + if(ec) + { + LogMessage("CanLoad filesystem error " + std::to_string(ec.value()) + ". " + ec.message()); + return false; + } + if(issubdir) + continue; + + // Skip unsupported file formats + auto fext = entry.path().extension().u8string(); + if(fext.size() == 0) + continue; + if(fext[0] == '.') + fext = fext.substr(1); + std::transform(fext.begin(), fext.end(), fext.begin(), [](char c) { return (char)tolower(c); }); + if(fext != "tiff" && fext != "tif" && fext != "g2s.tiff" && fext != "g2s.tif") + continue; + + // We found a supported file type -> Increment the counter + validfiles++; + } + return validfiles > 0; + } + else + { + // If file is selected check file extension + auto fext = xpath.extension().u8string(); + std::transform(fext.begin(), fext.end(), fext.begin(), [](char c) { return (char)tolower(c); }); + return fext == "tiff" || fext == "tif" || fext == "g2s.tiff" || fext == "g2s.tif"; + } + } + catch(std::exception& e) + { + LogMessage("CanLoad error: " + std::string(e.what())); + return false; + } +} + +/** + * Discard closed dataset storage descriptors from cache + * By default storage descriptors are preserved even after the dataset is closed + * To reclaim memory all closed descritors are evicted from cache + */ +void G2SBigTiffStorage::cacheReduce() noexcept +{ + for(auto it = cache.begin(); it != cache.end(); ) + { + if(!it->second.isOpen()) + it = cache.erase(it); + else + it++; + } +} + +/** + * Scan folder subtree for supported files + * @paramm path Folder path + * @param listOfDatasets Dataset path list [out] + * @param maxItems Max dataset count + * @param maxItemLength Max dataset path length + * @param cpos Current position in the list + * @return Was provided buffer large enough to store all dataset paths + */ +bool G2SBigTiffStorage::scanDir(const std::string& path, char** listOfDatasets, int maxItems, int maxItemLength, int cpos) noexcept +{ + if(listOfDatasets == nullptr) + return false; + try + { + auto dp = std::filesystem::u8path(path); + if(!std::filesystem::exists(dp)) + return true; + auto dit = std::filesystem::directory_iterator(dp); + if(!std::filesystem::is_directory(dp)) + return false; + for(const auto& entry : dit) + { + // Skip auto folder paths + auto fname = entry.path().filename().u8string(); + if(fname == "." || fname == "..") + continue; + + // Skip regular files + if(!std::filesystem::is_directory(entry)) + continue; + + // If the folder extension is invalid -> scan the subtree + auto abspath = std::filesystem::absolute(entry).u8string(); + auto fext = entry.path().extension().u8string(); + if(fext.size() == 0) + return scanDir(abspath, listOfDatasets, maxItems, maxItemLength, cpos); + if(fext[0] == '.') + fext = fext.substr(1); + std::transform(fext.begin(), fext.end(), fext.begin(), [](char c) { return (char)tolower(c); }); + if(std::find(supportedFormats.begin(), supportedFormats.end(), fext) == supportedFormats.end()) + return scanDir(abspath, listOfDatasets, maxItems, maxItemLength, cpos); + + // We found a supported dataset folder + // Check result buffer limit + if(cpos >= maxItems || listOfDatasets[cpos] == nullptr) + return false; + + // Add to results list + strncpy(listOfDatasets[cpos], abspath.c_str(), maxItemLength); + cpos++; + } + return true; + } + catch(std::filesystem::filesystem_error&) + { + return false; + } +} + +/** + * Validate image coordinates + * @param fs Dataset handle + * @param coordinates Image coordinates + * @param numCoordinates Coordinate count + * @return Are coordinates valid + */ +bool G2SBigTiffStorage::validateCoordinates(const G2SBigTiffDataset* fs, int coordinates[], int numCoordinates, bool flexaxis0) noexcept +{ + if((std::size_t)numCoordinates != fs->getDimension() && (std::size_t)numCoordinates != fs->getDimension() - 2) + return false; + for(int i = 0; i < (int)fs->getDimension() - 2; i++) + { + if(coordinates[i] < 0) + return false; + if(coordinates[i] >= (int)fs->getActualShape()[i]) + { + if(i > 0 || !flexaxis0) + return false; + } + } + return true; +} + +/** + * Get direct I/O property + * @return Is direct I/O enabled + */ +bool G2SBigTiffStorage::getDirectIO() const noexcept +{ + char buf[MM::MaxStrLength]; + int ret = GetProperty(PROP_DIRECTIO, buf); + if(ret != DEVICE_OK) + return false; + try + { + return std::atoi(buf) != 0; + } + catch(...) { return false; } +} + +/** + * Get flush cycle property + * @return Flush cycle count + */ +int G2SBigTiffStorage::getFlushCycle() const noexcept +{ + char buf[MM::MaxStrLength]; + int ret = GetProperty(PROP_FLUSHCYCLE, buf); + if(ret != DEVICE_OK) + return 0; + try + { + return std::atoi(buf); + } + catch(...) { return 0; } +} + +/** + * Get chunk size property + * @return Chunk size - number of slowest changing dimension coordinates in a single file + */ +int G2SBigTiffStorage::getChunkSize() const noexcept +{ + char buf[MM::MaxStrLength]; + int ret = GetProperty(PROP_CHUNKSIZE, buf); + if(ret != DEVICE_OK) + return 0; + try + { + return std::atoi(buf); + } + catch(...) { return 0; } +} diff --git a/DeviceAdapters/Go2Scope/G2SBigTiffStorage.h b/DeviceAdapters/Go2Scope/G2SBigTiffStorage.h new file mode 100644 index 000000000..47d5b8b33 --- /dev/null +++ b/DeviceAdapters/Go2Scope/G2SBigTiffStorage.h @@ -0,0 +1,103 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SBigTiffStorage.h +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope devices. Includes the experimental StorageDevice +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#pragma once +#include "G2SStorage.h" +#include "G2SBigTiffDataset.h" + +/** + * Storage writer driver for Go2Scope BigTIFF format + * @author Miloš Jovanović + * @version 1.0 + */ +class G2SBigTiffStorage : public CStorageBase +{ +public: + //========================================================================================================================= + // Constructors & destructors + //========================================================================================================================= + G2SBigTiffStorage(); + virtual ~G2SBigTiffStorage() noexcept { Shutdown(); } + +public: + //========================================================================================================================= + // Public interface - Device API + //========================================================================================================================= + int Initialize(); + int Shutdown() noexcept; + void GetName(char* pszName) const; + bool Busy() noexcept { return false; } + +public: + //========================================================================================================================= + // Public interface - Storage API + //========================================================================================================================= + int Create(int handle, const char* path, const char* name, int numberOfDimensions, const int shape[], MM::StorageDataType pixType, + const char* meta, int metaLength) noexcept; + int ConfigureDimension(int handle, int dimension, const char* name, const char* meaning) noexcept; + int ConfigureCoordinate(int handle, int dimension, int coordinate, const char* name) noexcept; + int Close(int handle) noexcept; + int Load(int handle, const char* path) noexcept; + int GetShape(int handle, int shape[]) noexcept; + int GetDataType(int handle, MM::StorageDataType& pixelDataType) noexcept; + int Delete(int handle) noexcept; + int List(const char* path, char** listOfDatasets, int maxItems, int maxItemLength) noexcept; + int AddImage(int handle, int sizeInBytes, unsigned char* pixels, int coordinates[], int numCoordinates, const char* imageMeta, int metaLength) noexcept; + int AppendImage(int handle, int sizeInBytes, unsigned char* pixels, const char* imageMeta, int metaLength) noexcept; + int GetSummaryMeta(int handle, char** meta) noexcept; + int GetImageMeta(int handle, int coordinates[], int numCoordinates, char** meta) noexcept; + const unsigned char* GetImage(int handle, int coordinates[], int numCoordinates) noexcept; + int GetNumberOfDimensions(int handle, int& numDimensions) noexcept; + int GetDimension(int handle, int dimension, char* name, int nameLength, char* meaning, int meaningLength) noexcept; + int GetCoordinate(int handle, int dimension, int coordinate, char* name, int nameLength) noexcept; + int GetImageCount(int handle, int& imgcnt) noexcept; + int SetCustomMetadata(int handle, const char* key, const char* content, int contentLength) noexcept; + int GetCustomMetadata(int handle, const char* key, char** content) noexcept; + bool IsOpen(int handle) noexcept; + bool IsReadOnly(int handle) noexcept; + int GetPath(int handle, char* path, int maxPathLength) noexcept; + bool CanLoad(const char* path) noexcept; + void ReleaseStringBuffer(char* buffer) { delete[] buffer; } + +protected: + //========================================================================================================================= + // Internal methods + //========================================================================================================================= + void cacheReduce() noexcept; + bool scanDir(const std::string& path, char** listOfDatasets, int maxItems, int maxItemLength, int cpos) noexcept; + bool validateCoordinates(const G2SBigTiffDataset* fs, int coordinates[], int numCoordinates, bool flexaxis0 = false) noexcept; + bool getDirectIO() const noexcept; + int getFlushCycle() const noexcept; + int getChunkSize() const noexcept; + +private: + //========================================================================================================================= + // Data members + //========================================================================================================================= + std::map cache; ///< Storage entries cache + std::vector supportedFormats = { "g2s" }; ///< Supported file formats + bool initialized = false; ///< Is driver initialized +}; diff --git a/DeviceAdapters/Go2Scope/G2SBigTiffStream.cpp b/DeviceAdapters/Go2Scope/G2SBigTiffStream.cpp new file mode 100644 index 000000000..33c9776d5 --- /dev/null +++ b/DeviceAdapters/Go2Scope/G2SBigTiffStream.cpp @@ -0,0 +1,1209 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SBigTiffStream.cpp +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope devices. Includes the experimental StorageDevice +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#define _LARGEFILE64_SOURCE +#include +#include +#include +#include "G2SBigTiffStream.h" +#ifdef _WIN32 +#include +#else +#include +#include +#include +#include +#endif + +/** + * Class constructor + * Constructor doesn't open the file, just creates an object set sets the configuration + * @param path File path + * @param dio Use direct I/O + * @param fbig Use BigTIFF format + * @param chunk Chunk index + */ +G2SBigTiffStream::G2SBigTiffStream(const std::string& path, bool dio, bool fbig, std::uint32_t chunk) noexcept +{ + currpos = 0; + writepos = 0; + readpos = 0; + lastifdpos = 0; + lastifdsize = 0; + currentifdpos = 0; + currentifdsize = 0; + nextifdpos = 0; + readbuffoff = 0; + imgcounter = 0; + currentimage = 0; + chunkindex = chunk; +#ifdef _WIN32 + fhandle = nullptr; +#else + fhandle = 0; +#endif + fpath = path; + directIo = dio; + bigTiff = fbig; + ssize = 0; +} + +/** + * Open / create a file + * File path is optional (if it's already been set) + * If the file doesn't exist it will be created (write mode) + * If the file exists and 'trunc' is set to true existing file will be discared and new one will be created (write mode) + * If the file exists and 'trunc' is set to false dataset shape, pixel format and metadata will be parsed (read / append mode) + * @param trunc Trucate existing file + * @param index Index IFDs + * @throws std::runtime_error + */ +void G2SBigTiffStream::open(bool trunc) +{ + if(isOpen()) + throw std::runtime_error("Invalid operation. File stream is already open"); + if(fpath.empty()) + throw std::runtime_error("Unable to open a file stream. File path is undefined"); + auto fexists = std::filesystem::exists(std::filesystem::u8path(fpath)); + auto xpath = std::filesystem::u8path(fpath); + + // Obtain a file handle +#ifdef _WIN32 + // Convert file path + int slength = (int)fpath.length() + 1; + int len = MultiByteToWideChar(CP_UTF8, 0, fpath.c_str(), slength, nullptr, 0); + wchar_t* buf = new wchar_t[len]; + MultiByteToWideChar(CP_UTF8, 0, fpath.c_str(), slength, buf, len); + + // Open a file + DWORD cattr = trunc ? CREATE_ALWAYS : OPEN_ALWAYS; + DWORD fattr = directIo ? FILE_FLAG_WRITE_THROUGH | FILE_FLAG_NO_BUFFERING : 0; + fhandle = CreateFile(buf, GENERIC_WRITE | GENERIC_READ, 0, NULL, cattr, FILE_ATTRIBUTE_NORMAL | fattr, NULL); + if(fhandle == INVALID_HANDLE_VALUE) + { + delete[] buf; + fhandle = nullptr; + throw std::runtime_error("File '" + fpath + "' open failed, error code " + std::to_string(GetLastError())); + } + delete[] buf; +#else + int flags = O_LARGEFILE | O_RDWR | O_SYNC | (trunc ? O_CREAT | O_TRUNC : 0) | (directIo ? O_DIRECT : 0); + fhandle = ::open(fpath.c_str(), flags); + if(fhandle < 0) + { + fhandle = 0; + throw std::runtime_error("File '" + fpath + "' open failed, error code " + std::to_string(errno)); + } +#endif + + // Obtain a block size + if(ssize == 0) + { +#ifdef _WIN32 + auto apath = std::filesystem::absolute(xpath); + auto dsepind = apath.u8string().find_first_of('\\'); + auto drivepath = dsepind == std::string::npos ? apath.u8string() : apath.u8string().substr(0, dsepind + 1); + slength = (int)drivepath.length() + 1; + len = MultiByteToWideChar(CP_UTF8, 0, drivepath.c_str(), slength, nullptr, 0); + wchar_t* dbuf = new wchar_t[len]; + MultiByteToWideChar(CP_UTF8, 0, drivepath.c_str(), slength, dbuf, len); + + GetDiskFreeSpace(dbuf, NULL, (LPDWORD)&ssize, NULL, NULL); + delete[] dbuf; +#elif __linux__ + size_t blockSize = 0; + ioctl(fhandle, BLKSSZGET, &blockSize); + ssize = (std::uint32_t)blockSize; +#elif __APPLE__ + size_t blockSize = 0; + ioctl(fhandle, DKIOCGETPHYSICALBLOCKSIZE, &blockSize); + ssize = (std::uint32_t)blockSize; +#endif + } + if(directIo) + writebuff.reserve(ssize); + + currpos = 0; + writepos = 0; + readpos = 0; + lastifdpos = 0; + lastifdsize = 0; + currentifdpos = 0; + currentifdsize = 0; + nextifdpos = 0; + currentimage = 0; + readbuffoff = 0; + header.clear(); + ifdcache.clear(); + + // If a new file is created or existing file was truncated -> Create file header + bool freshfile = !fexists || trunc; + if(freshfile) + formHeader(); + else + { + // We are in read mode -> Parse file header + header.resize(G2STIFF_HEADER_SIZE); + auto nb = fetch(&header[0], header.size()); + if(nb == 0) + // Empty file detected -> Create file header + formHeader(); + // To extract paramters from the header call parse() + } +} + +/** + * Close / commit a file stream + * If a file hasn't been open this method will have no effect + * File handle will be released / closed + * In the write / append mode during closing final section (dataset metadat) is commited to file + * File header is also updated with the offset of the final section + * File opened for writing, but without any images should be empty after closing + */ +void G2SBigTiffStream::close() noexcept +{ + if(!isOpen()) + return; + + if(writepos > 0 && lastifdsize > 0 && !lastifd.empty()) + { + // This section is skipped if file is empty, or no write operation has been peformed + std::uint64_t fsize = writepos; + try + { + // Reposition file cursor if last operation was a file read + if(writepos != currpos) + seek(writepos); + + // Commit last sector + if(directIo && !writebuff.empty()) + { + auto padsize = ssize - writebuff.size(); + std::vector pbuff(padsize); + commit(&pbuff[0], pbuff.size()); + } + fsize = writepos; + + // Clear last IFD chain offset + if(bigTiff) + writeInt(&lastifd[lastifdsize - 8], 8, 0); + else + writeInt(&lastifd[lastifdsize - 4], 4, 0); + seek(lastifdpos); + commit(&lastifd[0], lastifd.size()); + + // Update file header + // Set image count + writeInt(&header[bigTiff ? 48 : 36], 4, imgcounter); + seek(0); + commit(&header[0], header.size()); + + // Move cursor at the end of the file stream + seek(0, false); + } + catch(...) { } + + // Set actual file size limit for direct I/O + if(directIo) + { +#ifdef _WIN32 + FILE_END_OF_FILE_INFO inf = {}; + inf.EndOfFile.QuadPart = (LONGLONG)fsize; + SetFileInformationByHandle((HANDLE)fhandle, FileEndOfFileInfo, &inf, sizeof(inf)); +#else + // TODO +#endif + } + } + + // Close the file +#ifdef _WIN32 + CloseHandle((HANDLE)fhandle); + fhandle = nullptr; +#else + ::close(fhandle); + fhandle = 0; +#endif + imgcounter = 0; + currentimage = 0; + chunkindex = 0; + currpos = 0; + writepos = 0; + readpos = 0; + lastifdpos = 0; + lastifdsize = 0; + currentifdpos = 0; + currentifdsize = 0; + nextifdpos = 0; + readbuffoff = 0; + + writebuff.clear(); + readbuff.clear(); + lastifd.clear(); + currentifd.clear(); + header.clear(); + ifdcache.clear(); +} + +/** + * Parse & Validate file header + * This method will also index all IFDs + * Next IFD cursor will be set + * The following parameters will be obtained: + * - TIFF format (BigTIFF or plain TIFF) + * - Chunk index + * - Chunk size + * - Image count + * - First IFD offset + * - Dataset UID + * - Dataset shape + * - Dataset metadata + * - Image bit depth + * - Write cursor position + * @param datasetuid Dataset UID [out] + * @param shape Dataset shape [out] + * @param chunksize Chunk size [out] + * @param metadata Metadata buffer [out] + * @param bitdepth Image bit depth [out] + * @param index Index IFDs + * @throws std::runtime_error + */ +void G2SBigTiffStream::parse(std::string& datasetuid, std::vector& shape, std::uint32_t& chunksize, std::vector& metadata, std::uint8_t& bitdepth, bool index) +{ + // Check header size + if(header.size() != G2STIFF_HEADER_SIZE) + throw std::runtime_error("File '" + fpath + "' open failed. File header is missing"); + + // Check TIFF format header + if(header[0] != 0x49 || header[1] != 0x49 || (header[2] != 0x2a && header[2] != 0x2b)) + throw std::runtime_error("File '" + fpath + "' open failed. Unsupported file format"); + if(header[2] == 0x2a) + { + // Regular TIFF format detected + if(header[3] != 0) + throw std::runtime_error("File '" + fpath + "' open failed. Unsupported file format"); + // Check G2S Format header + if(header[8] != 0x3c || header[9] != 0x1d || header[10] != 0x59 || header[11] != 0x69) + throw std::runtime_error("File '" + fpath + "' open failed. Unsupported file format"); + bigTiff = false; + } + else + { + // BigTIFF format detected + if(header[4] != 0x08 || header[5] != 0x00 || header[6] != 0x00 || header[7] != 0x00) + throw std::runtime_error("File '" + fpath + "' open failed. Unsupported file format"); + // Check G2S Format header + if(header[16] != 0x3c || header[17] != 0x1d || header[18] != 0x59 || header[19] != 0x69) + throw std::runtime_error("File '" + fpath + "' open failed. Unsupported file format"); + bigTiff = true; + } + currentifdpos = readInt(&header[bigTiff ? 8 : 4], bigTiff ? 8 : 4); + chunkindex = (std::uint32_t)readInt(&header[bigTiff ? 20 : 12], 4); + imgcounter = (std::uint32_t)readInt(&header[bigTiff ? 48 : 36], 4); + + // Parse dataset UID + auto sind = bigTiff ? 24 : 16; + for(std::size_t i = 0; i < 16; i++) + { + if(i == 4 || i == 6 || i == 8 || i == 10) + datasetuid += "-"; + char dig1 = (header[sind + i] & 0xf0) >> 4; + char dig2 = (header[sind + i] & 0x0f); + if(0 <= dig1 && dig1 <= 9) + dig1 += 48; + if(10 <= dig1 && dig1 <= 15) + dig1 += 97 - 10; + if(0 <= dig2 && dig2 <= 9) + dig2 += 48; + if(10 <= dig2 && dig2 <= 15) + dig2 += 97 - 10; + datasetuid.append(&dig1, 1); + datasetuid.append(&dig2, 1); + } + if(datasetuid == "00000000-0000-0000-0000-000000000000") + datasetuid = ""; + + // Parse shape data + auto shapedim = (std::uint32_t)readInt(&header[bigTiff ? 52 : 40], 4); + shape.clear(); + for(std::uint32_t i = 0; i < shapedim; i++) + shape.push_back((std::uint32_t)readInt(&header[(std::size_t)(bigTiff ? 56 : 44) + (std::size_t)i * 4], 4)); + + // Parse chunk size + chunksize = (std::uint32_t)readInt(&header[(std::size_t)(bigTiff ? 56 : 44) + (std::size_t)shapedim * 4], 4); + + // Get file size + auto fsize = std::filesystem::file_size(std::filesystem::u8path(fpath)); + + // Parse metadata + auto metaoffset = readInt(&header[bigTiff ? 40 : 32], bigTiff ? 8 : 4); + if(metaoffset > 0) + { + metadata.resize(fsize - metaoffset); + seek(metaoffset); + fetch(&metadata[0], metadata.size()); + } + + // Set write position at the metadata section start or at the end of the file stream + writepos = metaoffset == 0 ? fsize : metaoffset; + + // Index IFDs + bool pixformatset = false; + if(index && currentifdpos > 0) + { + seek(currentifdpos); + ifdcache.push_back(currentifdpos); + + int i = 0; + std::vector lbuff; + lbuff.reserve(G2STIFF_HEADER_SIZE); + while(true) + { + lbuff.clear(); + lbuff.resize(8); + auto nb = fetch(&lbuff[0], lbuff.size()); + if(nb == 0) + throw std::runtime_error("File '" + fpath + "' open failed. IFD " + std::to_string(i) + " is corrupted"); + std::uint32_t tagcount = (std::uint32_t)readInt(&lbuff[0], bigTiff ? 8 : 2); + + std::uint32_t ifdsz = 0, basesz = 0; + calcDescSize(0, tagcount, &ifdsz, &basesz, nullptr); + lbuff.resize(basesz); + nb = fetch(&lbuff[8], lbuff.size() - 8); + + // Obtain pixel format from the first IFD) + if(!pixformatset) + { + bitdepth = (uint8_t)readInt(&lbuff[bigTiff ? 60 : 34], 2); + pixformatset = true; + } + + auto nextoffset = readInt(&lbuff[(std::size_t)ifdsz - (std::size_t)(bigTiff ? 8 : 4)], bigTiff ? 8 : 4); + if(nextoffset >= fsize) + throw std::runtime_error("File '" + fpath + "' open failed. IFD " + std::to_string(i) + " link is corrupted"); + if(nextoffset == 0) + break; + lastifdpos = nextoffset; + lastifdsize = ifdsz; + ifdcache.push_back(lastifdpos); + seek(lastifdpos); + i++; + } + } + + // Rewind read cursor + seek(currentifdpos); + moveReadCursor(currentifdpos); + nextifdpos = currentifdpos; +} + +/** + * Construct header cache (write mode only) + * Next IFD cursor will be set + */ +void G2SBigTiffStream::formHeader() noexcept +{ + header.clear(); + header.resize(G2STIFF_HEADER_SIZE); + if(bigTiff) + { + // BigTIFF file header + writeInt(&header[0], 4, 0x002b4949); + writeInt(&header[4], 4, 0x08); + + // Set first IFD to 0x00 00 02 00 (512) + writeInt(&header[8], 8, G2STIFF_HEADER_SIZE); + + // Write G2STIFF format signature + writeInt(&header[16], 4, 0x69591d3c); + + // Write chunk index + writeInt(&header[20], 4, chunkindex); + } + else + { + // TIFF file header + writeInt(&header[0], 4, 0x002a4949); + + // Set first IFD to 0x00 00 02 00 (512) + writeInt(&header[4], 4, G2STIFF_HEADER_SIZE); + + // Write G2STIFF format signature + writeInt(&header[8], 4, 0x69591d3c); + + // Write chunk index + writeInt(&header[12], 4, chunkindex); + } + + currentifdpos = G2STIFF_HEADER_SIZE; + lastifdpos = G2STIFF_HEADER_SIZE; + nextifdpos = currentifdpos; +} + +/** + * Add image / write image to the file + * Images are added sequentially + * Image data is stored uncompressed + * Metadata is stored in plain text, after the pixel data + * Image IFD is stored before pixel data + * @param buff Image buffer + * @param len Image buffer length + * @param imgw Image width + * @param imgh Image height + * @param imgdepth Image bit depth + * @param meta Image metadata (optional) + * @throws std::runtime_error + */ +void G2SBigTiffStream::addImage(const unsigned char* buff, std::size_t len, std::uint32_t imgw, std::uint32_t imgh, std::uint32_t imgdepth, const std::string& meta) +{ + // Check file size limits + std::uint32_t tot = 0; + calcDescSize(meta.empty() ? 0 : meta.size() + 1, getTagCount(meta), nullptr, nullptr, &tot); + if(meta.size() + len + currpos + tot > getMaxFileSize()) + throw std::runtime_error("Invalid operation. File size limit exceeded"); + + // Commit header if empty file + if(writepos == 0) + { + commit(&header[0], header.size()); + lastifdpos = readInt(&header[bigTiff ? 8 : 4], bigTiff ? 8 : 4); + } + // Update last IFD for images in read mode + else if(lastifd.empty() && lastifdpos > 0) + { + // Move read cursor to the last IFD + auto lreadpos = readpos; + auto lwritepos = writepos; + seek(lastifdpos); + moveReadCursor(currpos); + + // Load last IFD and change the next IFD offset + auto nextoff = parseIFD(lastifd, lastifdsize); + if(nextoff == 0) + writeInt(&lastifd[(std::size_t)lastifdsize - (std::size_t)(bigTiff ? 8 : 4)], bigTiff ? 8 : 4, writepos); + + // Update last IFD + seek(lastifdpos); + commit(&lastifd[0], lastifd.size()); + + // Reset cursors + moveReadCursor(lreadpos); + moveWriteCursor(lwritepos); + } + + // Reposition file cursor if last operation was a file read + if(writepos != currpos) + seek(writepos); + + // Compose next IFD and write image metadata + appendIFD(imgw, imgh, imgdepth, len, meta); + + // Write pixel data + commit(buff, len); + + // Add padding bytes + auto alignsz = directIo ? ssize : 2; + if(len % alignsz != 0) + { + auto padsize = len - (len / alignsz) * alignsz; + std::vector pbuff(padsize); + commit(&pbuff[0], pbuff.size()); + } + + // Flush pending data + ifdcache.push_back(lastifdpos); + imgcounter++; +} + +/** + * Get image data (pixel buffer) (for the current IFD) + * @return Image data + * @throws std::runtime_error + */ +std::vector G2SBigTiffStream::getImage() +{ + // Obtain pixel data strip locations + auto offind = (bigTiff ? 8 : 2) + 5 * (bigTiff ? 20 : 12); + auto lenind = (bigTiff ? 8 : 2) + 7 * (bigTiff ? 20 : 12); + auto dataoffset = readInt(¤tifd[(std::size_t)offind + (std::size_t)(bigTiff ? 12 : 8)], bigTiff ? 8 : 4); + auto datalen = readInt(¤tifd[(std::size_t)lenind + (std::size_t)(bigTiff ? 12 : 8)], bigTiff ? 8 : 4); + if(dataoffset == 0 || datalen == 0) + return {}; + + std::vector ret(datalen); + moveReadCursor(seek(dataoffset)); + fetch(&ret[0], ret.size()); + return ret; +} + +/** + * Get image metadata (for the current IFD) + * If no metadata is defined this method will return an empty string + * @return Image metadata + * @throws std::runtime_error + */ +std::string G2SBigTiffStream::getImageMetadata() const +{ + // Check IFD tag count + auto tagcount = readInt(¤tifd[0], bigTiff ? 8 : 2); + if(tagcount == G2STIFF_TAG_COUNT_NOMETA) + return ""; + + // Obtain metadata OFFSET and length + std::size_t metatagind = (std::size_t)(bigTiff ? 8 : 2) + (std::size_t)G2STIFF_TAG_COUNT_NOMETA * (bigTiff ? 20 : 12); + auto metalen = readInt(¤tifd[metatagind + 4], bigTiff ? 8 : 4); + auto metaoffset = readInt(¤tifd[metatagind + (bigTiff ? 12 : 8)], bigTiff ? 8 : 4); + if(metalen == 0 || metaoffset == 0) + return ""; + if(metaoffset < currentifdpos) + throw std::runtime_error("Unable to obtain image metadata. File is corrupted"); + + // Copy metadata from the IFD + auto roff = metaoffset - currentifdpos; + auto strlen = roff + metalen > currentifd.size() ? currentifd.size() - roff - metalen : metalen - 1; + std::string str(¤tifd[roff], ¤tifd[roff + strlen]); + return str; +} + +/** + * Write shape info to the header cache + * @param shape Dataset shape + * @param chunksz Chunk size + */ +void G2SBigTiffStream::writeShapeInfo(const std::vector& shape, std::uint32_t chunksz) noexcept +{ + if(header.size() < G2STIFF_HEADER_SIZE || shape.empty()) + return; + + auto startind = bigTiff ? 52 : 40; + writeInt(&header[startind], 4, shape.size()); + for(std::size_t i = 0; i < shape.size(); i++) + writeInt(&header[startind + (i + 1) * 4], 4, shape[i]); + writeInt(&header[startind + (shape.size() + 1) * 4], 4, chunksz); +} + +/** + * Write dataset UID to the header cache + * @param datasetuid Dataset UID + */ +void G2SBigTiffStream::writeDatasetUid(const std::string& datasetuid) noexcept +{ + if(header.size() < G2STIFF_HEADER_SIZE) + return; + + // Write UID to the header cache + auto startind = bigTiff ? 24 : 16; + auto cind = 0; + for(int i = 0; i < 16; i++) + { + if(i == 4 || i == 6 || i == 8 || i == 10) + cind++; + char cv1 = datasetuid[cind++]; + char cv2 = datasetuid[cind++]; + std::uint8_t vx1 = cv1 >= 48 && cv1 <= 57 ? cv1 - 48 : (cv1 >= 65 && cv1 <= 70 ? cv1 - 55 : cv1 - 87); + std::uint8_t vx2 = cv2 >= 48 && cv2 <= 57 ? cv2 - 48 : (cv2 >= 65 && cv2 <= 70 ? cv2 - 55 : cv2 - 87); + auto xval = (std::uint8_t)(((vx1 & 0x0f) << 4) | (vx2 & 0x0f)); + header[startind + i] = xval; + } +} + +/** + * Set chunk index + * @param val Chunk index (zero based) + */ +void G2SBigTiffStream::setChunkIndex(std::uint32_t val) noexcept +{ + chunkindex = val; + if(header.size() == G2STIFF_HEADER_SIZE) + writeInt(&header[bigTiff ? 20 : 12], 4, chunkindex); +} + +/** +* Append metadata at the end of the file +* @param meta Metadata buffer +* @throws std::runtime_error +*/ +void G2SBigTiffStream::appendMetadata(const std::vector& meta) +{ + if(!isOpen() || meta.empty()) + return; + + // Reposition file cursor if last operation was a file read + if(writepos != currpos) + seek(writepos); + + // Write metadata + writeInt(&header[bigTiff ? 40 : 32], bigTiff ? 8 : 4, writepos); + commit(&meta[0], meta.size()); +} + +/** + * Send data to be written to the file stream + * if direct I/O is used for small data sizes data will be stored in the temporary buffer + * For cached I/O and large buffer sizes for direct I/O data is written to the file stream + * @param buff Source buffer + * @param len Source buffer length + * @return Number of bytes written + * @throws std::runtime_error + */ +std::size_t G2SBigTiffStream::commit(const unsigned char* buff, std::size_t len) +{ + if(!isOpen()) + throw std::runtime_error("File write failed. No valid file stream available"); + if(directIo) + { + // Clear write cache if file cursor has been moved + if(currpos != writepos) + writebuff.clear(); + + // Direct I/O - Buffer size must be a multiple of a disk sector size (usually 512 B) + std::size_t ret = 0; + + // Fill pending data with the first sector / write pending data + std::size_t boffset = 0; + if(!writebuff.empty()) + { + auto curr = writebuff.size(); + auto cnt = curr + len > ssize ? ssize - curr : len; + writebuff.resize(writebuff.size() + cnt); + std::memcpy(&writebuff[curr], buff, cnt); + boffset += cnt; + if(writebuff.size() == ssize) + { + ret += write(&writebuff[0], ssize); + writebuff.clear(); + } + else + return ret; + } + if(len - boffset == 0) + return ret; + + // Write middle sectors directly + if(len - boffset >= ssize) + { + auto wcnt = ((len - boffset) / ssize) * ssize; + ret += write(buff + boffset, wcnt); + boffset += wcnt; + } + + // Write remaining data (last sector) to the pending data buffer + // Pending data buffer should be empty at this point + if(len - boffset > 0) + { + auto cpos = writebuff.size(); + writebuff.resize(len - boffset); + std::memcpy(&writebuff[cpos], &buff[boffset], len - boffset); + } + return ret; + } + else + // Standard - Cached I/O + return write(buff, len); +} + +/** + * Write data to the file stream + * @param buff Source buffer + * @param len Source buffer length + * @return Number of bytes written + * @throws std::runtime_error + */ +std::size_t G2SBigTiffStream::write(const unsigned char* buff, std::size_t len) +{ + std::size_t pos = 0; + std::size_t ret = 0; + while(pos < len) + { + std::uint32_t trans = len - pos < std::numeric_limits::max() ? (std::uint32_t)(len - pos) : std::numeric_limits::max(); +#ifdef _WIN32 + DWORD nb = 0; + auto sc = WriteFile((HANDLE)fhandle, (LPCVOID)&buff[pos], (DWORD)trans, &nb, NULL); + if(sc == FALSE) + throw std::runtime_error("File write failed. Error code: " + std::to_string(GetLastError())); + currpos += nb; +#else + auto nb = ::write(fhandle, &buff[pos], trans); + if(nb < 0) + throw std::runtime_error("File write failed. Error code: " + std::to_string(errno)); + currpos += (std::uint64_t)nb; +#endif + ret += nb; + pos += trans; + } + writepos = currpos; + return ret; +} + +/** + * Fetch data from the file stream + * if direct I/O is used for small data sizes entire block is read from file and stored in the temporary buffer + * For cached I/O and large buffer sizes for direct I/O data is read from the file stream + * @param buff Destination buffer + * @param len Destination buffer length + * @return Number of bytes read + * @throws std::runtime_error + */ +std::size_t G2SBigTiffStream::fetch(unsigned char* buff, std::size_t len) +{ + if(!isOpen()) + throw std::runtime_error("File read failed. No valid file stream available"); + if(directIo) + { + // Clear write cache if file cursor has been moved + if(currpos != readpos) + { + readbuff.clear(); + readbuffoff = 0; + } + + // Direct I/O - Buffer size must be a multiple of a disk sector size (usually 512 B) + std::size_t ret = 0; + + // Fetch pending data before requesting another read operation + std::size_t boffset = 0; + if(!readbuff.empty()) + { + std::size_t cnt = (readbuff.size() - readbuffoff) >= len ? len : (readbuff.size() - readbuffoff); + std::memcpy(buff, &readbuff[readbuffoff], cnt); + readbuffoff += cnt; + if(readbuffoff == readbuff.size()) + { + readbuff.clear(); + readbuffoff = 0; + } + boffset += cnt; + } + + // Check if all data has been fetched and if not + if(len - boffset == 0) + return ret; + + // Read middle sectors directly and prefetch last sector + if(len - boffset >= ssize) + { + auto rcnt = ((len - boffset) / ssize) * ssize; + ret += read(&buff[boffset], rcnt); + boffset += rcnt; + } + + // Prefetch last sector + if(len - boffset > 0) + { + readbuff.resize(ssize); + readbuffoff = 0; + ret += read(&readbuff[0], ssize); + std::size_t cnt = len - boffset; + std::memcpy(&buff[boffset], &readbuff[0], cnt); + readbuffoff += cnt; + } + return ret; + } + else + return read(buff, len); +} + +/** + * Read data from the file stream + * @param buff Destination buffer + * @param len Destination buffer length + * @return Number of bytes read + * @throws std::runtime_error + */ +std::size_t G2SBigTiffStream::read(unsigned char* buff, std::size_t len) +{ + if(!isOpen()) + return 0; + std::size_t pos = 0; + std::size_t ret = 0; + while(pos < len) + { + std::uint32_t trans = len - pos < std::numeric_limits::max() ? (std::uint32_t)(len - pos) : std::numeric_limits::max(); +#ifdef _WIN32 + DWORD nb = 0; + auto sc = ReadFile((HANDLE)fhandle, (LPVOID)&buff[pos], trans, &nb, NULL); + if(sc == FALSE) + throw std::runtime_error("File read failed. Error code: " + std::to_string(GetLastError())); + currpos += nb; +#else + auto nb = ::read(fhandle, &buff[pos], trans); + if(nb < 0) + throw std::runtime_error("File read failed. Error code: " + std::to_string(errno)); + currpos += (std::uint64_t)nb; +#endif + ret += nb; + pos += trans; + } + readpos = currpos; + return ret; +} + +/** + * Set cursor position (position in the file stream) + * Moving the cursor will clear temporary buffer (for direct I/O) + * @param pos Position / Offset + * @param beg Use stream begining as a base point (or stream end) + * @return Current position + * @throws std::runtime_error + */ +std::uint64_t G2SBigTiffStream::seek(std::int64_t pos, bool beg) +{ + if(!isOpen()) + throw std::runtime_error("File seek failed. No valid file stream available"); + if(beg && pos < 0) + throw std::runtime_error("File seek failed. Invalid file position"); + if(beg && (std::uint64_t)pos == currpos) + return currpos; +#ifdef _WIN32 + LARGE_INTEGER li = {}; + li.QuadPart = pos; + auto ret = SetFilePointer((HANDLE)fhandle, li.LowPart, &li.HighPart, beg ? FILE_BEGIN : FILE_END); + if(ret == INVALID_SET_FILE_POINTER) + throw std::runtime_error("File seek failed. Error code: " + std::to_string(GetLastError())); + ULARGE_INTEGER ri = {}; + ri.LowPart = ret; + ri.HighPart = (DWORD)li.HighPart; + currpos = ri.QuadPart; +#else + auto ret = lseek64(fhandle, (off64_t)pos, beg ? SEEK_SET : SEEK_END); + if(ret < 0) + throw std::runtime_error("File seek failed. Error code: " + std::to_string(errno)); + currpos = (std::uint64_t)ret; +#endif + return currpos; +} + +/** + * Advance cursor position in the relation to the current cursor position + * Moving the cursor will clear temporary buffer (for direct I/O) + * @param pos Offset + * @return Current position + * @throws std::runtime_error + */ +std::uint64_t G2SBigTiffStream::offset(std::int64_t off) +{ + if(!isOpen()) + throw std::runtime_error("File seek failed. No valid file stream available"); + if(off == 0) + return currpos; +#ifdef _WIN32 + LARGE_INTEGER li = {}; + li.QuadPart = off; + auto ret = SetFilePointer((HANDLE)fhandle, li.LowPart, &li.HighPart, FILE_CURRENT); + if(ret == INVALID_SET_FILE_POINTER) + throw std::runtime_error("File seek failed. Error code: " + std::to_string(GetLastError())); + ULARGE_INTEGER ri = {}; + ri.LowPart = ret; + ri.HighPart = (DWORD)li.HighPart; + currpos = ri.QuadPart; +#else + auto ret = lseek64(fhandle, (off64_t)off, SEEK_CUR); + if(ret < 0) + throw std::runtime_error("File seek failed. Error code: " + std::to_string(errno)); + currpos = (std::uint64_t)ret; +#endif + return currpos; +} + +/** + * Flush I/O write opertions to disk + */ +void G2SBigTiffStream::flush() const +{ +#ifdef _WIN32 + auto sc = FlushFileBuffers((HANDLE)fhandle); + if(sc == FALSE) + throw std::runtime_error("File write flush failed. Error code: " + std::to_string(GetLastError())); +#else + auto sc = ::fsync(fhandle); + if(sc != 0) + throw std::runtime_error("File write flush failed. Error code: " + std::to_string(GetLastError())); +#endif +} + +/** + * Get file size + * @return File size in bytes + */ +std::uint64_t G2SBigTiffStream::getFileSize() const noexcept +{ + if(isOpen()) + { +#ifdef _WIN32 + LARGE_INTEGER fsize = {}; + auto sc = GetFileSizeEx((HANDLE)fhandle, &fsize); + if(sc == 0) + return 0; + return (std::uint64_t)fsize.QuadPart; +#else + auto sz = lseek(fhandle, 0L, SEEK_END); + lseek(fhandle, (off_t)currpos, SEEK_SET); + return (std::uint64_t)sz; +#endif + } + if(fpath.empty()) + return 0; + return std::filesystem::file_size(std::filesystem::u8path(fpath)); +} + +/** + * Form IFD and write it to file stream at the current position + * Metadata is placed immediately after pixel data + * Image / Matadata offsets calculated automatically + * Image sections are block aligned + * @param imgw Image width + * @param imgh Image height + * @param imgdepth Image bit depth + * @param imagelen Pixel data length + * @param meta Image metadata + * @throws std::runtime_error + */ +void G2SBigTiffStream::appendIFD(std::uint32_t imgw, std::uint32_t imgh, std::uint32_t imgdepth, std::size_t imagelen, const std::string& meta) +{ + std::uint32_t descsz = 0, totsz = 0, tagcnt = getTagCount(meta); + auto actimglen = imagelen + (imagelen % ssize == 0 ? 0 : (ssize - (imagelen % ssize))); + calcDescSize(meta.empty() ? 0 : meta.size() + 1, tagcnt, &lastifdsize, &descsz, &totsz); + lastifdpos = currpos; + lastifd.clear(); + lastifd.resize(totsz); + + // Set TAG count + writeInt(&lastifd[0], bigTiff ? 8 : 2, tagcnt); + + // Add TAGS + std::size_t ind = bigTiff ? 8 : 2; + ind += setIFDTag(&lastifd[ind], 0x0100, 4, imgw); + ind += setIFDTag(&lastifd[ind], 0x0101, 4, imgh); + ind += setIFDTag(&lastifd[ind], 0x0102, 3, imgdepth); + ind += setIFDTag(&lastifd[ind], 0x0103, 3, 1); + ind += setIFDTag(&lastifd[ind], 0x0106, 3, 1); + ind += setIFDTag(&lastifd[ind], 0x0111, bigTiff ? 16 : 4, currpos + totsz); + ind += setIFDTag(&lastifd[ind], 0x0116, 4, imgh); + ind += setIFDTag(&lastifd[ind], 0x0117, bigTiff ? 16 : 4, imagelen); + if(bigTiff) + { + ind += setIFDTag(&lastifd[ind], 0x011a, 5, 0x0100000000); + ind += setIFDTag(&lastifd[ind], 0x011b, 5, 0x0100000000); + } + else + { + ind += setIFDTag(&lastifd[ind], 0x011a, 5, currpos + lastifdsize); + ind += setIFDTag(&lastifd[ind], 0x011b, 5, currpos + lastifdsize + 8); + } + ind += setIFDTag(&lastifd[ind], 0x0128, 3, 1); + if(!meta.empty()) + ind += setIFDTag(&lastifd[ind], 0x010e, 2, currpos + descsz, meta.size() + 1); + + // Write next IFD offset + std::uint64_t nextifd = currpos + totsz + actimglen; + writeInt(&lastifd[ind], bigTiff ? 8 : 4, nextifd); + + // Write image resolution values + if(!bigTiff) + { + writeInt(&lastifd[lastifdsize + 0], 8, 0x0100000000); + writeInt(&lastifd[lastifdsize + 8], 8, 0x0100000000); + } + + // Write metadata + if(!meta.empty()) + std::copy(meta.begin(), meta.end(), lastifd.begin() + descsz); + + // Write IFD + metadata + commit(&lastifd[0], lastifd.size()); +} + +/** + * Set IFD field + * Field data will be appended to the IFD buffer + * @param ifd IFD buffer + * @param tag Field tag + * @param dtype Data type + * @param val Field value / offset + * @param cnt Value count + * @return Number of bytes written + */ +std::size_t G2SBigTiffStream::setIFDTag(unsigned char* ifd, std::uint16_t tag, std::uint16_t dtype, std::uint64_t val, std::uint64_t cnt) const noexcept +{ + // Write tag + writeInt(&ifd[0], 2, tag); + + // Write data type + writeInt(&ifd[2], 2, dtype); + + // Write value count + writeInt(&ifd[4], bigTiff ? 8 : 4, cnt); + + // Write value / offset + writeInt(&ifd[bigTiff ? 12 : 8], bigTiff ? 8 : 4, val); + + return bigTiff ? 20 : 12; +} + +/** + * Parse IFD at the current read cursor + * @param ifd IFD buffer [out] + * @param ifdsz IFD size [out] + * @return Next IFD offset + * @throws std::runtime_error + */ +std::uint64_t G2SBigTiffStream::parseIFD(std::vector& ifd, std::uint32_t& ifdsz) +{ + if(readpos != currpos) + seek(readpos); + + ifd.clear(); + ifd.resize(8); + auto nb = fetch(&ifd[0], ifd.size()); + if(nb == 0) + throw std::runtime_error("IFD loading failed. File is corrupted"); + + // Obtain tag count + std::uint32_t tagcount = (std::uint32_t)readInt(&ifd[0], bigTiff ? 8 : 2); + std::uint32_t ifdsize = 0, basesz = 0, totsz = 0; + calcDescSize(0, tagcount, nullptr, &basesz, nullptr); + if(basesz <= 8) + throw std::runtime_error("IFD loading failed. File is corrupted"); + + // Load base IFD + ifd.resize(basesz); + fetch(&ifd[8], ifd.size() - 8); + + // Obtain image metadata length + auto metatagind = (bigTiff ? 8 : 2) + G2STIFF_TAG_COUNT_NOMETA * (bigTiff ? 20 : 12); + auto metalen = (std::size_t)readInt(&ifd[(std::size_t)metatagind + 4], bigTiff ? 8 : 4); + calcDescSize(metalen, tagcount, &ifdsize, &basesz, &totsz); + + // Load entire image descriptor (with metadata) + ifd.resize(totsz); + fetch(&ifd[basesz], ifd.size() - basesz); + + // Set current IFD offset to next IFD offset + auto nextifd = readInt(&ifd[(std::size_t)ifdsize - (bigTiff ? 8 : 4)], bigTiff ? 8 : 4); + ifdsz = ifdsize; + return nextifd; +} + +/** + * Load IFD for reading + * Selected IFD will become the current IFD + * Next IFD offset will be updated + * If the last (written) IFD offset is the same as the selected IFD offset + * no file operation will be performed (IFD will be copied from cache) + * @throws std::runtime_error + */ +void G2SBigTiffStream::loadIFD(std::uint64_t off) +{ + if(!currentifd.empty() && off == currentifdpos) + return; + if(off == 0) + { + // Reset current IFD + currentifd.clear(); + currentifdpos = 0; + currentifdsize = 0; + nextifdpos = 0; + } + else if(!lastifd.empty() && lastifdpos == off) + { + // Copy IFD from cache + currentifd = lastifd; + currentifdpos = off; + currentifdsize = lastifdsize; + nextifdpos = readInt(¤tifd[currentifdsize - (bigTiff ? 8 : 4)], bigTiff ? 8 : 4); + } + else + { + // Load IFD from the file stream + moveReadCursor(seek(off)); + currentifdpos = off; + nextifdpos = parseIFD(currentifd, currentifdsize); + } +} + +/** + * Calculate image descriptor / IFD size + * Image descriptor contains IFD, additional heap for large values (X/Y resolution), image metadata and padding bytes + * Image descriptors are block aligned + * @param metalen Image metadata length + * @param tags Number of IFD tags + * @param ifd IFD size [out] + * @param desc Base descriptor (IFD + additional heap) size [out] + * @parma tot Total image descriptor size [out] + */ +void G2SBigTiffStream::calcDescSize(std::size_t metalen, std::uint32_t tags, std::uint32_t* ifd, std::uint32_t* desc, std::uint32_t* tot) noexcept +{ + // Calculate IFD size + std::uint32_t lifd = bigTiff ? 8 + tags * 20 + 8 : 2 + tags * 12 + 4; + + // Calculate base descriptor size (IFD + additional heap) + auto heapsz = bigTiff ? 0 : 16; + std::uint32_t ldesc = lifd + heapsz; + + // Calculate total size (IFD + additional heap + metadata + padding) + auto basesz = ldesc + (std::uint32_t)metalen; + auto padsz = basesz % ssize == 0 ? 0 : ssize - (basesz % ssize); + std::uint32_t ltot = basesz + padsz; + + // Assign values + if(ifd != nullptr) + *ifd = lifd; + if(desc != nullptr) + *desc = ldesc; + if(tot != nullptr) + *tot = ltot; +} + +/** + * Move file read cursor + * This wont affect current file position (cursor) + * @param pos New position + */ +void G2SBigTiffStream::moveReadCursor(std::uint64_t pos) noexcept +{ + if(pos == readpos) + return; + readpos = pos; + if(directIo && !readbuff.empty()) + { + readbuff.clear(); + readbuffoff = 0; + } +} + +/** + * Move file write cursor + * This wont affect current file position (cursor) + * @param pos New position + */ +void G2SBigTiffStream::moveWriteCursor(std::uint64_t pos) noexcept +{ + if(pos == writepos) + return; + writepos = pos; + if(directIo) + writebuff.clear(); +} diff --git a/DeviceAdapters/Go2Scope/G2SBigTiffStream.h b/DeviceAdapters/Go2Scope/G2SBigTiffStream.h new file mode 100644 index 000000000..75c44b305 --- /dev/null +++ b/DeviceAdapters/Go2Scope/G2SBigTiffStream.h @@ -0,0 +1,149 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SBigTiffStream.h +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope devices. Includes the experimental StorageDevice +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#pragma once +#include +#include +#include +#include "G2SFileUtil.h" + +/** + * BigTIFF file stream descriptor / state cache + * @author Miloš Jovanović + * @version 1.0 + */ +class G2SBigTiffStream +{ +public: + //============================================================================================================================ + // Constructors & Destructors + //============================================================================================================================ + G2SBigTiffStream(const std::string& path, bool dio, bool fbig = DEFAULT_BIGTIFF, std::uint32_t chunk = 0) noexcept; + G2SBigTiffStream(const G2SBigTiffStream& src) noexcept = default; + ~G2SBigTiffStream() noexcept { close(); } + +public: + //============================================================================================================================ + // Public interface - Common methods + //============================================================================================================================ + void open(bool trunc); + void close() noexcept; + void parse(std::string& datasetuid, std::vector& shape, std::uint32_t& chunksize, std::vector& metadata, std::uint8_t& bitdepth, bool index = true); + void formHeader() noexcept; + void addImage(const unsigned char* buff, std::size_t len, std::uint32_t imgw, std::uint32_t imgh, std::uint32_t imgdepth, const std::string& meta = ""); + std::vector getImage(); + std::string getImageMetadata() const; + void writeShapeInfo(const std::vector& shape, std::uint32_t chunksz) noexcept; + void writeDatasetUid(const std::string& datasetuid) noexcept; + void advanceIFD() noexcept { currentifd.clear(); currentifdpos = nextifdpos; } + void setChunkIndex(std::uint32_t val) noexcept; + std::uint32_t getChunkIndex() const noexcept { return chunkindex; } + void setCurrentImage(std::uint32_t val) noexcept { currentimage = val; } + std::uint32_t getCurrentImage() const noexcept { return currentimage; } + void appendMetadata(const std::vector& meta); + const std::vector& getHeader() const noexcept { return header; } + std::vector& getHeader() noexcept { return header; } + const std::vector& getCurrentIFD() const noexcept { return currentifd; } + const std::vector& getLastIFD() const noexcept { return lastifd; } + const std::vector& getIFDOffsets() const noexcept { return ifdcache; } + std::uint64_t getCurrentIFDOffset() const noexcept { return currentifdpos; } + std::uint64_t getNextIFDOffset() const noexcept { return nextifdpos; } + std::string getFilePath() const noexcept { return fpath; } + std::uint64_t getFileSize() const noexcept; + std::uint64_t getMaxFileSize() const noexcept { return bigTiff ? std::numeric_limits::max() : std::numeric_limits::max(); } + std::uint32_t getImageCount() const noexcept { return imgcounter; } + bool isBigTiff() const noexcept { return bigTiff; } +#ifdef _WIN32 + bool isOpen() const noexcept { return fhandle != nullptr; } +#else + bool isOpen() const noexcept { return fhandle > 0; } +#endif + +public: + //============================================================================================================================ + // Public interface - File stream manipulation + //============================================================================================================================ + std::size_t commit(const unsigned char* buff, std::size_t len); + std::size_t write(const unsigned char* buff, std::size_t len); + std::size_t fetch(unsigned char* buff, std::size_t len); + std::size_t read(unsigned char* buff, std::size_t len); + std::uint64_t seek(std::int64_t pos, bool beg = true); + std::uint64_t offset(std::int64_t off); + void flush() const; + +public: + //============================================================================================================================ + // Public interface - Helper methods + //============================================================================================================================ + void appendIFD(std::uint32_t imgw, std::uint32_t imgh, std::uint32_t imgdepth, std::size_t imagelen, const std::string& meta); + std::size_t setIFDTag(unsigned char* ifd, std::uint16_t tag, std::uint16_t dtype, std::uint64_t val, std::uint64_t cnt = 1) const noexcept; + std::uint32_t getTagCount(const std::string& meta) const noexcept { return meta.empty() ? G2STIFF_TAG_COUNT_NOMETA : G2STIFF_TAG_COUNT; } + std::uint64_t parseIFD(std::vector& ifd, std::uint32_t& ifdsz); + void loadNextIFD() { loadIFD(currentifdpos); } + void loadIFD(std::uint64_t off); + void calcDescSize(std::size_t metalen, std::uint32_t tags, std::uint32_t* ifd, std::uint32_t* desc, std::uint32_t* tot) noexcept; + void moveReadCursor(std::uint64_t pos) noexcept; + void moveWriteCursor(std::uint64_t pos) noexcept; + +private: + //============================================================================================================================ + // Data members - Stream state / cache + //============================================================================================================================ + std::vector header; ///< Header (cache) + std::vector lastifd; ///< Last IFD (cache) + std::vector currentifd; ///< Current IFD (cache) + std::vector writebuff; ///< Write buffer for direct I/O + std::vector readbuff; ///< Read buffer for direct I/O + std::size_t readbuffoff; ///< Read buffer offset + std::uint64_t currpos; ///< Current file stream offset + std::uint64_t writepos; ///< Write stream offset + std::uint64_t readpos; ///< Read stream offset + std::uint64_t lastifdpos; ///< Offset of the last image descriptor + std::uint32_t lastifdsize; ///< Last IFD size + std::uint64_t currentifdpos; ///< Offset of the current image descriptor + std::uint32_t currentifdsize; ///< Current IFD size + std::uint64_t nextifdpos; ///< Offset of the next image descriptor + std::uint32_t currentimage; ///< Current image index (used for reading only) + std::uint32_t imgcounter; ///< Image counter + std::uint32_t chunkindex; ///< Chunk index + std::vector ifdcache; ///< IFD offset cache +#ifdef _WIN32 + void* fhandle; ///< File handle +#else + int fhandle; ///< File descriptor +#endif + +private: + //============================================================================================================================ + // Data members - Configuration + //============================================================================================================================ + std::string fpath; ///< File path + std::uint32_t ssize; ///< Sector size (for direct I/O) + bool directIo; ///< Use direct I/O for file operations + bool bigTiff; ///< Use big TIFF format +}; + +typedef std::shared_ptr G2SFileStreamHandle; \ No newline at end of file diff --git a/DeviceAdapters/Go2Scope/G2SFileUtil.h b/DeviceAdapters/Go2Scope/G2SFileUtil.h new file mode 100644 index 000000000..cf900f0d9 --- /dev/null +++ b/DeviceAdapters/Go2Scope/G2SFileUtil.h @@ -0,0 +1,52 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SFileUtil.h +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope devices. Includes the experimental StorageDevice +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#pragma once +#include +#include +#include + +//=============================================================================================================================== +// Driver version +//=============================================================================================================================== +#define G2STIFF_VERSION "1.0.3" + +//=============================================================================================================================== +// Literals +//=============================================================================================================================== +#define DEFAULT_BIGTIFF true +#define DEFAULT_DIRECT_IO false +#define TIFF_MAX_BUFFER_SIZE 2147483648U +#define G2STIFF_HEADER_SIZE 512 +#define G2STIFF_TAG_COUNT 12 +#define G2STIFF_TAG_COUNT_NOMETA 11 + +//=============================================================================================================================== +// Utility functions +//=============================================================================================================================== +void writeInt(unsigned char* buff, std::uint8_t len, std::uint64_t val) noexcept; +std::uint64_t readInt(const unsigned char* buff, std::uint8_t len) noexcept; +std::vector splitLineCSV(const std::string& line) noexcept; diff --git a/DeviceAdapters/Go2Scope/G2SStorage.cpp b/DeviceAdapters/Go2Scope/G2SStorage.cpp new file mode 100644 index 000000000..4ce0c11e9 --- /dev/null +++ b/DeviceAdapters/Go2Scope/G2SStorage.cpp @@ -0,0 +1,55 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SStorage.cpp +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope devices. Includes the experimental StorageDevice +// +// AUTHOR: Nenad Amodaj +// Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#include "G2SStorage.h" +#include "ModuleInterface.h" +#include "G2SBigTiffStorage.h" + + +/////////////////////////////////////////////////////////////////////////////// +// Exported MMDevice API +/////////////////////////////////////////////////////////////////////////////// +MODULE_API void InitializeModuleData() +{ + RegisterDevice(g_BigTiffStorage, MM::StorageDevice, "BigTIFF storage based on Go2Scope"); +} + +MODULE_API MM::Device* CreateDevice(const char* deviceName) +{ + if(deviceName == 0) + return 0; + + if(strcmp(deviceName, g_BigTiffStorage) == 0) + return new G2SBigTiffStorage(); + + return 0; +} + +MODULE_API void DeleteDevice(MM::Device* pDevice) +{ + delete pDevice; +} diff --git a/DeviceAdapters/Go2Scope/G2SStorage.h b/DeviceAdapters/Go2Scope/G2SStorage.h new file mode 100644 index 000000000..d016ebca4 --- /dev/null +++ b/DeviceAdapters/Go2Scope/G2SStorage.h @@ -0,0 +1,94 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SStorage.h +// PROJECT: Micro-Manager +// SUBSYSTEM: DeviceAdapters +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope devices. Includes the experimental StorageDevice +// +// AUTHORS: Milos Jovanovic +// Nenad Amodaj +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#pragma once +#include "MMDevice.h" +#include "DeviceBase.h" + +////////////////////////////////////////////////////////////////////////////// +// Error codes +// +////////////////////////////////////////////////////////////////////////////// +#define ERR_INTERNAL 144002 + +#define ERR_TIFF 140500 +#define ERR_TIFF_STREAM_UNAVAILABLE 140501 +#define ERR_TIFF_INVALID_PATH 140502 +#define ERR_TIFF_INVALID_DIMENSIONS 140503 +#define ERR_TIFF_INVALID_PIXEL_TYPE 140504 +#define ERR_TIFF_CACHE_OVERFLOW 140505 +#define ERR_TIFF_OPEN_FAILED 140506 +#define ERR_TIFF_CACHE_INSERT 140507 +#define ERR_TIFF_HANDLE_INVALID 140508 +#define ERR_TIFF_STRING_TOO_LONG 140509 +#define ERR_TIFF_INVALID_COORDINATE 140510 +#define ERR_TIFF_DATASET_CLOSED 140511 +#define ERR_TIFF_DATASET_READONLY 140512 +#define ERR_TIFF_DELETE_FAILED 140513 +#define ERR_TIFF_ALLOCATION_FAILED 140514 +#define ERR_TIFF_CORRUPTED_METADATA 140515 +#define ERR_TIFF_UPDATE_FAIL 140516 +#define ERR_TIFF_FILESYSTEM_ERROR 140517 +#define ERR_TIFF_INVALID_META_KEY 140518 + +////////////////////////////////////////////////////////////////////////////// +// Cache configuration +// +////////////////////////////////////////////////////////////////////////////// +#define MAX_CACHE_SIZE 1024 +#define CACHE_HARD_LIMIT 0 + +static const char* g_BigTiffStorage = "G2SBigTiffStorage"; + +/** + * Storage entry descriptor + * @author Miloš Jovanović + * @version 1.0 + */ +struct G2SStorageEntry +{ + /** + * Default initializer + * @param vpath Absoulute path on disk + * @param shape Axis sizes + */ + G2SStorageEntry(const std::string& vpath) noexcept : Path(vpath), FileHandle(nullptr) { } + + /** + * Close the descriptor + */ + void close() noexcept { FileHandle = nullptr; ImageData.clear(); } + /** + * Check if file handle is open + * @return Is file handle open + */ + bool isOpen() noexcept { return FileHandle != nullptr; } + + std::string Path; ///< Absoulute path on disk + std::vector ImageData; ///< Current image data + void* FileHandle; ///< File handle +}; diff --git a/DeviceAdapters/Go2Scope/Go2Scope.vcxproj b/DeviceAdapters/Go2Scope/Go2Scope.vcxproj new file mode 100644 index 000000000..d2f846dfa --- /dev/null +++ b/DeviceAdapters/Go2Scope/Go2Scope.vcxproj @@ -0,0 +1,116 @@ + + + + + Debug + x64 + + + Release + x64 + + + + Go2Scope + {2916e620-1157-4154-8223-887d383e9de6} + Go2Scope + Win32Proj + 10.0 + + + + DynamicLibrary + Unicode + v142 + false + + + DynamicLibrary + Unicode + v142 + true + + + + + + + + + + + + + + + + + <_ProjectFileVersion>10.0.40219.1 + true + false + + + + X64 + + + Disabled + NOMINMAX;WIN32;_DEBUG;_WINDOWS;%(PreprocessorDefinitions) + EnableFastChecks + true + + + 4290;%(DisableSpecificWarnings) + MultiThreadedDebug + stdcpp17 + + + Windows + + + + + + + X64 + + + NOMINMAX;WIN32;NDEBUG;_WINDOWS;%(PreprocessorDefinitions) + true + + + 4290;%(DisableSpecificWarnings) + MultiThreaded + stdcpp17 + + + Windows + true + true + + + + + + + + + + + + + + + + + + + + + {af3143a4-5529-4c78-a01a-9f2a8977ed64} + + + + + + \ No newline at end of file diff --git a/DeviceAdapters/Go2Scope/Go2Scope.vcxproj.filters b/DeviceAdapters/Go2Scope/Go2Scope.vcxproj.filters new file mode 100644 index 000000000..8228fc117 --- /dev/null +++ b/DeviceAdapters/Go2Scope/Go2Scope.vcxproj.filters @@ -0,0 +1,45 @@ + + + + + {52a0a423-e1bc-49cf-9983-c785611e671f} + + + {891fcdc1-6279-4733-a91f-9e785177f914} + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + \ No newline at end of file diff --git a/MMCore/CoreProperty.cpp b/MMCore/CoreProperty.cpp index ce5a43f5d..0b7b94ce2 100644 --- a/MMCore/CoreProperty.cpp +++ b/MMCore/CoreProperty.cpp @@ -160,6 +160,10 @@ void CorePropertyCollection::Execute(const char* propName, const char* value) { core_->setChannelGroup(value); } + else if (strcmp(propName, MM::g_Keyword_CoreStorage) == 0) + { + core_->setStorageDevice(value); + } // unknown property else { diff --git a/MMCore/Devices/DeviceInstances.h b/MMCore/Devices/DeviceInstances.h index 8dfacf4d7..33b7d34f9 100644 --- a/MMCore/Devices/DeviceInstances.h +++ b/MMCore/Devices/DeviceInstances.h @@ -33,3 +33,4 @@ #include "SLMInstance.h" #include "GalvoInstance.h" #include "HubInstance.h" +#include "StorageInstance.h" diff --git a/MMCore/Devices/StorageInstance.cpp b/MMCore/Devices/StorageInstance.cpp new file mode 100644 index 000000000..2ef685f32 --- /dev/null +++ b/MMCore/Devices/StorageInstance.cpp @@ -0,0 +1,248 @@ +// PROJECT: Micro-Manager +// SUBSYSTEM: MMCore +// +// DESCRIPTION: Camera device instance wrapper +// +// COPYRIGHT: Nenad Amodaj 2024 +// All Rights reserved +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// AUTHOR: Nenad Amodaj + +#include "StorageInstance.h" + +int StorageInstance::Create(int handle, const char* path, const char* name, const std::vector& shape, MM::StorageDataType pixType, const char* meta, int metaLength) +{ + RequireInitialized(__func__); + + int ret = GetImpl()->Create(handle, path, name, (int)shape.size(), &shape[0], pixType, meta, metaLength); + if (ret != DEVICE_OK) + return ret; + + return DEVICE_OK; +} + +int StorageInstance::ConfigureDimension(int handle, int dimension, const char* name, const char* meaning) +{ + RequireInitialized(__func__); + + return GetImpl()->ConfigureDimension(handle, dimension, name, meaning); +} + +int StorageInstance::ConfigureCoordinate(int handle, int dimension, int coordinate, const char* name) +{ + RequireInitialized(__func__); + + return GetImpl()->ConfigureCoordinate(handle, dimension, coordinate, name); +} + +int StorageInstance::Close(int handle) +{ + RequireInitialized(__func__); + + return GetImpl()->Close(handle); +} + +int StorageInstance::Load(int handle, const char* path) +{ + RequireInitialized(__func__); + + int ret = GetImpl()->Load(handle, path); + if (ret != DEVICE_OK) + return ret; + + return DEVICE_OK; +} + +int StorageInstance::GetShape(int handle, std::vector& shape) +{ + RequireInitialized(__func__); + int numDim(0); + int ret = GetImpl()->GetNumberOfDimensions(handle, numDim); + if (ret != DEVICE_OK) + return ret; + int* shapeArray = new int[numDim]; + ret = GetImpl()->GetShape(handle, shapeArray); + if (ret != DEVICE_OK) + return ret; + shape.clear(); + for (int i = 0; i < numDim; i++) + shape.push_back(shapeArray[i]); + + return DEVICE_OK; +} + +int StorageInstance::GetPixelType(int handle, MM::StorageDataType& dataType) +{ + RequireInitialized(__func__); + int ret = GetImpl()->GetDataType(handle, dataType); + if (ret != DEVICE_OK) + return ret; + + return DEVICE_OK; +} + +int StorageInstance::Delete(int handle) +{ + RequireInitialized(__func__); + + return GetImpl()->Delete(handle); +} + +int StorageInstance::List(const char* path, std::vector& listOfDatasets) +{ + RequireInitialized(__func__); + const int maxItems(5000); + const int maxItemLength(1024); + std::vector cList(maxItems, nullptr); + for (auto c : cList) + { + c = new char[maxItemLength]; + memset(c, 0, maxItemLength); + } + int ret = GetImpl()->List(path, &cList[0], maxItems, maxItemLength); + if (ret == DEVICE_OK) + { + listOfDatasets.clear(); + + for (auto c : cList) + { + if (strlen(c) == 0) break; + listOfDatasets.push_back(std::string(c)); + } + } + + for (auto c : cList) delete[] c; + + return ret; +} + +int StorageInstance::AddImage(int handle, int sizeInBytes, unsigned char* pixels, std::vector& coordinates, const char* imageMeta, int metaLength) +{ + RequireInitialized(__func__); + return GetImpl()->AddImage(handle, sizeInBytes, pixels, &coordinates[0], (int)coordinates.size(), imageMeta, metaLength); +} + +int StorageInstance::AppendImage(int handle, int sizeInBytes, unsigned char* pixels, const char* imageMeta, int metaLength) +{ + RequireInitialized(__func__); + return GetImpl()->AppendImage(handle, sizeInBytes, pixels, imageMeta, metaLength); +} + +int StorageInstance::GetSummaryMeta(int handle, std::string& meta) +{ + char* cMeta(nullptr); + RequireInitialized(__func__); + int ret = GetImpl()->GetSummaryMeta(handle, &cMeta); + if (ret == DEVICE_OK) + { + if (cMeta) + meta = cMeta; + GetImpl()->ReleaseStringBuffer(cMeta); + } + return ret; +} + +int StorageInstance::GetImageMeta(int handle, const std::vector& coordinates, std::string& meta) +{ + char* cMeta(nullptr); + RequireInitialized(__func__); + int ret = GetImpl()->GetImageMeta(handle, const_cast(&coordinates[0]), (int)coordinates.size(), &cMeta); + if (ret == DEVICE_OK) + { + if (cMeta) + meta = cMeta; + GetImpl()->ReleaseStringBuffer(cMeta); + } + return ret; +} + +int StorageInstance::GetCustomMeta(int handle, const std::string& key, std::string& meta) +{ + char* cMeta(nullptr); + RequireInitialized(__func__); + int ret = GetImpl()->GetCustomMetadata(handle, key.c_str(), &cMeta); + if (ret == DEVICE_OK) + { + if (cMeta) + meta = cMeta; + GetImpl()->ReleaseStringBuffer(cMeta); + } + return ret; +} + +int StorageInstance::SetCustomMeta(int handle, const std::string& key, const char* meta, int metaLength) +{ + RequireInitialized(__func__); + return GetImpl()->SetCustomMetadata(handle, key.c_str(), meta, metaLength); +} + +const unsigned char* StorageInstance::GetImage(int handle, const std::vector& coordinates) +{ + RequireInitialized(__func__); + return GetImpl()->GetImage(handle, const_cast(&coordinates[0]), (int)coordinates.size()); +} + +int StorageInstance::GetNumberOfDimensions(int handle, int& numDim) +{ + RequireInitialized(__func__); + return GetImpl()->GetNumberOfDimensions(handle, numDim); +} + +int StorageInstance::GetDimension(int handle, int dimension, std::string& name, std::string& meaning) +{ + char nameStr[MM::MaxStrLength]; + char meaningStr[MM::MaxStrLength]; + memset(nameStr, 0, MM::MaxStrLength); + memset(meaningStr, 0, MM::MaxStrLength); + int ret = GetImpl()->GetDimension(handle, dimension, nameStr, MM::MaxStrLength, meaningStr, MM::MaxStrLength); + name = nameStr; + meaning = meaningStr; + return ret; +} + +int StorageInstance::GetCoordinate(int handle, int dimension, int coordinate, std::string& name) +{ + char cName[MM::MaxStrLength]; + memset(cName, 0, MM::MaxStrLength); + int ret = GetImpl()->GetCoordinate(handle, dimension, coordinate, cName, MM::MaxStrLength); + name = cName; + return ret; +} + +int StorageInstance::GetImageCount(int handle, int& imgcount) +{ + RequireInitialized(__func__); + return GetImpl()->GetImageCount(handle, imgcount); +} + +int StorageInstance::GetPath(int handle, std::string& path) +{ + char cPath[MM::MaxStrLength]; + memset(cPath, 0, MM::MaxStrLength); + int ret = GetImpl()->GetPath(handle, cPath, MM::MaxStrLength); + path = cPath; + return ret; +} + +bool StorageInstance::IsOpen(int handle) +{ + RequireInitialized(__func__); + return GetImpl()->IsOpen(handle); +} + +bool StorageInstance::IsReadOnly(int handle) +{ + RequireInitialized(__func__); + return GetImpl()->IsReadOnly(handle); +} diff --git a/MMCore/Devices/StorageInstance.h b/MMCore/Devices/StorageInstance.h new file mode 100644 index 000000000..e4a8b7da2 --- /dev/null +++ b/MMCore/Devices/StorageInstance.h @@ -0,0 +1,62 @@ +// PROJECT: Micro-Manager +// SUBSYSTEM: MMCore +// +// COPYRIGHT: University of California, San Francisco, 2014, +// Nenad Amodaj 2024 +// All Rights reserved +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// AUTHOR: Nenad Amodaj + +#pragma once + +#include "DeviceInstanceBase.h" + + +class StorageInstance : public DeviceInstanceBase +{ +public: + StorageInstance(CMMCore* core, + std::shared_ptr adapter, + const std::string& name, + MM::Device* pDevice, + DeleteDeviceFunction deleteFunction, + const std::string& label, + mm::logging::Logger deviceLogger, + mm::logging::Logger coreLogger) : + DeviceInstanceBase(core, adapter, name, pDevice, deleteFunction, label, deviceLogger, coreLogger) {} + + int Create(int handle, const char* path, const char* name, const std::vector& shape, MM::StorageDataType pixType, const char* meta, int metaLength); + int ConfigureDimension(int handle, int dimension, const char* name, const char* meaning); + int ConfigureCoordinate(int handle, int dimension, int coordinate, const char* name); + int GetPath(int handle, std::string& path); + int Close(int handle); + int Load(int handle, const char* path); + int GetShape(int handle, std::vector& shape); + int GetPixelType(int handle, MM::StorageDataType& dataType); + int Delete(int handle); + int List(const char* path, std::vector& datasets); + int AddImage(int handle, int sizeInBytes, unsigned char* pixels, std::vector& coordinates, const char* imageMeta, int imageMetaLength); + int AppendImage(int handle, int sizeInBytes, unsigned char* pixels, const char* imageMeta, int imageMetaLength); + int GetSummaryMeta(int handle, std::string& meta); + int GetImageMeta(int handle, const std::vector& coordinates, std::string& meta); + int GetCustomMeta(int handle, const std::string& key, std::string& meta); + int SetCustomMeta(int handle, const std::string& key, const char* meta, int metaLength); + const unsigned char* GetImage(int handle, const std::vector& coordinates); + int GetNumberOfDimensions(int handle, int& numDimensions); + int GetDimension(int handle, int dimension, std::string& name, std::string& meaning); + int GetCoordinate(int handle, int dimension, int coordinate, std::string& name); + int GetImageCount(int handle, int& imgcnt); + bool IsOpen(int handle); + bool IsReadOnly(int handle); +}; diff --git a/MMCore/ErrorCodes.h b/MMCore/ErrorCodes.h index b63a2e918..4ff5cf433 100644 --- a/MMCore/ErrorCodes.h +++ b/MMCore/ErrorCodes.h @@ -78,4 +78,8 @@ #define MMERR_CreatePeripheralFailed 50 #define MMERR_PropertyNotInCache 51 #define MMERR_BadAffineTransform 52 +#define MMERR_StorageNotAvailable 53 +#define MMERR_StorageImageNotAvailable 54 +#define MMERR_StorageMetadataNotAvailable 55 +#define MMERR_StorageInvalidHandle 56 #endif //_ERRORCODES_H_ diff --git a/MMCore/LoadableModules/LoadedDeviceAdapter.cpp b/MMCore/LoadableModules/LoadedDeviceAdapter.cpp index caf2a3404..d39f7cc62 100644 --- a/MMCore/LoadableModules/LoadedDeviceAdapter.cpp +++ b/MMCore/LoadableModules/LoadedDeviceAdapter.cpp @@ -185,6 +185,9 @@ LoadedDeviceAdapter::LoadDevice(CMMCore* core, const std::string& name, return std::make_shared(core, shared_this, name, pDevice, deleter, label, deviceLogger, coreLogger); case MM::HubDevice: return std::make_shared(core, shared_this, name, pDevice, deleter, label, deviceLogger, coreLogger); + case MM::StorageDevice: + return std::make_shared(core, shared_this, name, pDevice, deleter, label, deviceLogger, coreLogger); + default: deleter(pDevice); throw CMMError("Device " + ToQuotedString(name) + diff --git a/MMCore/MMCore.cpp b/MMCore/MMCore.cpp index a768c7cdf..494799861 100644 --- a/MMCore/MMCore.cpp +++ b/MMCore/MMCore.cpp @@ -745,6 +745,13 @@ void CMMCore::assignDefaultRole(std::shared_ptr pDevice) LOG_INFO(coreLogger_) << "Default galvo set to " << label; break; + case MM::StorageDevice: + currentStorage_ = + std::static_pointer_cast(pDevice); + LOG_INFO(coreLogger_) << "Default storage set to " << label; + break; + + default: // no action on unrecognized device break; @@ -1023,6 +1030,23 @@ int CMMCore::initializeVectorOfDevices(std::vector CMMCore::getStorageInstanceFromHandle(int handle) +{ + auto it = openDatasetDevices_.find(handle); + if (it != openDatasetDevices_.end()) + { + auto storage = it->second.lock(); + if (storage) + return storage; + } + throw CMMError(getCoreErrorText(MMERR_StorageInvalidHandle).c_str(), MMERR_StorageInvalidHandle); +} + /** * Updates CoreProperties (currently all Core properties are * devices types) with the loaded hardware. @@ -7581,6 +7605,10 @@ void CMMCore::InitializeErrorMessages() errorText_[MMERR_NullPointerException] = "Null Pointer Exception."; errorText_[MMERR_CreatePeripheralFailed] = "Hub failed to create specified peripheral device."; errorText_[MMERR_BadAffineTransform] = "Bad affine transform. Affine transforms need to have 6 numbers; 2 rows of 3 column."; + errorText_[MMERR_StorageNotAvailable] = "Storage not loaded or initialized."; + errorText_[MMERR_StorageImageNotAvailable] = "Image not available at specified coordinates."; + errorText_[MMERR_StorageMetadataNotAvailable] = "Metadata not available."; + errorText_[MMERR_StorageInvalidHandle] = "Invalid or obsolete handle."; } void CMMCore::CreateCoreProperties() @@ -8032,6 +8060,700 @@ std::vector CMMCore::getLoadedPeripheralDevices(const char* hubLabe return deviceManager_->GetLoadedPeripherals(hubLabel); } +void CMMCore::setStorageDevice(const char* storageLabel) throw(CMMError) +{ + // TODO: prevent setting storage device if the current one has any datasets open + + if (storageLabel && strlen(storageLabel) > 0) + { + currentStorage_ = + deviceManager_->GetDeviceOfType(storageLabel); + LOG_INFO(coreLogger_) << "Default storage set to " << storageLabel; + } + else + { + currentStorage_.reset(); + LOG_INFO(coreLogger_) << "Default storage unset"; + } + properties_->Refresh(); + + std::string newStorageLabel = getStorageDevice(); + { + MMThreadGuard scg(stateCacheLock_); + stateCache_.addSetting(PropertySetting(MM::g_Keyword_CoreDevice, MM::g_Keyword_CoreCamera, newStorageLabel.c_str())); + } + +} + +std::string CMMCore::getStorageDevice() throw(CMMError) +{ + std::shared_ptr pStorage = currentStorage_.lock(); + if (pStorage) + { + return pStorage->GetLabel(); + } + return std::string(); +} + +int CMMCore::createDatasetImpl(std::shared_ptr pStorage, const char* path, const char* name, + const std::vector& shape, MM::StorageDataType pixelType, const char* meta, int metaLength) throw (CMMError) +{ + if (!pStorage) + throw CMMError(getCoreErrorText(MMERR_StorageNotAvailable).c_str(), MMERR_StorageNotAvailable); + + int handle = nextDatasetHandle_++; + { + mm::DeviceModuleLockGuard guard(pStorage); + std::vector intShape(shape.begin(), shape.end()); + int ret = pStorage->Create(handle, path, name, intShape, pixelType, meta, metaLength); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + // We don't reuse failed handle values, but that's okay. + } + } + openDatasetDevices_.insert({handle, pStorage}); + return handle; +} + +/** + * Create new dataset in the specifed path. Fails if the path already exists. + * + * \param path - parent directory for the dataset + * \param name - name for the dataset + * \param shape - array of max coordinates for each dimension (not counting image x and y) + * \param meta - serialized metadata + * \param metaLength - length of the metadata string + * \return - handle for the new dataset + */ +int CMMCore::createDataset(const char* path, const char* name, const std::vector& shape, MM::StorageDataType pixelType, const char* meta, int metaLength) throw (CMMError) +{ + // NOTE: vector is used instead of vector in the signature because of Swig idiosyncracies + std::shared_ptr pStorage = currentStorage_.lock(); + return createDatasetImpl(pStorage, path, name, shape, pixelType, meta, metaLength); + } + +/** + * Create new dataset in the specifed path. Fails if the path already exists. + * + * \param deviceLabel - storage device that we want to use + * \param path - parent directory for the dataset + * \param name - name for the dataset + * \param shape - array of max coordinates for each dimension (not counting image x and y) + * \param meta - serialized metadata + * \param metaLength - length of the metadata string + * \return - handle for the new dataset + */ + +int CMMCore::createDataset(const char* deviceLabel, const char* path, const char* name, const std::vector& shape, MM::StorageDataType pixelType, const char* meta, int metaLength) throw(CMMError) +{ + auto pStorage = deviceManager_->GetDeviceOfType(deviceLabel); + return createDatasetImpl(pStorage, path, name, shape, pixelType, meta, metaLength); +} + +/** + * Close the currently open dataset. + * After closing the handle becomes invalid. + * + * \param handle - handle to existing dataset + */ +void CMMCore::closeDataset(int handle) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + + mm::DeviceModuleLockGuard guard(pStorage); + + int ret = pStorage->Close(handle); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } + + openDatasetDevices_.erase(handle); +} + +/** + * Prevent further changes to the dataset. + * The handle is still valid and the data is available for reading. + * Once frozen dataset can't be thawed again. + * + * \param handle - handle to the open dataset + */ +void CMMCore::freezeDataset(int handle) throw(CMMError) +{ + // TODO: + throw CMMError("Feature not supported", MMERR_GENERIC); +} + +int CMMCore::loadDatasetImpl(std::shared_ptr pStorage, const char* path) throw (CMMError) +{ + if (!pStorage) + throw CMMError(getCoreErrorText(MMERR_StorageNotAvailable).c_str(), MMERR_StorageNotAvailable); + + int handle = nextDatasetHandle_++; + { + mm::DeviceModuleLockGuard guard(pStorage); + int ret = pStorage->Load(handle, path); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + // We don't reuse failed handle values, but that's okay. + } + } + openDatasetDevices_.insert({ handle, pStorage }); + return handle; +} + +/** + * Open an existing dataset in the specifed path. + * + * \param path - parent directory for the dataset + * \param name - name for the dataset + * \return - handle for the opened dataset + */ +int CMMCore::loadDataset(const char* path) throw (CMMError) +{ + std::shared_ptr pStorage = currentStorage_.lock(); + return loadDatasetImpl(pStorage, path); +} + +/** + * Open an existing dataset in the specifed path. + * + * \param deviceLabel - specific storage device that we want to use + * \param path - parent directory for the dataset + * \param name - name for the dataset + * \return - handle for the opened dataset + */ +int CMMCore::loadDataset(const char* deviceLabel, const char* path) throw(CMMError) +{ + auto pStorage = deviceManager_->GetDeviceOfType(deviceLabel); + return loadDatasetImpl(pStorage, path); +} + +/** + * Returns the device that can open a given dataset. + * It will cycle trough all loaded storage devices and find which one can open the dataset. + * + * \param path - path of the dataset + * \return - device name, or empty string if it can not find any devices + */ +std::string CMMCore::getDeviceNameToOpenDataset(const char* path) +{ + // TODO: + throw CMMError("Feature not supported", MMERR_GENERIC); +} + +/** + * Get dataset path + * + * \param handle - handle to the open dataset + * \return - Full dataset path + */ +std::string CMMCore::getDatasetPath(int handle) throw(CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + std::string path; + int ret = pStorage->GetPath(handle, path); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } + return path; +} + +/** + * Check if dataset is open + * + * \param handle - handle to the open dataset + * \return - Dataset is open + */ +bool CMMCore::isDatasetOpen(int handle) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + std::vector shape; + return pStorage->IsOpen(handle); +} + +/** + * Check if dataset is read-only (or in WRITE mode) + * + * \param handle - handle to the open dataset + * \return - Dataset is read-only + */ +bool CMMCore::isDatasetReadOnly(int handle) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + std::vector shape; + return pStorage->IsReadOnly(handle); +} + +/** + * Retrieves the shape (dimensions) of a dataset + * Gets the size of each dimension for the specified dataset handle. + * + * \param handle The handle of the dataset to query + * \return Vector of dimension sizes, where each element represents the size of one dimension + */ +std::vector CMMCore::getDatasetShape(int handle) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + std::vector shape; + int ret = pStorage->GetShape(handle, shape); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } + return shape; +} + +/** + * Returns the pixel type for the dataset. + * The pixel type enum is specific to storage device. + * + * \param handle The handle of the dataset to query + * \return storage type + */ + +MM::StorageDataType CMMCore::getDatasetPixelType(int handle) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + MM::StorageDataType pixType; + int ret = pStorage->GetPixelType(handle, pixType); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } + return pixType; +} + +/** + * Appends a new image to the dataset. Width, height and depth define the expected pixel array size. + * The implied order of images is the one specified by order of dimensions during creation of the dataset. + * + * \param handle - handle to the open dataset + * \param sizeInBytes - size of the pixel array + * \param pixels - pixel array + * \param imageMeta - image specific metadata + * \param imageMetaLength - size of the metadata string + */ +void CMMCore::appendImageToDataset(int handle, int sizeInBytes, const STORAGEIMG pixels, const char* imageMeta, int imageMetaLength) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + int ret = pStorage->AppendImage(handle, sizeInBytes, pixels, imageMeta, imageMetaLength); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } +} + +/** + * Adds a new image to the dataset. Width, height and depth define the expected pixel array size. + * Fails if coordinates do not fit into the dataset shape, or if the image dimensions are not supported. + * It can also fail if the underlying implementation does not support the order of image coordinates. + * + * \param handle - handle to the open dataset + * \param sizeInShorts - size of the pixel array + * \param pixels - pixel array + * \param coordinates - coordinates of the image in the dimension space + * \param imageMeta - image specific metadata + * \param imageMetaLength - size of the metadata string + */ +void CMMCore::appendImageToDataset(int handle, int sizeInShorts, const STORAGEIMG16 pixels, const char* imageMeta, int imageMetaLength) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + int ret = pStorage->AppendImage(handle, sizeInShorts * 2, reinterpret_cast(pixels), imageMeta, imageMetaLength); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } +} + +/** + * Configure metadata for a given dimension. + * + * \param handle - handle for the dataset + * \param dimension - dimension index + * \param name - name of the dimension + * \param meaning - Z,T,C, etc. (physical meaning) + */ +void CMMCore::configureDatasetDimension(int handle, int dimension, const char* name, const char* meaning) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + int ret = pStorage->ConfigureDimension(handle, dimension, name, meaning); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } +} + +/** + * Configure a particular coordinate name. E.g. channel name or position name. + * + * \param handle - dataset handle + * \param dimension - dimension index + * \param coordinate - coordinate index + * \param name - coordinate name + */ +void CMMCore::configureDatasetCoordinate(int handle, int dimension, int coordinate, const char* name) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + int ret = pStorage->ConfigureCoordinate(handle, dimension, coordinate, name); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } +} + +/** + * Obtain a dataset dimension name + * + * \param handle - dataset handle + * \param dimension - dimension index + */ +std::string CMMCore::getDatasetDimensionName(int handle, int dimension) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + std::string name, meaning; + int ret = pStorage->GetDimension(handle, dimension, name, meaning); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } + return name; +} + +/** + * Obtain a dataset dimension physical meaning (Z,T,C, etc) + * + * \param handle - dataset handle + * \param dimension - dimension index + */ +std::string CMMCore::getDatasetDimensionMeaning(int handle, int dimension) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + std::string name, meaning; + int ret = pStorage->GetDimension(handle, dimension, name, meaning); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } + return meaning; +} + +/** + * Obtain dataset coordinate name + * + * \param handle - dataset handle + * \param dimension - dimension index + * \param coordinate - coordinate index + */ +std::string CMMCore::getDatasetCoordinateName(int handle, int dimension, int coordinate) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + std::string name; + int ret = pStorage->GetCoordinate(handle, dimension, coordinate, name); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } + return name; +} + +/** + * Get dataset actual image count + * + * \param handle - dataset handle + */ +int CMMCore::getDatasetImageCount(int handle) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + int imgcount = 0; + int ret = pStorage->GetImageCount(handle, imgcount); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } + return imgcount; +} + +/** + * Returns summary metadata serialized as a string + * + * \param handle - handle of the currently loaded or open dataset + */ +std::string CMMCore::getDatasetSummaryMeta(int handle) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + std::string meta; + int ret = pStorage->GetSummaryMeta(handle, meta); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_StorageMetadataNotAvailable); + } + return meta; +} + +/** + * @brief Retrieves metadata for an image at specified coordinates + * + * @param handle open dataset handle + * @param coordinates A vector of spatial coordinates identifying the specific image + * + * @return std::string image metadata as JSON encoded string + */ +std::string CMMCore::getDatasetImageMeta(int handle, const std::vector& coordinates) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + std::string meta; + std::vector coords(coordinates.begin(), coordinates.end()); + int ret = pStorage->GetImageMeta(handle, coords, meta); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_StorageMetadataNotAvailable); + } + return meta; +} + +/** + * Sets custom metadata for a given dataset. + * Custom metadata is re-writable, i.e. it can be called at any time, + * during acquisition or after. + * + * @param handle Dataset handle to set metadata for + * @param key Metadata key/name identifier + * @param meta Metadata string to be stored + * @param metaLength length of the metadata string + * + * @throws MMCoreException If the specified device handle is invalid or key not found + */ +void CMMCore::setDatasetCustomMeta(int handle, const char* key, const char* meta, int metaLength) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + int ret = pStorage->SetCustomMeta(handle, key, meta, metaLength); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError("Error writing custom metadata", MMERR_DEVICE_GENERIC); + } +} + +/** + * Retrieves custom metadata for a given dataset. + * + * @param handle Dataset handle to get metadata from + * @param key Metadata key/name identifier + * + * @return Stored metadata string for the specified key + * @throws MMCoreException If the specified device handle is invalid or key not found + */ +std::string CMMCore::getDatasetCustomMeta(int handle, const char* key) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + std::string meta; + int ret = pStorage->GetCustomMeta(handle, key, meta); + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_StorageMetadataNotAvailable); + } + return meta; +} + +/** + * Returns image pixels at specified coordinates + * \param handle - dataset handle + * \param coordinates - array of coordinates, one for each dimension + * \return - image pixels + */ +STORAGEIMGOUT CMMCore::getImageFromDataset(int handle, const std::vector& coordinates) throw (CMMError) +{ + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard guard(pStorage); + std::vector coords(coordinates.begin(), coordinates.end()); + const unsigned char* img = pStorage->GetImage(handle, coords); + if (!img) + { + logError("CMMCore::getImage()", getCoreErrorText(MMERR_StorageImageNotAvailable).c_str()); + throw CMMError(getCoreErrorText(MMERR_StorageImageNotAvailable).c_str(), MMERR_StorageImageNotAvailable); + } + return const_cast(img); +} + +/** + * Snaps an image on the default camera and immediately save to specified dataset. + * This can be used instead of getImage() and addImage() combination, + * to save the roundtrip of binary data from and to MMCore. + * + * After this command we can still use getImage() if we want to display the image. + * + * \param handle - currently open dataset + * \param imageMeta - image metadata that we wish to add to the image + * \param imageMetaLenght - length of the image metadata + */ +void CMMCore::snapAndAppendToDataset(int handle, const std::vector& coordinates, const char* imageMeta, int imageMetaLength) +{ + snapImage(); + + // TODO: at this point another thread can do something with the camera + // and disrupt the function + + std::shared_ptr camera = currentCameraDevice_.lock(); + try { + // get image + mm::DeviceModuleLockGuard guard(camera); + auto pBuf = const_cast (camera->GetImageBuffer()); + if (!pBuf) + { + logError("CMMCore::getImage()", getCoreErrorText(MMERR_CameraBufferReadFailed).c_str()); + throw CMMError(getCoreErrorText(MMERR_CameraBufferReadFailed).c_str(), MMERR_CameraBufferReadFailed); + } + + // process the image + std::shared_ptr imageProcessor = currentImageProcessor_.lock(); + if (imageProcessor) + { + imageProcessor->Process((unsigned char*)pBuf, camera->GetImageWidth(), camera->GetImageHeight(), camera->GetImageBytesPerPixel()); + } + + // store the image + auto pStorage = getStorageInstanceFromHandle(handle); + mm::DeviceModuleLockGuard storageGuard(pStorage); + int ret(0); + int imageSize = camera->GetImageWidth() * camera->GetImageHeight() * camera->GetImageBytesPerPixel(); + + if (coordinates.empty()) + { + ret = pStorage->AppendImage(handle, imageSize, pBuf, imageMeta, imageMetaLength); + } + else + { + std::vector coords(coordinates.begin(), coordinates.end()); + ret = pStorage->AddImage(handle, imageSize, pBuf, coords, imageMeta, imageMetaLength); + } + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } + + } + catch (CMMError& e) { + throw e; + } + catch (...) { + logError("CMMCore::snapAndSave()", getCoreErrorText(MMERR_UnhandledException).c_str()); + throw CMMError(getCoreErrorText(MMERR_UnhandledException).c_str(), MMERR_UnhandledException); + } +} + +/** + * Pops the next image from the circular buffer and saves it to the dataset. + * This is the counterpart to popNextImage(). Instead of fetching image pixels it + * it sends image directly to the storage. + * + * \param handle - currently open dataset handle + * \param imageMeta - image metadata + * \param imageMetaLength - image metadata length + */ +void CMMCore::appendNextToDataset(int handle, const std::vector& coordinates, const char* imageMeta, int imageMetaLength) throw (CMMError) +{ + const mm::ImgBuffer* img = cbuf_->GetNextImageBuffer(0); + if (!img) + throw CMMError(getCoreErrorText(MMERR_CircularBufferEmpty).c_str(), MMERR_CircularBufferEmpty); + + // store the image + auto pStorage = getStorageInstanceFromHandle(handle); + int ret(0); + int imageSize = img->Width() * img->Height() * img->Depth(); + mm::DeviceModuleLockGuard guard(pStorage); + if (coordinates.empty()) + { + ret = pStorage->AppendImage(handle, imageSize, const_cast(img->GetPixels()), imageMeta, imageMetaLength); + } + else + { + std::vector coords(coordinates.begin(), coordinates.end()); + ret = pStorage->AddImage(handle, imageSize, const_cast(img->GetPixels()), coords, imageMeta, imageMetaLength); + } + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } +} + +/** + * Pops the next image from the circular buffer and saves it to the dataset. + * This function merges popNextImage() and saveNextImage(). In addition to returning the image it also + * sends it to storage. The use case is when we want to save and display the image at the same time. + * + * \param handle - currently open dataset handle + * \param imageMeta - image metadata + * \param imageMetaLength - length of the image metadata + * \return - image pixels + */ +STORAGEIMGOUT CMMCore::appendAndGetNextToDataset(int handle, const std::vector& coordinates, const char* imageMeta, int imageMetaLength) throw(CMMError) +{ + const mm::ImgBuffer* img = cbuf_->GetNextImageBuffer(0); + if (!img) + throw CMMError(getCoreErrorText(MMERR_CircularBufferEmpty).c_str(), MMERR_CircularBufferEmpty); + + // store the image + auto pStorage = getStorageInstanceFromHandle(handle); + int ret(0); + int imageSize = img->Width() * img->Height() * img->Depth(); + mm::DeviceModuleLockGuard guard(pStorage); + if (coordinates.empty()) + { + ret = pStorage->AppendImage(handle, imageSize, const_cast(img->GetPixels()), imageMeta, imageMetaLength); + } + else + { + std::vector coords(coordinates.begin(), coordinates.end()); + ret = pStorage->AddImage(handle, imageSize, const_cast(img->GetPixels()), coords, imageMeta, imageMetaLength); + } + if (ret != DEVICE_OK) + { + logError(getDeviceName(pStorage).c_str(), getDeviceErrorText(ret, pStorage).c_str()); + throw CMMError(getDeviceErrorText(ret, pStorage).c_str(), MMERR_DEVICE_GENERIC); + } + return const_cast(img->GetPixels()); // returns the image buffer +} + std::string CMMCore::getInstalledDeviceDescription(const char* hubLabel, const char* deviceLabel) throw (CMMError) { std::shared_ptr pHub = diff --git a/MMCore/MMCore.h b/MMCore/MMCore.h index 80d463f07..243d36e96 100644 --- a/MMCore/MMCore.h +++ b/MMCore/MMCore.h @@ -86,6 +86,10 @@ # define MMCORE_DEPRECATED(prototype) prototype #endif +typedef unsigned char* STORAGEIMG; +typedef unsigned short* STORAGEIMG16; +typedef unsigned char* STORAGEIMGOUT; +typedef unsigned char* STORAGEMETA; class CPluginManager; class CircularBuffer; @@ -105,6 +109,7 @@ class SLMInstance; class ShutterInstance; class StageInstance; class XYStageInstance; +class StorageInstance; class CMMCore; @@ -121,7 +126,6 @@ enum DeviceInitializationState { InitializationFailed, }; - /// The Micro-Manager Core. /** * Provides a device-independent interface for hardware control. Additionally, @@ -278,6 +282,7 @@ class CMMCore std::string getSLMDevice(); std::string getGalvoDevice(); std::string getChannelGroup(); + std::string getStorageDevice(); void setCameraDevice(const char* cameraLabel) throw (CMMError); void setShutterDevice(const char* shutterLabel) throw (CMMError); void setFocusDevice(const char* focusLabel) throw (CMMError); @@ -287,6 +292,8 @@ class CMMCore void setSLMDevice(const char* slmLabel) throw (CMMError); void setGalvoDevice(const char* galvoLabel) throw (CMMError); void setChannelGroup(const char* channelGroup) throw (CMMError); + void setStorageDevice(const char* storageLabel) throw (CMMError); + ///@} /** \name System state cache. @@ -637,6 +644,39 @@ class CMMCore std::vector getLoadedPeripheralDevices(const char* hubLabel) throw (CMMError); ///@} + /** \name Storage API */ + ///@{ + int createDataset(const char* path, const char* name, const std::vector& shape, MM::StorageDataType pixelType, const char* meta, int metaLength) throw (CMMError); + int createDataset(const char* deviceLabel, const char* path, const char* name, const std::vector& shape, MM::StorageDataType pixelType, const char* meta, int metaLength) throw (CMMError); + void closeDataset(int handle) throw (CMMError); + void freezeDataset(int handle) throw (CMMError); + int loadDataset(const char* path) throw (CMMError); + int loadDataset(const char* deviceLabel, const char* path) throw (CMMError); + std::string getDeviceNameToOpenDataset(const char* path); + std::string getDatasetPath(int handle) throw (CMMError); + bool isDatasetOpen(int handle); + bool isDatasetReadOnly(int handle); + std::vector getDatasetShape(int handle) throw (CMMError); + MM::StorageDataType getDatasetPixelType(int handle) throw (CMMError); + void appendImageToDataset(int, int sizeinBytes, const STORAGEIMG pixels, const char* imageMeta, int imageMetaLength) throw (CMMError); + void appendImageToDataset(int handle, int sizeinShorts, const STORAGEIMG16 pixels, const char* imageMeta, int imageMetaLength) throw (CMMError); + void configureDatasetDimension(int handle, int dimension, const char* name, const char* meaning) throw (CMMError); + void configureDatasetCoordinate(int handle, int dimension, int coordinate, const char* name) throw (CMMError); + std::string getDatasetDimensionName(int handle, int dimension) throw (CMMError); + std::string getDatasetDimensionMeaning(int handle, int dimension) throw (CMMError); + std::string getDatasetCoordinateName(int handle, int dimension, int coordinate) throw (CMMError); + int getDatasetImageCount(int handle) throw (CMMError); + std::string getDatasetSummaryMeta(int handle) throw (CMMError); + std::string getDatasetImageMeta(int handle, const std::vector& coordinates) throw (CMMError); + void setDatasetCustomMeta(int handle, const char* key, const char* meta, int metaLength); + std::string getDatasetCustomMeta(int handle, const char* key); + STORAGEIMGOUT getImageFromDataset(int handle, const std::vector& coordinates) throw (CMMError); + void snapAndAppendToDataset(int handle, const std::vector& coordinates, const char* imageMeta, int imageMetaLength) throw (CMMError); + void appendNextToDataset(int handle, const std::vector& coordinates, const char* imageMeta, int imageMetaLength) throw (CMMError); + STORAGEIMGOUT appendAndGetNextToDataset(int handle, const std::vector& coordinates, const char* imageMeta, int imageMetaLength) throw (CMMError); + + ///@} + private: // make object non-copyable CMMCore(const CMMCore&); @@ -659,6 +699,7 @@ class CMMCore std::weak_ptr currentSLMDevice_; std::weak_ptr currentGalvoDevice_; std::weak_ptr currentImageProcessor_; + std::weak_ptr currentStorage_; std::string channelGroup_; long pollingIntervalMs_; @@ -672,6 +713,10 @@ class CMMCore PixelSizeConfigGroup* pixelSizeGroup_; CircularBuffer* cbuf_; + // Map open dataset handles to their storage devices + std::map > openDatasetDevices_; + int nextDatasetHandle_ = 0; + std::shared_ptr pluginManager_; std::shared_ptr deviceManager_; std::map errorText_; @@ -711,6 +756,10 @@ class CMMCore void initializeAllDevicesSerial() throw (CMMError); void initializeAllDevicesParallel() throw (CMMError); int initializeVectorOfDevices(std::vector, std::string> > pDevices); + std::shared_ptr getStorageInstanceFromHandle(int handle); + int createDatasetImpl(std::shared_ptr pStorage, const char* path, const char* name, + const std::vector& shape, MM::StorageDataType pixelType, const char* meta, int metaLength) throw (CMMError); + int loadDatasetImpl(std::shared_ptr pStorage, const char* path) throw (CMMError); }; #if defined(__GNUC__) && !defined(__clang__) diff --git a/MMCore/MMCore.vcxproj b/MMCore/MMCore.vcxproj index ebfdcb9ba..d90990cf2 100644 --- a/MMCore/MMCore.vcxproj +++ b/MMCore/MMCore.vcxproj @@ -94,6 +94,7 @@ + @@ -137,6 +138,7 @@ + diff --git a/MMCore/MMCore.vcxproj.filters b/MMCore/MMCore.vcxproj.filters index 1920583b6..ae1f3deb7 100644 --- a/MMCore/MMCore.vcxproj.filters +++ b/MMCore/MMCore.vcxproj.filters @@ -141,6 +141,9 @@ Source Files + + Source Files\Devices + @@ -305,5 +308,8 @@ Header Files + + Header Files\Devices + - + \ No newline at end of file diff --git a/MMCoreJ_wrap/MMCoreJ.i b/MMCoreJ_wrap/MMCoreJ.i index 84a71459b..ff83aeaf1 100644 --- a/MMCoreJ_wrap/MMCoreJ.i +++ b/MMCoreJ_wrap/MMCoreJ.i @@ -314,6 +314,58 @@ $result = data; } +// Map input argument: java byte[] -> C++ STORAGEIMG +%typemap(in) STORAGEIMG +{ + long expectedLength = arg3; + long receivedLength = JCALL1(GetArrayLength, jenv, $input); + + if (receivedLength != expectedLength) + { + jclass excep = jenv->FindClass("java/lang/Exception"); + if (excep) + jenv->ThrowNew(excep, "Byte array dimensions do not match declared size."); + return; + } + + $1 = (unsigned char *) JCALL2(GetByteArrayElements, jenv, $input, 0); +} + +// Map input argument: java short[] -> C++ STORAGEIMG16 +%typemap(jni) unsigned short* "jshortArray" +%typemap(jtype) unsigned short* "short[]" +%typemap(jstype) unsigned short* "short[]" +%typemap(in) STORAGEIMG16 +{ + long expectedLength = arg3; + long receivedLength = JCALL1(GetArrayLength, jenv, $input); + + if (receivedLength != expectedLength) + { + jclass excep = jenv->FindClass("java/lang/Exception"); + if (excep) + jenv->ThrowNew(excep, "Short array dimensions do not match declared size."); + return; + } + + $1 = (unsigned short *) JCALL2(GetShortArrayElements, jenv, $input, 0); +} +%typemap(in) unsigned short* +{ + long expectedLength = arg3; + long receivedLength = JCALL1(GetArrayLength, jenv, $input); + + if (receivedLength != expectedLength) + { + jclass excep = jenv->FindClass("java/lang/Exception"); + if (excep) + jenv->ThrowNew(excep, "Short array dimensions do not match declared size."); + return; + } + + $1 = (unsigned short *) JCALL2(GetShortArrayElements, jenv, $input, 0); +} + // Map input argument: java byte[] -> C++ unsigned char * %typemap(in) unsigned char* { @@ -337,13 +389,21 @@ // Allow the Java byte array to be garbage collected. JCALL3(ReleaseByteArrayElements, jenv, $input, (jbyte *) $1, JNI_ABORT); // JNI_ABORT = Don't alter the original array. } +%typemap(freearg) unsigned short* { + // Allow the Java byte array to be garbage collected. + JCALL3(ReleaseShortArrayElements, jenv, $input, (jshort *) $1, JNI_ABORT); // JNI_ABORT = Don't alter the original array. +} // change Java wrapper output mapping for unsigned char* %typemap(javaout) unsigned char* { return $jnicall; } + %typemap(javaout) unsigned short* { + return $jnicall; + } %typemap(javain) unsigned char* "$javainput" +%typemap(javain) unsigned short* "$javainput" // Map input argument: java List -> C++ std::vector @@ -510,6 +570,96 @@ } } +// Java typemap +// change default SWIG mapping of STORAGEIMG return values +// to return CObject containing array of pixel values +// +// Assumes that class has the following methods defined: +// std::vector getDatasetShape(handle) +// MM::StorageDataType getDatasetPixelType(handle) + +%typemap(jni) STORAGEIMGOUT "jobject" +%typemap(jtype) STORAGEIMGOUT "Object" +%typemap(jstype) STORAGEIMGOUT "Object" +%typemap(javaout) STORAGEIMGOUT { + return $jnicall; +} +%typemap(out) STORAGEIMGOUT +{ + std::vector shape = (arg1)->getDatasetShape(arg2); + MM::StorageDataType pixformat = (arg1)->getDatasetPixelType(arg2); + if(shape.size() < 2) + { + jclass excep = jenv->FindClass("java/lang/Exception"); + if(excep) + jenv->ThrowNew(excep, "Invalid dataset shape"); + } + long lSize = shape[shape.size() - 1] * shape[shape.size() - 2]; + + if(pixformat == MM::StorageDataType::StorageDataType_GRAY8) + { + // create a new byte[] object in Java + jbyteArray data = JCALL1(NewByteArray, jenv, lSize); + if (data == 0) + { + jclass excep = jenv->FindClass("java/lang/OutOfMemoryError"); + if (excep) + jenv->ThrowNew(excep, "The system ran out of memory!"); + + $result = 0; + return $result; + } + + // copy pixels from the image buffer + JCALL4(SetByteArrayRegion, jenv, data, 0, lSize, (jbyte*)result); + + $result = data; + } + else if(pixformat == MM::StorageDataType::StorageDataType_GRAY16) + { + // create a new short[] object in Java + jshortArray data = JCALL1(NewShortArray, jenv, lSize); + if (data == 0) + { + jclass excep = jenv->FindClass("java/lang/OutOfMemoryError"); + if (excep) + jenv->ThrowNew(excep, "The system ran out of memory!"); + $result = 0; + return $result; + } + + // copy pixels from the image buffer + JCALL4(SetShortArrayRegion, jenv, data, 0, lSize, (jshort*)result); + + $result = data; + } + else if(pixformat == MM::StorageDataType::StorageDataType_RGB32) + { + // create a new byte[] object in Java + jbyteArray data = JCALL1(NewByteArray, jenv, lSize * 4); + if(data == 0) + { + jclass excep = jenv->FindClass("java/lang/OutOfMemoryError"); + if (excep) + jenv->ThrowNew(excep, "The system ran out of memory!"); + + $result = 0; + return $result; + } + + // copy pixels from the image buffer + JCALL4(SetByteArrayRegion, jenv, data, 0, lSize * 4, (jbyte*)result); + + $result = data; + } + else + { + jclass excep = jenv->FindClass("java/lang/Exception"); + if(excep) + jenv->ThrowNew(excep, "Invalid dataset pixel format"); + } +} + // Java typemap // change default SWIG mapping of void* return values // to return CObject containing array of pixel values @@ -1220,6 +1370,7 @@ namespace std { %template(CharVector) vector; + %template(ByteVector) vector; %template(LongVector) vector; %template(DoubleVector) vector; %template(StrVector) vector; diff --git a/MMDevice/DeviceBase.h b/MMDevice/DeviceBase.h index 84cef7177..992039737 100644 --- a/MMDevice/DeviceBase.h +++ b/MMDevice/DeviceBase.h @@ -2480,6 +2480,17 @@ class CStateDeviceBase : public CDeviceBase std::map labels_; }; +/** +* Base class for creating Storage Device adapters. +*/ +template +class CStorageBase : public CDeviceBase +{ +public: + bool CanLoad(const char* /*path*/) { return false; } + int Freeze(int /*handle*/) { return DEVICE_UNSUPPORTED_COMMAND; } +}; + // _t, a macro for timing single lines. // This macros logs the text of the line, x, measures diff --git a/MMDevice/MMDevice.cpp b/MMDevice/MMDevice.cpp index 36c82ef97..94a62a182 100644 --- a/MMDevice/MMDevice.cpp +++ b/MMDevice/MMDevice.cpp @@ -47,5 +47,6 @@ const DeviceType Magnifier::Type = MagnifierDevice; const DeviceType SLM::Type = SLMDevice; const DeviceType Galvo::Type = GalvoDevice; const DeviceType Hub::Type = HubDevice; +const DeviceType Storage::Type = StorageDevice; } // namespace MM diff --git a/MMDevice/MMDevice.h b/MMDevice/MMDevice.h index c674ac0b7..a1adf4854 100644 --- a/MMDevice/MMDevice.h +++ b/MMDevice/MMDevice.h @@ -29,6 +29,7 @@ // If any of the class definitions changes, the interface version // must be incremented #define DEVICE_INTERFACE_VERSION 71 +// TODO: determine the correct version number /////////////////////////////////////////////////////////////////////////////// // N.B. @@ -204,6 +205,22 @@ namespace MM { MMTime interval_; // interval in milliseconds }; + inline int GetPixelDataSizeInBytes(StorageDataType dataType) + { + switch (dataType) + { + case StorageDataType_GRAY8: + return 1; + + case StorageDataType_GRAY16: + return 2; + + case StorageDataType_RGB32: + return 4; + } + + return 0; + } /** * Generic device interface. @@ -1428,4 +1445,310 @@ namespace MM { MM_DEPRECATED(virtual void ClearPostedErrors(void)) = 0; }; + /** + * \brief Device interface for managing multi-dimensional datasets in storage + * \details The Storage class provides an interface for creating, managing, and accessing + * multi-dimensional datasets. It supports operations such as creating new datasets, + * adding images, retrieving metadata, and managing dataset dimensions and coordinates. + * + * \note Implementation Notes: + * - All metadata variables are ASCII strings, typically JSON encoded, but not required + * - Functions returning metadata allocate buffers on the heap that must be released by the caller. + * - There are no explicit restrictions to image size, number of dimensions, size of dimensions and size of metadata + * - The same device may support opening of multiple datasets simultaneously, or only one dataset at a time. + * - The API allows for random access in image insertions and image retrieval, but the implementation may not support it. + * - Most formats allow only appending images in the order of dimensions. + * - Allowing random access to images (during retrievel) is recommended. + * - Lazy loading of pixel data is recommended for large datasets. + * + * \inherit Device + */ + class Storage : public Device { + public: + Storage() {} + virtual ~Storage() {} + + virtual DeviceType GetType() const { return Type; } + static const DeviceType Type; + + /** + * \brief Creates a new dataset + * + * \param handle Unique dataset handle assigned by MMCore + * \param path The path where the dataset will be created (parent directory) + * \param name The name of the dataset (may be modified by the implementation to avoid overwriting existing datasets) + * \param numberOfDimensions Number of dimensions in the dataset + * \param shape Array defining the size of each dimension + * \param pixType The data type for pixel storage + * \param meta Dataset metadata string + * \param metaLength length of the metadata + * \return Status code indicating success or failure + * + * \note - Dimensions are ordered from slowest changing to fastest changing. + * - Typically we are storing images and the last two dimensions are always image height (Y) and width (X) + * - Some implementation may allow non-image data to be stored + * - Declared size for the slowest (first) dimension can be exceeded during acquisition + */ + virtual int Create(int handle, const char* path, const char* name, int numberOfDimensions, const int shape[], + MM::StorageDataType pixType, const char* meta, int metaLength) = 0; + + /** + * \brief Retrieves the filesystem path of an opened dataset + * + * \param handle Dataset handle + * \param [out] path Buffer to store the dataset path + * \param maxPathLength Maximum length of the path buffer + * \return Status code indicating success or failure + */ + virtual int GetPath(int handle, char* path, int maxPathLength) = 0; + + /** + * \brief Configures a dimension's properties + * + * \param handle Dataset handle + * \param dimension The dimension index to configure + * \param name Name for the dimension + * \param meaning Semantic meaning of the dimension + * \return Status code indicating success or failure + * + * \note Recommended meanings: "T" for time, "Z" for focus, "C" for channel, "P" for position + * The last two dimensions should be "Y" and "X" if we are dealing with images + */ + virtual int ConfigureDimension(int handle, int dimension, + const char* name, const char* meaning) = 0; + + /** + * \brief Configures a coordinate's properties + * + * \param handle Dataset handle + * \param dimension The dimension index + * \param coordinate The coordinate index within the dimension + * \param name Name for the coordinate + * \return Status code indicating success or failure + */ + virtual int ConfigureCoordinate(int handle, int dimension, + int coordinate, const char* name) = 0; + + /** + * \brief Closes an opened dataset + * + * \param handle Dataset handle (becomes invalid after closing) + * \return Status code indicating success or failure + */ + virtual int Close(int handle) = 0; + + /** + * \brief Closes an opened dataset + * + * \param handle Dataset handle (becomes invalid after closing) + * \return Status code indicating success or failure + */ + virtual int Freeze(int handle) = 0; + + /** + * \brief Checks if a dataset is currently open + * + * \param handle Dataset handle + * \return true if dataset is open, false otherwise + */ + virtual bool IsOpen(int handle) = 0; + + /** + * \brief Checks if a dataset is read-only + * \details Loaded datasets are read-only and cannot accept new images. + * Only new datasets accept images, until they are closed. + * \param handle Dataset handle + * \return true if dataset is read-only, false if it can accept new images + */ + virtual bool IsReadOnly(int handle) = 0; + + /** + * \brief Loads an existing dataset + * \details Loaded datasets are immutable and cannot accept new images. + * Implementation may use lazy loading for efficiency. + * + * \param handle Unique dataset handle assigned by MMCore + * \param path Path to the dataset + * \return Status code indicating success or failure + */ + virtual int Load(int handle, const char* path) = 0; + + /** + * \brief Checks if the device can load a dataset at the specified path + * \details This will be used by the caller to quickly determine which driver to use + * for a given dataset path. The implementation should ideally be very fast + * and not open any files. + * \param path Path to the dataset + * \return true if the device can load the dataset, false otherwise + */ + virtual bool CanLoad(const char* path) = 0; + + /** + * \brief Deletes a dataset + * + * \details The dataset is permanently removed from storage. + * \param handle Handle of the dataset to delete + * \return Status code indicating success or failure + */ + virtual int Delete(int handle) = 0; + + /** + * \brief Lists datasets in a specified path + * + * \param path Directory path to search for known datasets + * \param [out] listOfDatasets Array of dataset names, each with a maximum length of maxItemLength + * \param maxItems Maximum number of items to return + * \param maxItemLength Maximum length of each item name + * \return Status code indicating success or failure + */ + virtual int List(const char* path, char** listOfDatasets, int maxItems, int maxItemLength) = 0; + + /** + * \brief Inserts an image at specific coordinates + * + * \param handle Dataset handle + * \param sizeInBytes Size of the image data in bytes + * \param pixels Pointer to the image pixel data + * \param coordinates Array of coordinates for image insertion + * \param numCoordinates Number of coordinate values + * \param imageMeta Image metadata string + * \param imageMetaLength length of the image metadata + * \return Status code indicating success or failure + */ + virtual int AddImage(int handle, int sizeInBytes, unsigned char* pixels, + int coordinates[], int numCoordinates, const char* imageMeta, int imageMetaLength) = 0; + + /** + * \brief Appends an image to the dataset + * \param handle Dataset handle + * \param sizeInBytes Size of the image data in bytes + * \param pixels Pointer to the image pixel data + * \param imageMeta Image metadata string + * \param imageMetaLength length of the image metadata + * \return Status code indicating success or failure + */ + virtual int AppendImage(int handle, int sizeInBytes, unsigned char* pixels, const char* imageMeta, int imageMetaLength) = 0; + + /** + * \brief Retrieves dataset summary metadata + * + * \param handle Dataset handle + * \param [out] meta Buffer for metadata string allocated by the implementation + * \return Status code indicating success or failure + * + * \note Caller must release the metadata buffer using ReleaseStringBuffer() + */ + virtual int GetSummaryMeta(int handle, char** meta) = 0; + + /** + * \brief Retrieves metadata for a specific image + * + * \param handle Dataset handle + * \param coordinates Array of coordinates identifying the image + * \param numCoordinates Number of coordinate values + * \param [out] meta Buffer for metadata string + * \return Status code indicating success or failure + * + * \note Caller must release the metadata buffer using ReleaseStringBuffer() + */ + virtual int GetImageMeta(int handle, int coordinates[], int numCoordinates, char** meta) = 0; + + /** + * \brief Retrieves image pixel data + * \param handle Dataset handle + * \param coordinates Array of coordinates identifying the image + * \param numCoordinates Number of coordinate values + * \return Pointer to the image pixel data + */ + virtual const unsigned char* GetImage(int handle, int coordinates[], + int numCoordinates) = 0; + + /** + * \brief Gets the number of dimensions in the dataset + * \param handle Dataset handle + * \param [out] numDimensions Number of dimensions + * \return Status code indicating success or failure + */ + virtual int GetNumberOfDimensions(int handle, int& numDimensions) = 0; + + /** + * \brief Gets the shape of the dataset + * \param handle Dataset handle + * \param [out] shape Array to store dimension sizes, allocated by the caller + * \return Status code indicating success or failure + */ + virtual int GetShape(int handle, int shape[]) = 0; + + /** + * \brief Gets the pixel data type of the dataset + * \param handle Dataset handle + * \param [out] pixelDataType Data type enumeration value + * \return Status code indicating success or failure + */ + virtual int GetDataType(int handle, MM::StorageDataType& pixelDataType) = 0; + + /** + * \brief Gets information about a specific dimension + * \param handle Dataset handle + * \param dimension Dimension index + * \param [out] name Buffer for dimension name + * \param nameLength Maximum length of name buffer + * \param [out] meaning Buffer for dimension meaning + * \param meaningLength Maximum length of meaning buffer + * \return Status code indicating success or failure + */ + virtual int GetDimension(int handle, int dimension, char* name, int nameLength, char* meaning, int meaningLength) = 0; + + /** + * \brief Gets information about a specific coordinate + * + * \param handle Dataset handle + * \param dimension Dimension index + * \param coordinate Coordinate index + * \param [out] name Buffer for coordinate name + * \param nameLength Maximum length of name buffer + * \return Status code indicating success or failure + */ + virtual int GetCoordinate(int handle, int dimension, int coordinate, char* name, int nameLength) = 0; + + /** + * \brief Gets the total number of images in the dataset + * \note The number of images actually stored may not be equal to the product of the dimension sizes + * + * \param handle Dataset handle + * \param [out] imgcount Number of images + * \return Status code indicating success or failure + */ + virtual int GetImageCount(int handle, int& imgcount) = 0; + + /** + * \brief Sets custom metadata for the dataset + * \details This metadata is mutable and can be updated at any time + * \param handle Dataset handle + * \param key Metadata key + * \param content Metadata content. Unlimited size. + * \param contentLength length of the metadata string + * \return Status code indicating success or failure + */ + virtual int SetCustomMetadata(int handle, const char* key, const char* content, int contentLength) = 0; + + /** + * \brief Retrieves custom metadata from the dataset + * \param handle Dataset handle + * \param key Metadata key + * \param [out] content Buffer for metadata content, allocated by the implementation + * \return Status code indicating success or failure + * \note Caller must release the content buffer using ReleaseStringBuffer() + */ + virtual int GetCustomMetadata(int handle, const char* key, char** content) = 0; + + /** \brief Releases allocated string buffers + * \details Must be called to free memory allocated by GetSummaryMeta, + * GetImageMeta, and GetCustomMetadata + * \param buffer Pointer to the allocated buffer + */ + virtual void ReleaseStringBuffer(char* buffer) = 0; + }; + + } // namespace MM diff --git a/MMDevice/MMDeviceConstants.h b/MMDevice/MMDeviceConstants.h index 65b0a5c90..5b80dd242 100644 --- a/MMDevice/MMDeviceConstants.h +++ b/MMDevice/MMDeviceConstants.h @@ -96,6 +96,9 @@ namespace MM { // Code filling buffer should assume includes null terminator. const int MaxStrLength = 1024; + // Maximum length of serialized metadata strings + const int MaxMetadataLength = 200000; + // system-wide property names const char* const g_Keyword_Name = "Name"; const char* const g_Keyword_Description = "Description"; @@ -143,6 +146,7 @@ namespace MM { const char* const g_Keyword_CoreImageProcessor = "ImageProcessor"; const char* const g_Keyword_CoreSLM = "SLM"; const char* const g_Keyword_CoreGalvo = "Galvo"; + const char* const g_Keyword_CoreStorage = "Storage"; const char* const g_Keyword_CoreTimeoutMs = "TimeoutMs"; const char* const g_Keyword_Channel = "Channel"; const char* const g_Keyword_Version = "Version"; @@ -242,7 +246,8 @@ namespace MM { MagnifierDevice, SLMDevice, HubDevice, - GalvoDevice + GalvoDevice, + StorageDevice }; enum PropertyType { @@ -275,6 +280,15 @@ namespace MM { FocusDirectionAwayFromSample, }; + enum StorageDataType + { + StorageDataType_UNKNOWN = 0, + StorageDataType_GRAY8, // uint8 + StorageDataType_GRAY16, // uint16 + StorageDataType_RGB32 // RGBA, 8bits per color, A ignored defaults to 0 + }; + + ////////////////////////////////////////////////////////////////////////////// // Notification constants // diff --git a/Tests/G2SStorageTest/G2SAcqTest.cpp b/Tests/G2SStorageTest/G2SAcqTest.cpp new file mode 100644 index 000000000..9857c2490 --- /dev/null +++ b/Tests/G2SStorageTest/G2SAcqTest.cpp @@ -0,0 +1,128 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SAcqTest.cpp +// PROJECT: Micro-Manager +// SUBSYSTEM: Device Driver Tests +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope storage driver acquisition test +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include +#include +#include +#include "MMCore.h" + +extern std::string generateImageMeta(CMMCore& core, int imgind); + +/** + * Storage acquisition test + * @param core MM Core instance + * @param path Data folder path + * @param name Dataset name + * @param c Channel count + * @param t Time points + * @param p Positions count + * @throws std::runtime_error + */ +void testAcquisition(CMMCore& core, const std::string& path, const std::string& name, int c, int t, int p) +{ + std::cout << std::endl << "Starting G2SStorage driver acquisition test" << std::endl; + + // Take one image to "warm up" the camera and get actual image dimensions + core.snapImage(); + int w = (int)core.getImageWidth(); + int h = (int)core.getImageHeight(); + int imgSize = 2 * w * h; + double imgSizeMb = (double)imgSize / (1024.0 * 1024.0); + + // Shape convention: Z, T, C, Y, X + std::vector shape = { p, t, c, h, w }; + auto handle = core.createDataset(path.c_str(), name.c_str(), shape, MM::StorageDataType_GRAY16, "", 0); + + std::cout << "Dataset handle: " << handle << std::endl; + std::cout << "Dataset shape (P-T-C-H-W): " << p << " x " << t << " x " << c << " x " << h << " x " << w << " x 16-bit" << std::endl << std::endl; + std::cout << "START OF ACQUISITION" << std::endl; + + // Start acquisition + auto start = std::chrono::high_resolution_clock::now(); + core.startSequenceAcquisition(c * t * p, 0.0, true); + + int imgind = 0; + auto startAcq = start; + for(int i = 0; i < p; i++) + { + for(int j = 0; j < t; j++) + { + for(int k = 0; k < c; k++) + { + if(core.isBufferOverflowed()) + throw std::runtime_error("Buffer overflow!!"); + + while(core.getRemainingImageCount() == 0) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); // wait for images to become available + + // Reset acquisition timer when the first image becomes available) + if(start == startAcq) + startAcq = std::chrono::high_resolution_clock::now(); + + // fetch the image + unsigned char* img = reinterpret_cast(core.popNextImage()); + + // Generate image metadata + std::string meta = generateImageMeta(core, imgind); + + // Add image to the stream + auto startSave = std::chrono::high_resolution_clock::now(); + core.appendImageToDataset(handle, imgSize, img, meta.c_str(), (int)meta.size()); + auto endSave = std::chrono::high_resolution_clock::now(); + + // Calculate statistics + double imgSaveTimeMs = (endSave - startSave).count() / 1000000.0; + double bw = imgSizeMb / (imgSaveTimeMs / 1000.0); + std::cout << "Saved image " << imgind++ << " in "; + std::cout << std::fixed << std::setprecision(2) << imgSaveTimeMs << " ms, size "; + std::cout << std::fixed << std::setprecision(1) << imgSizeMb << " MB, BW: " << bw << " MB/s" << std::endl; + } + } + } + + // We are done so close the dataset + core.stopSequenceAcquisition(); + core.closeDataset(handle); + auto end = std::chrono::high_resolution_clock::now(); + std::cout << "END OF ACQUISITION" << std::endl << std::endl; + + // Calculate storage driver bandwidth + double totalTimeS = (end - start).count() / 1000000000.0; + double prepTimeS = (startAcq - start).count() / 1000000000.0; + double acqTimeS = (end - startAcq).count() / 1000000000.0; + double totalSizemb = (double)imgSize * p * t * c / (1024.0 * 1024.0); + double totbw = totalSizemb / totalTimeS; + double acqbw = totalSizemb / acqTimeS; + std::cout << std::fixed << std::setprecision(1) << "Dataset size " << totalSizemb << " MB" << std::endl; + std::cout << std::fixed << std::setprecision(3) << "Camera prep time: " << prepTimeS << " sec" << std::endl; + std::cout << std::fixed << std::setprecision(3) << "Active acquisition time: " << acqTimeS << " sec" << std::endl; + std::cout << std::fixed << std::setprecision(1) << "Storage driver bandwidth " << acqbw << " MB/s" << std::endl << std::endl; + std::cout << std::fixed << std::setprecision(3) << "Acquisition completed in " << totalTimeS << " sec" << std::endl; + std::cout << std::fixed << std::setprecision(1) << "Acquisition bandwidth " << totbw << " MB/s" << std::endl; +} \ No newline at end of file diff --git a/Tests/G2SStorageTest/G2SReaderTest.cpp b/Tests/G2SStorageTest/G2SReaderTest.cpp new file mode 100644 index 000000000..eb5f415f6 --- /dev/null +++ b/Tests/G2SStorageTest/G2SReaderTest.cpp @@ -0,0 +1,144 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SReaderTest.cpp +// PROJECT: Micro-Manager +// SUBSYSTEM: Device Driver Tests +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope storage driver reader test +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include +#include +#include +#include "MMCore.h" + +/** + * Calculate image coordinates for optimized access + * @param ind Image index + * @param shape Dataset shape + * @param coords Image coordinates [out] + */ +std::vector calcCoordsOptimized(long ind, const std::vector& shape) +{ + std::vector ret(shape.size()); + int fx = 0; + for(int j = 0; j < (int)shape.size() - 2; j++) { + int sum = 1; + for(int k = j + 1; k < (int)shape.size() - 2; k++) + sum *= shape[k]; + int ix = (ind - fx) / sum; + ret[j] = ix; + fx += ix * sum; + } + return ret; +} + +/** + * Calculate image coordinates for random access + * @param ind Image index + * @param shape Dataset shape + * @param coords Image coordinates [out] + */ +std::vector calcCoordsRandom(long ind, const std::vector& shape) +{ + std::vector ret(shape.size()); + int fx = 0; + for(int j = (int)shape.size() - 3; j >= 0; j--) { + int sum = 1; + for(int k = 0; k < j; k++) + sum *= shape[k]; + int ix = (ind - fx) / sum; + ret[j] = ix; + fx += ix * sum; + } + return ret; +} + +/** + * Storage read test + * @param core MM Core instance + * @param path Data folder path + * @param name Dataset name + * @param optimized Optimized / random access test + * @param printmeta Print image metadata + * @throws std::runtime_error + */ +void testReader(CMMCore& core, const std::string& path, const std::string& name, bool optimized, bool printmeta) +{ + std::cout << std::endl << "Starting G2SStorage driver reader test" << std::endl; + std::filesystem::path ds = std::filesystem::u8path(path) / name; + + // Load the dataset + auto start = std::chrono::high_resolution_clock::now(); + auto handle = core.loadDataset(ds.u8string().c_str()); + auto loadEnd = std::chrono::high_resolution_clock::now(); + double loadTimeS = (loadEnd - start).count() / 1000000000.0; + + // Obtain dataset shape + auto shape = core.getDatasetShape(handle); + auto ptype = core.getDatasetPixelType(handle); + auto imgcnt = shape[0] * shape[1] * shape[2]; + auto imgSize = shape[3] * shape[4] * (ptype == MM::StorageDataType_GRAY16 ? 2 : 1); + double imgSizeMb = (double)imgSize / (1024.0 * 1024.0); + double totalSizeMb = (double)imgSize * imgcnt / (1024.0 * 1024.0); + std::cout << std::fixed << std::setprecision(3) << "Dataset loaded in " << loadTimeS << " sec, contains " << imgcnt << " images" << std::endl; + std::cout << "Dataset handle: " << handle << std::endl; + std::cout << "Dataset shape (P-T-C-H-W): " << shape[0] << " x " << shape[1] << " x " << shape[2] << " x " << shape[3] << " x " << shape[4] << " x " << (ptype == MM::StorageDataType_GRAY16 ? 16 : 8) << "-bit" << std::endl << std::endl; + + // Read images + for(long i = 0; i < imgcnt; i++) + { + // Calculate coordinates + auto coords = optimized ? calcCoordsOptimized(i, shape) : calcCoordsRandom(i, shape); + + // Read image from the file stream + auto startRead = std::chrono::high_resolution_clock::now(); + auto img = core.getImageFromDataset(handle, coords); + auto emdRead = std::chrono::high_resolution_clock::now(); + if(img == nullptr) + throw std::runtime_error("Failed to fetch image " + i); + double readTimeMs = (emdRead - startRead).count() / 1000000.0; + + double bw = imgSizeMb / (readTimeMs / 1000.0); + std::cout << "Image " << std::setw(3) << i << " ["; + for(std::size_t i = 0; i < coords.size(); i++) + std::cout << (i == 0 ? "" : ", ") << coords[i]; + std::cout << "], size: " << std::fixed << std::setprecision(1) << imgSizeMb << " MB in " << readTimeMs << " ms (" << bw << " MB/s)" << std::endl; + + auto meta = core.getDatasetImageMeta(handle, coords); + if(printmeta) + std::cout << "Image metadata: " << meta << std::endl; + } + + // We are done so close the dataset + core.closeDataset(handle); + auto end = std::chrono::high_resolution_clock::now(); + std::cout << std::endl; + + // Calculate storage driver bandwidth + double totalTimeS = (end - start).count() / 1000000000.0; + double bw = totalSizeMb / totalTimeS; + std::cout << std::fixed << std::setprecision(3) << "Read completed in " << totalTimeS << " sec" << std::endl; + std::cout << std::fixed << std::setprecision(1) << "Dataset size " << totalSizeMb << " MB" << std::endl; + std::cout << std::fixed << std::setprecision(1) << "Storage driver bandwidth " << bw << " MB/s" << std::endl; +} \ No newline at end of file diff --git a/Tests/G2SStorageTest/G2SStorageTest.cpp b/Tests/G2SStorageTest/G2SStorageTest.cpp new file mode 100644 index 000000000..a7f3961c0 --- /dev/null +++ b/Tests/G2SStorageTest/G2SStorageTest.cpp @@ -0,0 +1,286 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SStorageTest.cpp +// PROJECT: Micro-Manager +// SUBSYSTEM: Device Driver Tests +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope storage driver test suite +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include "MMCore.h" + +#define TEST_WRITE 1 +#define TEST_READ 2 +#define TEST_ACQ 3 +#define TEST_VER "1.0.0" +#define ENGINE_BIGTIFF 1 +#define ENGINE_ZARR 2 +#define CAMERA_DEMO 1 +#define CAMERA_HAMAMATSU 2 + +extern void testWritter(CMMCore& core, const std::string& path, const std::string& name, int c, int t, int p); +extern void testReader(CMMCore& core, const std::string& path, const std::string& name, bool optimized, bool printmeta); +extern void testAcquisition(CMMCore& core, const std::string& path, const std::string& name, int c, int t, int p); + +/** + * Application entry point + * @param argc Argument count + * @param argv Argument list + * @return Status code + */ +int main(int argc, char** argv) +{ + std::cout << "Starting G2SStorage driver test suite..." << std::endl; + int selectedTest = TEST_ACQ; + int storageEngine = ENGINE_BIGTIFF; + int selectedcamera = CAMERA_DEMO; + int flushcycle = 0; + int channels = 4; + int timepoints = 8; + int positions = 5; + int cbuffsize = 16384; + bool directIO = false; + bool printmeta = false; + bool optimalaccess = false; + std::string savelocation = "."; + + // Parse input arguments + if(argc < 2) + { + std::cout << "Invalid arguments specified. To see program options type G2SStorageTest -help" << std::endl; + return 2; + } + + // Obtain selected test + std::string carg(argv[1]); + std::transform(carg.begin(), carg.end(), carg.begin(), [](char c) { return std::tolower(c); }); + if(carg == "-v") + { + std::cout << "G2SStorageTest " << TEST_VER << std::endl; + return 0; + } + else if(carg == "-help") + { + std::cout << "Available test suites: write, read, acq" << std::endl << std::endl; + std::cout << "For write test type:" << std::endl; + std::cout << "G2SStorageTest write [storage_engine] [save_location] [camera] [channel_count] [time_points_count] [positions_count] [direct_io] [flush_cycle]" << std::endl << std::endl; + + std::cout << "For read test type:" << std::endl; + std::cout << "G2SStorageTest read [storage_engine] [save_location] [dataset_name] [direct_io] [optimal_access] [print_meta]" << std::endl << std::endl; + + std::cout << "For acquisition test type:" << std::endl; + std::cout << "G2SStorageTest acq [storage_engine] [save_location] [camera] [channel_count] [time_points_count] [positions_count] [direct_io] [flush_cycle]" << std::endl << std::endl; + + std::cout << "Available storage engines: zarr, bigtiff (default)" << std::endl; + std::cout << "Available cameras: demo (default), hamamatsu" << std::endl; + std::cout << "The following options are basic ON/OFF (1/0) flags: [direct_io] [optimal_access] [print_meta]" << std::endl; + std::cout << "Default save location is the current directory" << std::endl; + std::cout << "Default channel count is " << channels << std::endl; + std::cout << "Default time points count is " << timepoints << std::endl; + std::cout << "Default positions count is " << positions << std::endl; + std::cout << "Default camera is " << selectedcamera << std::endl; + std::cout << "Default DirectIO status is " << directIO << std::endl; + std::cout << "Default access type is " << optimalaccess << std::endl; + std::cout << "By default metadata is not printed on the command line" << std::endl; + std::cout << "Default flush cycle is 0 (file stream is flushed during closing)" << std::endl; + std::cout << "Default dataset name is test-[storage_engine]" << std::endl; + return 0; + } + else if(carg == "read" || carg == "write" || carg == "acq") + selectedTest = carg == "write" ? TEST_WRITE : (carg == "read" ? TEST_READ : TEST_ACQ); + else + { + std::cout << "Invalid test suite selected. To see program options type G2SStorageTest -help" << std::endl; + return 2; + } + + // Obtain storage engine + if(argc > 2) + { + carg = std::string(argv[2]); + std::transform(carg.begin(), carg.end(), carg.begin(), [](char c) { return std::tolower(c); }); + if(carg == "bigtiff" || carg == "zarr") + storageEngine = carg == "bigtiff" ? ENGINE_BIGTIFF : ENGINE_ZARR; + else + { + std::cout << "Invalid storage engine selected. To see program options type G2SStorageTest -help" << std::endl; + return 2; + } + } + std::string datasetname = "test-" + std::string(storageEngine == ENGINE_BIGTIFF ? "bigtiff" : "zarr"); + + // Obtain save location + if(argc > 3) + { + carg = std::string(argv[3]); + std::filesystem::path spath = std::filesystem::u8path(carg); + if(std::filesystem::exists(spath) && !std::filesystem::is_directory(spath)) + { + std::cout << "Invalid save location. To see program options type G2SStorageTest -help" << std::endl; + return 2; + } + savelocation = std::filesystem::absolute(spath).u8string(); + } + + // Obtain dataset name (READ test) + if(argc > 4 && selectedTest == TEST_READ) + datasetname = std::string(argv[4]); + // Obtain camera type (WRITE, ACQ tests) + else if(argc > 4) + { + carg = std::string(argv[4]); + std::transform(carg.begin(), carg.end(), carg.begin(), [](char c) { return std::tolower(c); }); + if(carg == "demo" || carg == "hamamatsu") + selectedcamera = carg == "demo" ? CAMERA_DEMO : CAMERA_HAMAMATSU; + else + { + std::cout << "Invalid camera selected. To see program options type G2SStorageTest -help" << std::endl; + return 2; + } + } + + // Obtain I/O type (READ test) + if(argc > 5 && selectedTest == TEST_READ) + try { directIO = std::stoi(argv[5]) != 0; } catch(std::exception& e) { std::cout << "Invalid argument value. " << e.what() << std::endl; return 1; } + // Obtain channel count (WRITE, ACQ tests) + else if(argc > 5) + try { channels = (int)std::stoul(argv[5]); } catch(std::exception& e) { std::cout << "Invalid argument value. " << e.what() << std::endl; return 1; } + + // Obtain access type (READ test) + if(argc > 6 && selectedTest == TEST_READ) + try { optimalaccess = std::stoi(argv[6]) != 0; } catch(std::exception& e) { std::cout << "Invalid argument value. " << e.what() << std::endl; return 1; } + // Obtain time points (WRITE, ACQ tests) + else if(argc > 6) + try { timepoints = (int)std::stoul(argv[6]); } catch(std::exception& e) { std::cout << "Invalid argument value. " << e.what() << std::endl; return 1; } + + // Obtain print metadata flag (READ test) + if(argc > 7 && selectedTest == TEST_READ) + try { printmeta = std::stoi(argv[7]) != 0; } catch(std::exception& e) { std::cout << "Invalid argument value. " << e.what() << std::endl; return 1; } + // Obtain positions (WRITE, ACQ tests) + else if(argc > 7) + try { positions = (int)std::stoul(argv[7]); } catch(std::exception& e) { std::cout << "Invalid argument value. " << e.what() << std::endl; return 1; } + + // Obtain I/O type (WRITE, ACQ tests) + if(argc > 8 && selectedTest != TEST_READ) + try { directIO = std::stoi(argv[8]) != 0; } catch(std::exception& e) { std::cout << "Invalid argument value. " << e.what() << std::endl; return 1; } + + // Obtain flush cycle (WRITE, ACQ tests) + if(argc > 9 && selectedTest != TEST_READ) + try { flushcycle = (int)std::stoul(argv[9]) != 0; } catch(std::exception& e) { std::cout << "Invalid argument value. " << e.what() << std::endl; return 1; } + + // Print configuration + std::cout << "Data location " << savelocation << std::endl; + std::cout << "Dataset name " << datasetname << std::endl << std::endl; + if(selectedTest != TEST_READ) + { + std::cout << "Circular buffer size: " << cbuffsize << " MB" << std::endl; + std::cout << "Camera " << (selectedcamera == CAMERA_DEMO ? "DEMO" : "HAMAMATSU") << std::endl; + } + std::cout << "Direct I/O " << (directIO ? "YES" : "NO") << std::endl; + std::cout << "Flush Cycle " << flushcycle << std::endl; + std::cout << "Optimized access " << (optimalaccess ? "YES" : "NO") << std::endl << std::endl; + try + { + // Initialize core + CMMCore core; +#ifdef NDEBUG + core.enableDebugLog(false); + core.enableStderrLog(false); +#else + core.enableDebugLog(true); + core.enableStderrLog(true); +#endif + + + // Load storage driver + std::cout << "Loading storage driver..." << std::endl; + if(storageEngine == ENGINE_ZARR) + core.loadDevice("Store", "AcquireZarr", "AcquireZarrStorage"); + else + core.loadDevice("Store", "Go2Scope", "G2SBigTiffStorage"); + + // Load camera driver + if(selectedTest != TEST_READ) + { + // Set circular buffer size + std::cout << "Setting buffer size..." << std::endl; + core.setCircularBufferMemoryFootprint(cbuffsize); + core.clearCircularBuffer(); + + std::cout << "Loading camera driver..." << std::endl; + if(selectedcamera == CAMERA_DEMO) + core.loadDevice("Camera", "DemoCamera", "DCam"); + else + core.loadDevice("Camera", "HamamatsuHam", "HamamatsuHam_DCAM"); + } + + // initialize the system, this will in turn initialize each device + std::cout << "Initializing device drivers..." << std::endl; + core.initializeAllDevices(); + + // Configure camera + if(selectedTest != TEST_READ) + { + std::cout << "Setting camera configuration..." << std::endl; + if(selectedcamera == CAMERA_DEMO) + { + // Configure demo camera + core.setProperty("Camera", "PixelType", "16bit"); + core.setProperty("Camera", "OnCameraCCDXSize", "4432"); + core.setProperty("Camera", "OnCameraCCDYSize", "2368"); + core.setExposure(10.0); + } + else + { + // Configure HamamatsuHam + core.setProperty("Camera", "PixelType", "16bit"); + core.setROI(1032, 0, 2368, 2368); + core.setExposure(5.0); + } + } + + // Set storage engine properties + if(storageEngine == ENGINE_BIGTIFF) { + std::cout << "Setting BigTIFF storage configuration..." << std::endl; + core.setProperty("Store", "DirectIO", directIO ? 1L : 0L); + core.setProperty("Store", "FlushCycle", (long)flushcycle); + } + + // Run test + if(selectedTest == TEST_WRITE) + testWritter(core, savelocation, datasetname, channels, timepoints, positions); + else if(selectedTest == TEST_READ) + testReader(core, savelocation, datasetname, optimalaccess, printmeta); + else if(selectedTest == TEST_ACQ) + testAcquisition(core, savelocation, datasetname, channels, timepoints, positions); + else + std::cout << "Invalid test suite selected. Exiting..." << std::endl; + + core.unloadAllDevices(); + } + catch(std::exception& e) + { + std::cout << "Error: " << e.what() << std::endl; + return 1; + } +} diff --git a/Tests/G2SStorageTest/G2SStorageTest.vcxproj b/Tests/G2SStorageTest/G2SStorageTest.vcxproj new file mode 100644 index 000000000..aa54c8431 --- /dev/null +++ b/Tests/G2SStorageTest/G2SStorageTest.vcxproj @@ -0,0 +1,96 @@ + + + + + Debug + x64 + + + Release + x64 + + + + 17.0 + Win32Proj + {c87f7284-62ef-4331-9444-9951db12a57a} + G2SStorageTest + 10.0 + + + + Application + true + v142 + Unicode + + + Application + false + v142 + true + Unicode + + + + + + + + + + + + + + + + + + Level3 + true + _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp17 + ..\..\MMCore;%(AdditionalIncludeDirectories) + + + Console + true + MMCore.lib;$(CoreLibraryDependencies);%(AdditionalDependencies) + $(MM_BUILDDIR)\$(Configuration)\$(Platform)\;%(AdditionalLibraryDirectories) + + + + + Level3 + true + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp17 + ..\..\MMCore;%(AdditionalIncludeDirectories) + + + Console + true + true + true + MMCore.lib;$(CoreLibraryDependencies);%(AdditionalDependencies) + $(MM_BUILDDIR)\$(Configuration)\$(Platform)\;%(AdditionalLibraryDirectories) + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tests/G2SStorageTest/G2SStorageTest.vcxproj.filters b/Tests/G2SStorageTest/G2SStorageTest.vcxproj.filters new file mode 100644 index 000000000..7aacf8d41 --- /dev/null +++ b/Tests/G2SStorageTest/G2SStorageTest.vcxproj.filters @@ -0,0 +1,34 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/Tests/G2SStorageTest/G2SWriterTest.cpp b/Tests/G2SStorageTest/G2SWriterTest.cpp new file mode 100644 index 000000000..c8423b1fb --- /dev/null +++ b/Tests/G2SStorageTest/G2SWriterTest.cpp @@ -0,0 +1,108 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: G2SWriterTest.cpp +// PROJECT: Micro-Manager +// SUBSYSTEM: Device Driver Tests +//----------------------------------------------------------------------------- +// DESCRIPTION: Go2Scope storage driver writer test +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include +#include +#include "MMCore.h" + +extern std::string generateImageMeta(CMMCore& core, int imgind); + +/** + * Storage writer test + * @param core MM Core instance + * @param path Data folder path + * @param name Dataset name + * @param c Channel count + * @param t Time points + * @param p Positions count + * @throws std::runtime_error + */ +void testWritter(CMMCore& core, const std::string& path, const std::string& name, int c, int t, int p) +{ + std::cout << std::endl << "Starting G2SStorage driver writer test" << std::endl; + + // Take one image to "warm up" the camera and get actual image dimensions + core.snapImage(); + int w = (int)core.getImageWidth(); + int h = (int)core.getImageHeight(); + int imgSize = 2 * w * h; + double imgSizeMb = (double)imgSize / (1024.0 * 1024.0); + + // Shape convention: Z, T, C, Y, X + std::vector shape = { p, t, c, h, w }; + auto handle = core.createDataset(path.c_str(), name.c_str(), shape, MM::StorageDataType_GRAY16, "", 0); + + std::cout << "Dataset handle: " << handle << std::endl; + std::cout << "Dataset shape (P-T-C-H-W): " << p << " x " << t << " x " << c << " x " << h << " x " << w << " x 16-bit" << std::endl << std::endl; + std::cout << "START OF ACQUISITION" << std::endl; + int imgind = 0; + auto start = std::chrono::high_resolution_clock::now(); + for(int i = 0; i < p; i++) + { + for(int j = 0; j < t; j++) + { + for(int k = 0; k < c; k++) + { + // Snap an image + core.snapImage(); + + // Fetch the image + unsigned char* img = reinterpret_cast(core.getImage()); + + // Generate image metadata + auto meta = generateImageMeta(core, imgind); + + // Add image to the stream + auto startSave = std::chrono::high_resolution_clock::now(); + core.appendImageToDataset(handle, imgSize, img, meta.c_str(), meta.size()); + auto endSave = std::chrono::high_resolution_clock::now(); + + // Calculate statistics + double imgSaveTimeMs = (endSave - startSave).count() / 1000000.0; + double bw = imgSizeMb / (imgSaveTimeMs / 1000.0); + std::cout << "Saved image " << imgind++ << " in "; + std::cout << std::fixed << std::setprecision(2) << imgSaveTimeMs << " ms, size "; + std::cout << std::fixed << std::setprecision(1) << imgSizeMb << " MB, BW: " << bw << " MB/s" << std::endl; + } + } + } + + // We are done so close the dataset + core.closeDataset(handle); + auto end = std::chrono::high_resolution_clock::now(); + std::cout << "END OF ACQUISITION" << std::endl << std::endl; + + // Calculate storage driver bandwidth + double totalTimeS = (end - start).count() / 1000000000.0; + double totalSizemb = (double)imgSize * p * t * c / (1024.0 * 1024.0); + double bw = totalSizemb / totalTimeS; + std::cout << std::fixed << std::setprecision(3) << "Acquisition completed in " << totalTimeS << " sec" << std::endl; + std::cout << std::fixed << std::setprecision(1) << "Dataset size " << totalSizemb << " MB" << std::endl; + std::cout << std::fixed << std::setprecision(1) << "Storage driver bandwidth " << bw << " MB/s" << std::endl; +} \ No newline at end of file diff --git a/Tests/G2SStorageTest/Util.cpp b/Tests/G2SStorageTest/Util.cpp new file mode 100644 index 000000000..6dc4aec5c --- /dev/null +++ b/Tests/G2SStorageTest/Util.cpp @@ -0,0 +1,112 @@ +/////////////////////////////////////////////////////////////////////////////// +// FILE: Util.h +// PROJECT: Micro-Manager +// SUBSYSTEM: Device Driver Tests +//----------------------------------------------------------------------------- +// DESCRIPTION: Helper methods +// +// AUTHOR: Milos Jovanovic +// +// COPYRIGHT: Nenad Amodaj, Chan Zuckerberg Initiative, 2024 +// +// LICENSE: This file is distributed under the BSD license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. +// +// NOTE: Storage Device development is supported in part by +// Chan Zuckerberg Initiative (CZI) +// +/////////////////////////////////////////////////////////////////////////////// +#include +#include "MMCore.h" + +/** + * Generate image metadata + * @param core MM Core instance + * @param imgind Image index + * @return Image metadata (JSON) + */ +std::string generateImageMeta(CMMCore& core, int imgind) +{ + // Calculate pixel type + std::string pixtype = ""; + auto depth = core.getBytesPerPixel(); + auto numComponents = core.getNumberOfComponents(); + switch (depth) { + case 1: + pixtype = "GRAY8"; + break; + case 2: + pixtype = "GRAY16"; + break; + case 4: + if(numComponents == 1) + pixtype = "GRAY32"; + else + pixtype = "RGB32"; + break; + case 8: + pixtype = "RGB64"; + break; + default: + break; + } + + // Calculate ROI + int x = 0, y = 0, w = 0, h = 0; + core.getROI(x, y, w, h); + std::string roi = std::to_string(x) + "-" + std::to_string(y) + "-" + std::to_string(w) + "-" + std::to_string(h); + + // Calculate ROI affine transform + auto aff = core.getPixelSizeAffine(true); + std::string psizeaffine = ""; + if(aff.size() == 6) + { + for(int i = 0; i < 5; ++i) + psizeaffine += std::to_string(aff[i]) + ";"; + psizeaffine += std::to_string(aff[5]); + } + + // Write JSON + std::stringstream ss; + ss << "{"; + Configuration config = core.getSystemStateCache(); + for(int i = 0; (long)i < config.size(); ++i) + { + PropertySetting setting = config.getSetting((long)i); + auto key = setting.getDeviceLabel() + "-" + setting.getPropertyName(); + auto value = setting.getPropertyValue(); + ss << "\"" << key << "\":\"" << value << "\","; + } + ss << "\"BitDepth\":" << core.getImageBitDepth() << ","; + ss << "\"PixelSizeUm\":" << core.getPixelSizeUm(true) << ","; + ss << "\"PixelSizeAffine\":\"" << psizeaffine << "\","; + ss << "\"ROI\":\"" << roi << "\","; + ss << "\"Width\":" << core.getImageWidth() << ","; + ss << "\"Height\":" << core.getImageHeight() << ","; + ss << "\"PixelType\":\"" << pixtype << "\","; + ss << "\"Frame\":0,"; + ss << "\"FrameIndex\":0,"; + ss << "\"Position\":\"Default\","; + ss << "\"PositionIndex\":0,"; + ss << "\"Slice\":0,"; + ss << "\"SliceIndex\":0,"; + auto channel = core.getCurrentConfigFromCache(core.getPropertyFromCache("Core", "ChannelGroup").c_str()); + if(channel.empty()) + channel = "Default"; + ss << "\"Channel\":\""<< channel << "\","; + ss << "\"ChannelIndex\":0,"; + + try { ss << "\"BitDepth\":\"" << core.getProperty(core.getCameraDevice().c_str(), "Binning") << "\","; } catch(...) { } + + ss << "\"Image-index\":" << imgind; + ss << "}"; + return ss.str(); +} diff --git a/micromanager.sln b/micromanager.sln index 5cf054ba5..6ff3d9d0b 100644 --- a/micromanager.sln +++ b/micromanager.sln @@ -506,6 +506,16 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "TeensyPulseGenerator", "Dev EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Elveflow", "DeviceAdapters\Elveflow\Elveflow.vcxproj", "{A4BA201A-BDA5-45F0-8F56-9CE8E1C81123}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Go2Scope", "DeviceAdapters\Go2Scope\Go2Scope.vcxproj", "{2916E620-1157-4154-8223-887D383E9DE6}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "G2SStorageTest", "Tests\G2SStorageTest\G2SStorageTest.vcxproj", "{C87F7284-62EF-4331-9444-9951DB12A57A}" + ProjectSection(ProjectDependencies) = postProject + {2916E620-1157-4154-8223-887D383E9DE6} = {2916E620-1157-4154-8223-887D383E9DE6} + {36571628-728C-4ACD-A47F-503BA91C5D43} = {36571628-728C-4ACD-A47F-503BA91C5D43} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "AcquireZarr", "DeviceAdapters\AcquireZarr\AcquireZarr.vcxproj", "{293B79CE-124B-4323-A21A-E2DCC8994D52}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -1520,6 +1530,18 @@ Global {A4BA201A-BDA5-45F0-8F56-9CE8E1C81123}.Debug|x64.Build.0 = Debug|x64 {A4BA201A-BDA5-45F0-8F56-9CE8E1C81123}.Release|x64.ActiveCfg = Release|x64 {A4BA201A-BDA5-45F0-8F56-9CE8E1C81123}.Release|x64.Build.0 = Release|x64 + {2916E620-1157-4154-8223-887D383E9DE6}.Debug|x64.ActiveCfg = Debug|x64 + {2916E620-1157-4154-8223-887D383E9DE6}.Debug|x64.Build.0 = Debug|x64 + {2916E620-1157-4154-8223-887D383E9DE6}.Release|x64.ActiveCfg = Release|x64 + {2916E620-1157-4154-8223-887D383E9DE6}.Release|x64.Build.0 = Release|x64 + {C87F7284-62EF-4331-9444-9951DB12A57A}.Debug|x64.ActiveCfg = Debug|x64 + {C87F7284-62EF-4331-9444-9951DB12A57A}.Debug|x64.Build.0 = Debug|x64 + {C87F7284-62EF-4331-9444-9951DB12A57A}.Release|x64.ActiveCfg = Release|x64 + {C87F7284-62EF-4331-9444-9951DB12A57A}.Release|x64.Build.0 = Release|x64 + {293B79CE-124B-4323-A21A-E2DCC8994D52}.Debug|x64.ActiveCfg = Debug|x64 + {293B79CE-124B-4323-A21A-E2DCC8994D52}.Debug|x64.Build.0 = Debug|x64 + {293B79CE-124B-4323-A21A-E2DCC8994D52}.Release|x64.ActiveCfg = Release|x64 + {293B79CE-124B-4323-A21A-E2DCC8994D52}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE