Implementing streamed tiles system for big georeferenced images

62_CAD/DrawResourcesFiller.h

@@ -18,6 +18,8 @@ static_assert(sizeof(DrawObject) == 16u);
 static_assert(sizeof(MainObject) == 20u);
 static_assert(sizeof(LineStyle) == 88u);
 
+//TODO[Francisco]: Update briefs for geotex related functions
+
 // ! DrawResourcesFiller
 // ! This class provides important functionality to manage resources needed for a draw.
 // ! Drawing new objects (polylines, hatches, etc.) should go through this function.
@@ -120,6 +122,36 @@ struct DrawResourcesFiller
 				geometryInfo.getAlignedStorageSize();
 		}
 	};
+
+	struct IGeoreferencedImageLoader : IReferenceCounted
+	{
+		virtual core::smart_refctd_ptr<ICPUBuffer> load(std::filesystem::path imagePath, uint32_t2 offset, uint32_t2 extent, uint32_t mipLevel) = 0;
+
+		virtual uint32_t2 getExtents(std::filesystem::path imagePath) = 0;
+
+		virtual asset::E_FORMAT getFormat(std::filesystem::path imagePath) = 0;
+	};
+
+	void setGeoreferencedImageLoader(core::smart_refctd_ptr<IGeoreferencedImageLoader>&& _georeferencedImageLoader)
+	{
+		georeferencedImageLoader = _georeferencedImageLoader;
+	}
+
+	uint32_t2 queryGeoreferencedImageExtents(std::filesystem::path imagePath)
+	{
+		return georeferencedImageLoader->getExtents(imagePath);
+	}
+
+	asset::E_FORMAT queryGeoreferencedImageFormat(std::filesystem::path imagePath)
+	{
+		return georeferencedImageLoader->getFormat(imagePath);
+	}
+
+	// These are vulkan standard, might be different in n4ce!
+	constexpr static float64_t3 topLeftViewportNDC = float64_t3(-1.0, -1.0, 1.0);
+	constexpr static float64_t3 topRightViewportNDC = float64_t3(1.0, -1.0, 1.0);
+	constexpr static float64_t3 bottomLeftViewportNDC = float64_t3(-1.0, 1.0, 1.0);
+	constexpr static float64_t3 bottomRightViewportNDC = float64_t3(1.0, 1.0, 1.0);
 
 	DrawResourcesFiller();
 
@@ -342,16 +374,16 @@ struct DrawResourcesFiller
 	 * @return true if the image was successfully cached and is ready for use; false if allocation failed.
 	 * [TODO]: should be internal protected member function.
 	 */
-	bool ensureGeoreferencedImageAvailability_AllocateIfNeeded(image_id imageID, const GeoreferencedImageParams& params, SIntendedSubmitInfo& intendedNextSubmit);
+	bool ensureGeoreferencedImageAvailability_AllocateIfNeeded(image_id imageID, GeoreferencedImageParams&& params, SIntendedSubmitInfo& intendedNextSubmit);
 
 	// [TODO]: should be internal protected member function.
 	bool queueGeoreferencedImageCopy_Internal(image_id imageID, const StreamedImageCopy& imageCopy);
 
 	// This function must be called immediately after `addStaticImage` for the same imageID.
 	void addImageObject(image_id imageID, const OrientedBoundingBox2D& obb, SIntendedSubmitInfo& intendedNextSubmit);
 
-	// This function must be called immediately after `addStaticImage` for the same imageID.
-	void addGeoreferencedImage(image_id imageID, const GeoreferencedImageParams& params, SIntendedSubmitInfo& intendedNextSubmit);
+	// This function must be called immediately after `ensureGeoreferencedImageAvailability_AllocateIfNeeded` for the same imageID.
+	void addGeoreferencedImage(image_id imageID, const float64_t3x3& NDCToWorld, SIntendedSubmitInfo& intendedNextSubmit);
 
 	/// @brief call this function before submitting to ensure all buffer and textures resourcesCollection requested via drawing calls are copied to GPU
 	/// records copy command into intendedNextSubmit's active command buffer and might possibly submits if fails allocation on staging upload memory.
@@ -596,7 +628,7 @@ struct DrawResourcesFiller
 	bool addImageObject_Internal(const ImageObjectInfo& imageObjectInfo, uint32_t mainObjIdx);;
 
 	/// Attempts to upload a georeferenced image info considering resource limitations (not accounting for the resource image added using ensureStaticImageAvailability function)
-	bool addGeoreferencedImageInfo_Internal(const GeoreferencedImageInfo& georeferencedImageInfo, uint32_t mainObjIdx);;
+	bool addGeoreferencedImageInfo_Internal(const GeoreferencedImageInfo& georeferencedImageInfo, uint32_t mainObjIdx);
 
 	uint32_t getImageIndexFromID(image_id imageID, const SIntendedSubmitInfo& intendedNextSubmit);
 
@@ -660,9 +692,9 @@ struct DrawResourcesFiller
 	 *
 	 * @param[out] outImageParams Structure to be filled with image creation parameters (format, size, etc.).
 	 * @param[out] outImageType Indicates whether the image should be fully resident or streamed.
-	 * @param[in] georeferencedImageParams Parameters describing the full image extents, viewport extents, and format.
+	 * @param[in] params Parameters for the georeferenced image
 	*/
-	void determineGeoreferencedImageCreationParams(nbl::asset::IImage::SCreationParams& outImageParams, ImageType& outImageType, const GeoreferencedImageParams& georeferencedImageParams);
+	ImageType determineGeoreferencedImageCreationParams(nbl::asset::IImage::SCreationParams& outImageParams, const GeoreferencedImageParams& params);
 
 	/**
 	 * @brief Used to implement both `drawHatch` and `drawFixedGeometryHatch` without exposing the transformation type parameter
@@ -762,7 +794,6 @@ struct DrawResourcesFiller
 		core::blake3_hash_t hash = {}; // actual hash, we will check in == operator
 		size_t lookupHash = 0ull; // for containers expecting size_t hash
 
-
 	private:
 
 		void computeBlake3Hash()
@@ -795,7 +826,29 @@ struct DrawResourcesFiller
 	uint32_t getMSDFIndexFromInputInfo(const MSDFInputInfo& msdfInfo, const SIntendedSubmitInfo& intendedNextSubmit);
 
 	uint32_t addMSDFTexture(const MSDFInputInfo& msdfInput, core::smart_refctd_ptr<ICPUImage>&& cpuImage, SIntendedSubmitInfo& intendedNextSubmit);
+
+	// These are mip 0 pixels per tile, also size of each physical tile into the gpu resident image
+	constexpr static uint32_t GeoreferencedImageTileSize = 128u;
+	constexpr static uint32_t GeoreferencedImagePaddingTiles = 2;
+
+	// Returns a tile range that encompasses the whole viewport in "image-world". Tiles are measured in the mip level required to fit the viewport entirely
+	// withing the gpu image.
+	GeoreferencedImageTileRange computeViewportTileRange(const float64_t3x3& NDCToWorld, const GeoreferencedImageStreamingState* imageStreamingState);
+
+	// Holds gpu image upload info (what tiles to upload and where to upload them), an obb that encompasses the viewport and uv coords into the gpu image
+	// for the corners of that obb 
+
+	struct TileUploadData
+	{
+		core::vector<StreamedImageCopy> tiles;
+		OrientedBoundingBox2D viewportEncompassingOBB;
+		float32_t2 minUV;
+		float32_t2 maxUV;
+	};
 
+	// Right now it's generating tile-by-tile. Can be improved to produce at worst 4 different rectangles to load (depending on how we need to load tiles)
+	TileUploadData generateTileUploadData(const ImageType imageType, const float64_t3x3& NDCToWorld, GeoreferencedImageStreamingState* imageStreamingState);
+
 	// Flushes Current Draw Call and adds to drawCalls
 	void flushDrawObjects();
 
@@ -861,6 +914,8 @@ struct DrawResourcesFiller
 	std::unique_ptr<ImagesCache> imagesCache;
 	smart_refctd_ptr<SubAllocatedDescriptorSet> suballocatedDescriptorSet;
 	uint32_t imagesArrayBinding = 0u;
+	// Georef - pushed here rn for simplicity
+	core::smart_refctd_ptr<IGeoreferencedImageLoader> georeferencedImageLoader;
 
 	std::unordered_map<image_id, std::vector<StreamedImageCopy>> streamedImageCopies;
 };

62_CAD/Images.h

@@ -28,7 +28,7 @@ struct GeoreferencedImageParams
 	uint32_t2 imageExtents = {};
 	uint32_t2 viewportExtents = {};
 	asset::E_FORMAT format = {};
-	// TODO: Need to add other stuff later.
+	std::filesystem::path storagePath = {};
 };
 
 /**
@@ -109,6 +109,190 @@ struct ImageCleanup : public core::IReferenceCounted
 
 };
 
+// Measured in tile coordinates in the image that the range spans, and the mip level the tiles correspond to
+struct GeoreferencedImageTileRange
+{
+	uint32_t2 topLeft;
+	uint32_t2 bottomRight;
+	uint32_t baseMipLevel;
+};
+
+// @brief Used to load tiles into VRAM, keep track of loaded tiles, determine how they get sampled etc.
+struct GeoreferencedImageStreamingState : public IReferenceCounted
+{
+	friend class DrawResourcesFiller;
+
+protected:
+	static smart_refctd_ptr<GeoreferencedImageStreamingState> create(GeoreferencedImageParams&& _georeferencedImageParams)
+	{
+		smart_refctd_ptr<GeoreferencedImageStreamingState> retVal(new GeoreferencedImageStreamingState{});
+		retVal->georeferencedImageParams = std::move(_georeferencedImageParams);
+		//	1. Get the displacement (will be an offset vector in world coords and world units) from the `topLeft` corner of the image to the point
+		//	2. Transform this displacement vector into the coordinates in the basis {dirU, dirV} (worldspace vectors that span the sides of the image).
+		//	The composition of these matrices therefore transforms any point in worldspace into uv coordinates in imagespace
+		//  To reduce code complexity, instead of computing the product of these matrices, since the first is a pure displacement matrix 
+		//  (non-homogenous 2x2 upper left is identity matrix) and the other is a pure rotation matrix (2x2) we can just put them together
+		//  by putting the rotation in the upper left 2x2 of the result and the post-rotated displacement in the upper right 2x1.
+		//  The result is also 2x3 and not 3x3 because we can drop he homogenous since the displacement yields a vector
+
+		// 2. Change of Basis. Since {dirU, dirV} are orthogonal, the matrix to change from world coords to `span{dirU, dirV}` coords has a quite nice expression
+		//    Non-uniform scaling doesn't affect this, but this has to change if we allow for shearing (basis vectors stop being orthogonal)
+		const float64_t2 dirU = retVal->georeferencedImageParams.worldspaceOBB.dirU;
+		const float64_t2 dirV = float32_t2(dirU.y, -dirU.x) * retVal->georeferencedImageParams.worldspaceOBB.aspectRatio;
+		const float64_t dirULengthSquared = nbl::hlsl::dot(dirU, dirU);
+		const float64_t dirVLengthSquared = nbl::hlsl::dot(dirV, dirV);
+		const float64_t2 firstRow = dirU / dirULengthSquared;
+		const float64_t2 secondRow = dirV / dirVLengthSquared;
+
+		const float64_t2 displacement = - retVal->georeferencedImageParams.worldspaceOBB.topLeft;
+		// This is the same as multiplying the change of basis matrix by the displacement vector
+		const float64_t postRotatedShiftX = nbl::hlsl::dot(firstRow, displacement);
+		const float64_t postRotatedShiftY = nbl::hlsl::dot(secondRow, displacement);
+
+		// Put them all together
+		retVal->world2UV = float64_t2x3(firstRow.x, firstRow.y, postRotatedShiftX, secondRow.x, secondRow.y, postRotatedShiftY);
+		return retVal;
+	}
+
+	GeoreferencedImageParams georeferencedImageParams = {};
+	std::vector<std::vector<bool>> currentMappedRegionOccupancy = {};
+
+	// These are NOT UV, pixel or tile coords into the mapped image region, rather into the real, huge image
+	// Tile coords are always in mip 0 tile size. Translating to other mips levels is trivial
+	float64_t2 transformWorldCoordsToUV(const float64_t3 worldCoords) const { return nbl::hlsl::mul(world2UV, worldCoords); }
+	float64_t2 transformWorldCoordsToPixelCoords(const float64_t3 worldCoords) const { return float64_t2(georeferencedImageParams.imageExtents) * transformWorldCoordsToUV(worldCoords); }
+	float64_t2 transformWorldCoordsToTileCoords(const float64_t3 worldCoords, const uint32_t TileSize) const { return (1.0 / TileSize) * transformWorldCoordsToPixelCoords(worldCoords); }
+
+	// When the current mapped region is inadequate to fit the viewport, we compute a new mapped region
+	void remapCurrentRegion(const GeoreferencedImageTileRange& viewportTileRange)
+	{
+		// Zoomed out
+		if (viewportTileRange.baseMipLevel > currentMappedRegion.baseMipLevel)
+		{
+			// TODO: Here we would move some mip 1 tiles to mip 0 image to save the work of reuploading them, reflect that in the tracked tiles
+		}
+		// Zoomed in
+		else if (viewportTileRange.baseMipLevel < currentMappedRegion.baseMipLevel)
+		{
+			// TODO: Here we would move some mip 0 tiles to mip 1 image to save the work of reuploading them, reflect that in the tracked tiles
+		}
+		currentMappedRegion = viewportTileRange;
+
+		currentMappedRegionOccupancy.resize(gpuImageSideLengthTiles);
+		for (auto i = 0u; i < gpuImageSideLengthTiles; i++)
+		{
+			currentMappedRegionOccupancy[i].clear();
+			currentMappedRegionOccupancy[i].resize(gpuImageSideLengthTiles, false);
+		}
+		gpuImageTopLeft = uint32_t2(0, 0);
+	}
+
+	// When we can shift the mapped a region a bit and avoid tile uploads by using toroidal shifting
+	void shiftAndExpandCurrentRegion(const GeoreferencedImageTileRange& viewportTileRange)
+	{
+		// `topLeftDiff` starts as the vector (in tiles) from the current mapped region's top left to the top left of the range encompassing the viewport
+		int32_t2 topLeftDiff = int32_t2(viewportTileRange.topLeft) - int32_t2(currentMappedRegion.topLeft);
+		// Since we only consider expanding the mapped region by moving the top left up and to the left, we clamp the above vector to `(-infty, 0] x (-infty, 0]`
+		topLeftDiff = nbl::hlsl::min(topLeftDiff, int32_t2(0, 0));
+		int32_t2 nextTopLeft = int32_t2(currentMappedRegion.topLeft) + topLeftDiff;
+		// Same logic for bottom right but considering it only moves down and to the right, so clamped to `[0, infty) x [0, infty)`
+		int32_t2 bottomRightDiff = int32_t2(viewportTileRange.bottomRight) - int32_t2(currentMappedRegion.bottomRight);
+		bottomRightDiff = nbl::hlsl::max(bottomRightDiff, int32_t2(0, 0));
+		int32_t2 nextBottomRight = int32_t2(currentMappedRegion.bottomRight) + bottomRightDiff;
+
+		// If the number of tiles resident in this new mapped region along any axis becomes bigger than the max number of tiles the gpu image can hold, 
+		// we need to shrink this next mapped region. For this to happen, we have to have expanded in only one direction, the one that has `diff != 0`
+		// Therefore, we need to shrink the mapped region along the axis that has `diff = 0`, just enough tiles so that the mapped region's tile size stays within
+		// the max number of tiles the gpu image can hold.
+		int32_t2 nextMappedRegionDimensions = nextBottomRight - nextTopLeft + 1;
+		uint32_t2 currentMappedRegionDimensions = currentMappedRegion.bottomRight - currentMappedRegion.topLeft + 1u;
+		uint32_t2 gpuImageBottomRight = (gpuImageTopLeft + currentMappedRegionDimensions - 1u) % gpuImageSideLengthTiles;
+
+		// Shrink along x axis
+		if (nextMappedRegionDimensions.x > gpuImageSideLengthTiles)
+		{
+			int32_t tilesToFit = nextMappedRegionDimensions.x - gpuImageSideLengthTiles;
+			if (0 == topLeftDiff.x)
+			{
+				// Move topLeft to the right to fit tiles on the other side
+				nextTopLeft.x += tilesToFit;
+				topLeftDiff.x += tilesToFit;
+				// Mark all these tiles as non-resident
+				for (uint32_t tile = 0; tile < tilesToFit; tile++)
+				{
+					// Get actual tile index with wraparound
+					uint32_t tileIdx = (tile + gpuImageTopLeft.x) % gpuImageSideLengthTiles;
+					for (uint32_t i = 0u; i < gpuImageSideLengthTiles; i++)
+						currentMappedRegionOccupancy[tileIdx][i] = false;
+				}
+			}
+			else
+			{
+				// Move bottomRight to the left to fit tiles on the other side
+				nextBottomRight.x -= tilesToFit;
+				// Mark all these tiles as non-resident
+				for (uint32_t tile = 0; tile < tilesToFit; tile++)
+				{
+					// Get actual tile index with wraparound
+					uint32_t tileIdx = (gpuImageBottomRight.x + (gpuImageSideLengthTiles - tile)) % gpuImageSideLengthTiles;
+					for (uint32_t i = 0u; i < gpuImageSideLengthTiles; i++)
+						currentMappedRegionOccupancy[tileIdx][i] = false;
+				}
+			}
+		}
+		// Shrink along y axis
+		if (nextMappedRegionDimensions.y > gpuImageSideLengthTiles)
+		{
+			int32_t tilesToFit = nextMappedRegionDimensions.y - gpuImageSideLengthTiles;
+			if (0 == topLeftDiff.y)
+			{
+				// Move topLeft down to fit tiles on the other side
+				nextTopLeft.y += tilesToFit;
+				topLeftDiff.y += tilesToFit;
+				// Mark all these tiles as non-resident
+				for (uint32_t tile = 0; tile < tilesToFit; tile++)
+				{
+					// Get actual tile index with wraparound
+					uint32_t tileIdx = (tile + gpuImageTopLeft.y) % gpuImageSideLengthTiles;
+					for (uint32_t i = 0u; i < gpuImageSideLengthTiles; i++)
+						currentMappedRegionOccupancy[i][tileIdx] = false;
+				}
+			}
+			else
+			{
+				// Move bottomRight up to fit tiles on the other side
+				nextBottomRight.y -= tilesToFit;
+				// Mark all these tiles as non-resident
+				for (uint32_t tile = 0; tile < tilesToFit; tile++)
+				{
+					// Get actual tile index with wraparound
+					uint32_t tileIdx = (gpuImageBottomRight.y + (gpuImageSideLengthTiles - tile)) % gpuImageSideLengthTiles;
+					for (uint32_t i = 0u; i < gpuImageSideLengthTiles; i++)
+						currentMappedRegionOccupancy[i][tileIdx] = false;
+				}
+			}
+		}
+
+		// Set new values for mapped region
+		currentMappedRegion.topLeft = nextTopLeft;
+		currentMappedRegion.bottomRight = nextBottomRight;
+
+		// Toroidal shift for the gpu image top left
+		gpuImageTopLeft = (gpuImageTopLeft + uint32_t2(topLeftDiff + int32_t(gpuImageSideLengthTiles))) % gpuImageSideLengthTiles;
+	}
+
+	// Sidelength of the gpu image, in tiles that are `GeoreferencedImageTileSize` pixels wide
+	uint32_t gpuImageSideLengthTiles = {};
+	// Size of the image (minus 1), in tiles of `GeoreferencedImageTileSize` sidelength
+	uint32_t2 fullImageLastTileIndices = {};
+	// Set mip level to extreme value so it gets recreated on first iteration
+	GeoreferencedImageTileRange currentMappedRegion = { .baseMipLevel = std::numeric_limits<uint32_t>::max() };
+	// Indicates on which tile of the gpu image the current mapped region's `topLeft` resides
+	uint32_t2 gpuImageTopLeft = {};
+	// Converts a point (z = 1) in worldspace to UV coordinates in image space (origin shifted to topleft of the image)
+	float64_t2x3 world2UV = {};
+};
+
 struct CachedImageRecord
 {
 	static constexpr uint32_t InvalidTextureIndex = nbl::hlsl::numeric_limits<uint32_t>::max;
@@ -121,6 +305,7 @@ struct CachedImageRecord
 	uint64_t allocationSize = 0ull;
 	core::smart_refctd_ptr<IGPUImageView> gpuImageView = nullptr;
 	core::smart_refctd_ptr<ICPUImage> staticCPUImage = nullptr; // cached cpu image for uploading to gpuImageView when needed.
+	core::smart_refctd_ptr<GeoreferencedImageStreamingState> georeferencedImageState = nullptr; // Used to track tile residency for georeferenced images
 
 	// In LRU Cache `insert` function, in case of cache miss, we need to construct the refereence with semaphore value
 	CachedImageRecord(uint64_t currentFrameIndex) 
@@ -205,7 +390,7 @@ class ImagesCache : public core::ResizableLRUCache<image_id, CachedImageRecord>
 struct StreamedImageCopy
 {
 	asset::E_FORMAT srcFormat;
-	core::smart_refctd_ptr<ICPUBuffer> srcBuffer; // Make it 'std::future' later?
+	smart_refctd_ptr<ICPUBuffer> srcBuffer; // Make it 'std::future' later?
 	asset::IImage::SBufferCopy region;
 };