Implementing streamed tiles system for big georeferenced images

62_CAD/DrawResourcesFiller.h

@@ -18,6 +18,8 @@ static_assert(sizeof(DrawObject) == 16u);
 static_assert(sizeof(MainObject) == 20u);
 static_assert(sizeof(LineStyle) == 88u);
 
+//TODO[Francisco]: Update briefs for geotex related functions
+
 // ! DrawResourcesFiller
 // ! This class provides important functionality to manage resources needed for a draw.
 // ! Drawing new objects (polylines, hatches, etc.) should go through this function.
@@ -120,6 +122,36 @@ struct DrawResourcesFiller
 				geometryInfo.getAlignedStorageSize();
 		}
 	};
+
+	struct IGeoreferencedImageLoader : IReferenceCounted
+	{
+		virtual core::smart_refctd_ptr<ICPUBuffer> load(std::filesystem::path imagePath, uint32_t2 offset, uint32_t2 extent, uint32_t mipLevel) = 0;
+
+		virtual uint32_t2 getExtents(std::filesystem::path imagePath) = 0;
+
+		virtual asset::E_FORMAT getFormat(std::filesystem::path imagePath) = 0;
+	};
+
+	void setGeoreferencedImageLoader(core::smart_refctd_ptr<IGeoreferencedImageLoader>&& _georeferencedImageLoader)
+	{
+		georeferencedImageLoader = _georeferencedImageLoader;
+	}
+
+	uint32_t2 queryGeoreferencedImageExtents(std::filesystem::path imagePath)
+	{
+		return georeferencedImageLoader->getExtents(imagePath);
+	}
+
+	asset::E_FORMAT queryGeoreferencedImageFormat(std::filesystem::path imagePath)
+	{
+		return georeferencedImageLoader->getFormat(imagePath);
+	}
+
+	// These are vulkan standard, might be different in n4ce!
+	constexpr static float64_t3 topLeftViewportNDCH = float64_t3(-1.0, -1.0, 1.0);
+	constexpr static float64_t3 topRightViewportNDCH = float64_t3(1.0, -1.0, 1.0);
+	constexpr static float64_t3 bottomLeftViewportNDCH = float64_t3(-1.0, 1.0, 1.0);
+	constexpr static float64_t3 bottomRightViewportNDCH = float64_t3(1.0, 1.0, 1.0);
 
 	DrawResourcesFiller();
 
@@ -342,16 +374,16 @@ struct DrawResourcesFiller
 	 * @return true if the image was successfully cached and is ready for use; false if allocation failed.
 	 * [TODO]: should be internal protected member function.
 	 */
-	bool ensureGeoreferencedImageAvailability_AllocateIfNeeded(image_id imageID, const GeoreferencedImageParams& params, SIntendedSubmitInfo& intendedNextSubmit);
+	bool ensureGeoreferencedImageAvailability_AllocateIfNeeded(image_id imageID, GeoreferencedImageParams&& params, SIntendedSubmitInfo& intendedNextSubmit);
 
 	// [TODO]: should be internal protected member function.
 	bool queueGeoreferencedImageCopy_Internal(image_id imageID, const StreamedImageCopy& imageCopy);
 
 	// This function must be called immediately after `addStaticImage` for the same imageID.
 	void addImageObject(image_id imageID, const OrientedBoundingBox2D& obb, SIntendedSubmitInfo& intendedNextSubmit);
 
-	// This function must be called immediately after `addStaticImage` for the same imageID.
-	void addGeoreferencedImage(image_id imageID, const GeoreferencedImageParams& params, SIntendedSubmitInfo& intendedNextSubmit);
+	// This function must be called immediately after `ensureGeoreferencedImageAvailability_AllocateIfNeeded` for the same imageID.
+	void addGeoreferencedImage(image_id imageID, const float64_t3x3& NDCToWorld, SIntendedSubmitInfo& intendedNextSubmit);
 
 	/// @brief call this function before submitting to ensure all buffer and textures resourcesCollection requested via drawing calls are copied to GPU
 	/// records copy command into intendedNextSubmit's active command buffer and might possibly submits if fails allocation on staging upload memory.
@@ -596,7 +628,7 @@ struct DrawResourcesFiller
 	bool addImageObject_Internal(const ImageObjectInfo& imageObjectInfo, uint32_t mainObjIdx);;
 
 	/// Attempts to upload a georeferenced image info considering resource limitations (not accounting for the resource image added using ensureStaticImageAvailability function)
-	bool addGeoreferencedImageInfo_Internal(const GeoreferencedImageInfo& georeferencedImageInfo, uint32_t mainObjIdx);;
+	bool addGeoreferencedImageInfo_Internal(const GeoreferencedImageInfo& georeferencedImageInfo, uint32_t mainObjIdx);
 
 	uint32_t getImageIndexFromID(image_id imageID, const SIntendedSubmitInfo& intendedNextSubmit);
 
@@ -660,9 +692,9 @@ struct DrawResourcesFiller
 	 *
 	 * @param[out] outImageParams Structure to be filled with image creation parameters (format, size, etc.).
 	 * @param[out] outImageType Indicates whether the image should be fully resident or streamed.
-	 * @param[in] georeferencedImageParams Parameters describing the full image extents, viewport extents, and format.
+	 * @param[in] manager Manager for the georeferenced image
 	*/
-	void determineGeoreferencedImageCreationParams(nbl::asset::IImage::SCreationParams& outImageParams, ImageType& outImageType, const GeoreferencedImageParams& georeferencedImageParams);
+	ImageType determineGeoreferencedImageCreationParams(nbl::asset::IImage::SCreationParams& outImageParams, const GeoreferencedImageParams& params);
 
 	/**
 	 * @brief Used to implement both `drawHatch` and `drawFixedGeometryHatch` without exposing the transformation type parameter
@@ -762,7 +794,6 @@ struct DrawResourcesFiller
 		core::blake3_hash_t hash = {}; // actual hash, we will check in == operator
 		size_t lookupHash = 0ull; // for containers expecting size_t hash
 
-
 	private:
 
 		void computeBlake3Hash()
@@ -795,7 +826,29 @@ struct DrawResourcesFiller
 	uint32_t getMSDFIndexFromInputInfo(const MSDFInputInfo& msdfInfo, const SIntendedSubmitInfo& intendedNextSubmit);
 
 	uint32_t addMSDFTexture(const MSDFInputInfo& msdfInput, core::smart_refctd_ptr<ICPUImage>&& cpuImage, SIntendedSubmitInfo& intendedNextSubmit);
+
+	// These are mip 0 pixels per tile, also size of each physical tile into the gpu resident image
+	constexpr static uint32_t GeoreferencedImageTileSize = 128u;
+	constexpr static uint32_t GeoreferencedImagePaddingTiles = 2;
+
+	// Returns a tile range that encompasses the whole viewport in "image-world". Tiles are measured in the mip level required to fit the viewport entirely
+	// withing the gpu image.
+	GeoreferencedImageTileRange computeViewportTileRange(const float64_t3x3& NDCToWorld, const GeoreferencedImageStreamingState* imageStreamingState);
+
+	// Holds gpu image upload info (what tiles to upload and where to upload them), an obb that encompasses the viewport and uv coords into the gpu image
+	// for the corners of that obb 
+
+	struct TileUploadData
+	{
+		core::vector<StreamedImageCopy> tiles;
+		OrientedBoundingBox2D worldspaceOBB;
+		float32_t2 minUV;
+		float32_t2 maxUV;
+	};
 
+	// Right now it's generating tile-by-tile. Can be improved to produce at worst 4 different rectangles to load (depending on how we need to load tiles)
+	TileUploadData generateTileUploadData(const ImageType imageType, const float64_t3x3& NDCToWorld, GeoreferencedImageStreamingState* imageStreamingState);
+
 	// Flushes Current Draw Call and adds to drawCalls
 	void flushDrawObjects();
 
@@ -861,6 +914,8 @@ struct DrawResourcesFiller
 	std::unique_ptr<ImagesCache> imagesCache;
 	smart_refctd_ptr<SubAllocatedDescriptorSet> suballocatedDescriptorSet;
 	uint32_t imagesArrayBinding = 0u;
+	// Georef - pushed here rn for simplicity
+	core::smart_refctd_ptr<IGeoreferencedImageLoader> georeferencedImageLoader;
 
 	std::unordered_map<image_id, std::vector<StreamedImageCopy>> streamedImageCopies;
 };

62_CAD/Images.h

@@ -28,7 +28,7 @@ struct GeoreferencedImageParams
 	uint32_t2 imageExtents = {};
 	uint32_t2 viewportExtents = {};
 	asset::E_FORMAT format = {};
-	// TODO: Need to add other stuff later.
+	std::filesystem::path storagePath = {};
 };
 
 /**
@@ -109,6 +109,71 @@ struct ImageCleanup : public core::IReferenceCounted
 
 };
 
+// Measured in tile coordinates in the image that the range spans, and the mip level the tiles correspond to
+struct GeoreferencedImageTileRange
+{
+	uint32_t2 topLeft;
+	uint32_t2 bottomRight;
+	uint32_t baseMipLevel;
+};
+
+// @brief Used to load tiles into VRAM, keep track of loaded tiles, determine how they get sampled etc.
+struct GeoreferencedImageStreamingState : public IReferenceCounted
+{
+	friend class DrawResourcesFiller;
+
+protected:
+	static smart_refctd_ptr<GeoreferencedImageStreamingState> create(GeoreferencedImageParams&& _georeferencedImageParams)
+	{
+		smart_refctd_ptr<GeoreferencedImageStreamingState> retVal(new GeoreferencedImageStreamingState{});
+		retVal->georeferencedImageParams = std::move(_georeferencedImageParams);
+		//	1. Get the displacement (will be an offset vector in world coords and world units) from the `topLeft` corner of the image to the point
+		//	2. Transform this displacement vector into the coordinates in the basis {dirU, dirV} (worldspace vectors that span the sides of the image).
+		//	The composition of these matrices there fore transforms any point in worldspace into uv coordinates in imagespace
+
+
+		// 1. Displacement. The following matrix computes the offset for an input point `p` with homogenous worldspace coordinates. 
+		//    By foregoing the homogenous coordinate we can keep only the vector part, that's why it's `2x3` and not `3x3`
+		float64_t2 topLeftWorld = retVal->georeferencedImageParams.worldspaceOBB.topLeft;
+		float64_t2x3 displacementMatrix(1., 0., - topLeftWorld.x, 0., 1., - topLeftWorld.y);
+
+		// 2. Change of Basis. Since {dirU, dirV} are orthogonal, the matrix to change from world coords to `span{dirU, dirV}` coords has a quite nice expression
+		//    Non-uniform scaling doesn't affect this, but this has to change if we allow for shearing (basis vectors stop being orthogonal)
+		float64_t2 dirU = retVal->georeferencedImageParams.worldspaceOBB.dirU;
+		float64_t2 dirV = float32_t2(dirU.y, -dirU.x) * retVal->georeferencedImageParams.worldspaceOBB.aspectRatio;
+		float64_t dirULengthSquared = nbl::hlsl::dot(dirU, dirU);
+		float64_t dirVLengthSquared = nbl::hlsl::dot(dirV, dirV);
+		float64_t2 firstRow = dirU / dirULengthSquared;
+		float64_t2 secondRow = dirV / dirVLengthSquared;
+		float64_t2x2 changeOfBasisMatrix(firstRow, secondRow);
+
+		// Put them all together
+		retVal->world2UV = nbl::hlsl::mul(changeOfBasisMatrix, displacementMatrix);
+		return retVal;
+	}
+
+	GeoreferencedImageParams georeferencedImageParams = {};
+	std::vector<std::vector<bool>> currentMappedRegionOccupancy = {};
+
+	// These are NOT UV, pixel or tile coords into the mapped image region, rather into the real, huge image
+	// Tile coords are always in mip 0 tile size. Translating to other mips levels is trivial
+	float64_t2 transformWorldCoordsToUV(const float64_t3 worldCoordsH) const { return nbl::hlsl::mul(world2UV, worldCoordsH); }
+	float64_t2 transformWorldCoordsToPixelCoords(const float64_t3 worldCoordsH) const { return float64_t2(georeferencedImageParams.imageExtents) * transformWorldCoordsToUV(worldCoordsH); }
+	float64_t2 transformWorldCoordsToTileCoords(const float64_t3 worldCoordsH, const uint32_t TileSize) const { return (1.0 / TileSize) * transformWorldCoordsToPixelCoords(worldCoordsH); }
+
+	// When the current mapped region is inadequate to fit the viewport, we compute a new mapped region
+	void remapCurrentRegion(const GeoreferencedImageTileRange& viewportTileRange);
+
+	// Sidelength of the gpu image, in tiles that are `GeoreferencedImageTileSize` pixels wide
+	uint32_t maxResidentTiles = {};
+	// Size of the image (minus 1), in tiles of `GeoreferencedImageTileSize` sidelength
+	uint32_t2 maxImageTileIndices = {};
+	// Set topLeft to extreme value so it gets recreated on first iteration
+	GeoreferencedImageTileRange currentMappedRegion = { .topLeft = uint32_t2(std::numeric_limits<uint32_t>::max(), std::numeric_limits<uint32_t>::max()) };
+	// Converts a point (z = 1) in worldspace to UV coordinates in image space (origin shifted to topleft of the image)
+	float64_t2x3 world2UV = {};
+};
+
 struct CachedImageRecord
 {
 	static constexpr uint32_t InvalidTextureIndex = nbl::hlsl::numeric_limits<uint32_t>::max;
@@ -121,6 +186,7 @@ struct CachedImageRecord
 	uint64_t allocationSize = 0ull;
 	core::smart_refctd_ptr<IGPUImageView> gpuImageView = nullptr;
 	core::smart_refctd_ptr<ICPUImage> staticCPUImage = nullptr; // cached cpu image for uploading to gpuImageView when needed.
+	core::smart_refctd_ptr<GeoreferencedImageStreamingState> georeferencedImageState = nullptr; // Used to track tile residency for georeferenced images
 
 	// In LRU Cache `insert` function, in case of cache miss, we need to construct the refereence with semaphore value
 	CachedImageRecord(uint64_t currentFrameIndex) 
@@ -205,7 +271,7 @@ class ImagesCache : public core::ResizableLRUCache<image_id, CachedImageRecord>
 struct StreamedImageCopy
 {
 	asset::E_FORMAT srcFormat;
-	core::smart_refctd_ptr<ICPUBuffer> srcBuffer; // Make it 'std::future' later?
+	smart_refctd_ptr<ICPUBuffer> srcBuffer; // Make it 'std::future' later?
 	asset::IImage::SBufferCopy region;
 };