Implementing streamed tiles system for big georeferenced images

62_CAD/DrawResourcesFiller.h

@@ -18,6 +18,8 @@ static_assert(sizeof(DrawObject) == 16u);
 static_assert(sizeof(MainObject) == 20u);
 static_assert(sizeof(LineStyle) == 88u);
 
+//TODO[Francisco]: Update briefs for geotex related functions
+
 // ! DrawResourcesFiller
 // ! This class provides important functionality to manage resources needed for a draw.
 // ! Drawing new objects (polylines, hatches, etc.) should go through this function.
@@ -120,6 +122,117 @@ struct DrawResourcesFiller
 				geometryInfo.getAlignedStorageSize();
 		}
 	};
+
+	// Used to load pieces of an ECW from disk - currently just emulated
+	struct ImageLoader
+	{
+		ImageLoader(core::smart_refctd_ptr<ICPUImage>&& _geoReferencedImage) : geoReferencedImage(std::move(_geoReferencedImage)) {}
+
+		// Emulates the loading of a rectangle from the original image
+		core::smart_refctd_ptr<ICPUImage> load(uint32_t2 offset, uint32_t2 extent)
+		{
+			// Create a new buffer pointing to same data - this is not what the class will be doing when streaming ECW
+			auto imageBuffer = geoReferencedImage->getBuffer();
+			asset::IBuffer::SCreationParams bufCreationParams = { .size = imageBuffer->getSize(), .usage = imageBuffer->getUsageFlags() };
+			ICPUBuffer::SCreationParams cpuBufCreationParams(std::move(bufCreationParams));
+			cpuBufCreationParams.data = imageBuffer->getPointer();
+			cpuBufCreationParams.memoryResource = core::getNullMemoryResource();
+			auto imageBufferAlias = ICPUBuffer::create(std::move(cpuBufCreationParams), core::adopt_memory_t{});
+			// Now set up the image region
+			auto bytesPerPixel = getTexelOrBlockBytesize(geoReferencedImage->getCreationParameters().format);
+
+			auto regions = core::make_refctd_dynamic_array<core::smart_refctd_dynamic_array<IImage::SBufferCopy>>(1u);
+			auto& region = regions->front();
+			// Row-major location of the first pixel (topleft) given by `offset`
+			region.bufferOffset = (offset.y * geoReferencedImage->getCreationParameters().extent.width + offset.x) * bytesPerPixel;
+			region.bufferRowLength = geoReferencedImage->getCreationParameters().extent.width;
+			region.bufferImageHeight = 0;
+			region.imageSubresource.aspectMask = IImage::EAF_COLOR_BIT;
+			region.imageSubresource.mipLevel = 0u;
+			region.imageSubresource.baseArrayLayer = 0u;
+			region.imageSubresource.layerCount = 1u;
+			// We're creating an image of exactly `extent` size and filling it entirely
+			region.imageOffset = { 0u, 0u, 0u };
+			region.imageExtent.width = extent.x;
+			region.imageExtent.height = extent.y;
+			region.imageExtent.depth = 1;
+
+			ICPUImage::SCreationParams loadedImageParams = geoReferencedImage->getCreationParameters();
+			loadedImageParams.extent = { extent.x, extent.y, 1u };
+			auto loadedImage = ICPUImage::create(std::move(loadedImageParams));
+			loadedImage->setBufferAndRegions(std::move(imageBufferAlias), regions);
+
+			return loadedImage;
+		}
+
+		// This will be the path to the image, and the loading will obviously be different
+		core::smart_refctd_ptr<ICPUImage> geoReferencedImage;
+	};
+
+	// @brief Used to load tiles into VRAM, keep track of loaded tiles, determine how they get sampled etc.
+	struct StreamedImageManager
+	{
+		friend class DrawResourcesFiller;
+		// These are mip 0 pixels per tile, also size of each physical tile into the gpu resident image
+		constexpr static uint32_t TileSize = 128u;
+		constexpr static uint32_t PaddingTiles = 2;
+
+		// These are vulkan standard, might be different in n4ce!
+		constexpr static float64_t3 topLeftViewportNDCH = float64_t3(-1.0, -1.0, 1.0);
+		constexpr static float64_t3 topRightViewportNDCH = float64_t3(1.0, -1.0, 1.0);
+		constexpr static float64_t3 bottomLeftViewportNDCH = float64_t3(-1.0, 1.0, 1.0);
+		constexpr static float64_t3 bottomRightViewportNDCH = float64_t3(1.0, 1.0, 1.0);
+
+		// Measured in tile coordinates in the image, and the mip level the tiles correspond to
+		struct TileLatticeAlignedObb
+		{
+			uint32_t2 topLeft;
+			uint32_t2 bottomRight;
+			uint32_t baseMipLevel;
+		};
+
+		// Holds gpu image upload info (what tiles to upload and where to upload them), an obb that encompasses the viewport and uv coords into the gpu image
+		// for the corners of that obb 
+		struct TileUploadData
+		{
+			core::vector<StreamedImageCopy> tiles;
+			OrientedBoundingBox2D worldspaceOBB;
+			float32_t2 minUV;
+			float32_t2 maxUV;
+		};
+
+		StreamedImageManager(image_id _imageID, GeoreferencedImageParams&& _georeferencedImageParams, ImageLoader&& _loader);
+
+		// Right now it's generating tile-by-tile. Can be improved to produce at worst 4 different rectangles to load (depending on how we need to load tiles)
+		TileUploadData generateTileUploadData(const ImageType imageType, const float64_t3x3& NDCToWorld, const float64_t2 viewportExtents);
+
+		image_id imageID = {};
+		GeoreferencedImageParams georeferencedImageParams = {};
+	private:
+		// These are NOT UV, pixel or tile coords into the mapped image region, rather into the real, huge image
+		// Tile coords are always in mip 0 tile size. Translating to other mips levels is trivial
+		float64_t2 transformWorldCoordsToUV(const float64_t3 worldCoordsH) { return nbl::hlsl::mul(world2UV, worldCoordsH); }
+		float64_t2 transformWorldCoordsToPixelCoords(const float64_t3 worldCoordsH) { return float64_t2(georeferencedImageParams.imageExtents) * transformWorldCoordsToUV(worldCoordsH); }
+		float64_t2 transformWorldCoordsToTileCoords(const float64_t3 worldCoordsH) { return (1.0 / TileSize) * transformWorldCoordsToPixelCoords(worldCoordsH); }
+
+		// Returns a tile aligned obb that encompasses the whole viewport in "image-world". Tiles are measured in the mip level required to fit the viewport entirely
+		// withing the gpu image.
+		TileLatticeAlignedObb computeViewportTileAlignedObb(const float64_t3x3& NDCToWorld);
+
+		// When the current mapped region is inadequate to fit the viewport, we compute a new mapped region
+		void remapCurrentRegion(const TileLatticeAlignedObb& viewportObb);
+
+		// Sidelength of the gpu image, in tiles that are `TileSize` pixels wide
+		uint32_t maxResidentTiles = {};
+		// Size of the image (minus 1), in tiles of `TileSize` sidelength
+		uint32_t2 maxImageTileIndices = {};
+		// Set topLeft to extreme value so it gets recreated on first iteration
+		TileLatticeAlignedObb currentMappedRegion = { .topLeft = uint32_t2(std::numeric_limits<uint32_t>::max(), std::numeric_limits<uint32_t>::max())};
+		std::vector<std::vector<bool>> currentMappedRegionOccupancy = {};
+		// Converts a point (z = 1) in worldspace to UV coordinates in image space (origin shifted to topleft of the image)
+		float64_t2x3 world2UV = {};
+		ImageLoader loader;
+	};
 
 	DrawResourcesFiller();
 
@@ -351,7 +464,7 @@ struct DrawResourcesFiller
 	void addImageObject(image_id imageID, const OrientedBoundingBox2D& obb, SIntendedSubmitInfo& intendedNextSubmit);
 
 	// This function must be called immediately after `addStaticImage` for the same imageID.
-	void addGeoreferencedImage(image_id imageID, const GeoreferencedImageParams& params, SIntendedSubmitInfo& intendedNextSubmit);
+	void addGeoreferencedImage(StreamedImageManager& manager, const float64_t3x3& NDCToWorld, uint32_t2 viewportExtents, SIntendedSubmitInfo& intendedNextSubmit);
 
 	/// @brief call this function before submitting to ensure all buffer and textures resourcesCollection requested via drawing calls are copied to GPU
 	/// records copy command into intendedNextSubmit's active command buffer and might possibly submits if fails allocation on staging upload memory.
@@ -596,7 +709,7 @@ struct DrawResourcesFiller
 	bool addImageObject_Internal(const ImageObjectInfo& imageObjectInfo, uint32_t mainObjIdx);;
 
 	/// Attempts to upload a georeferenced image info considering resource limitations (not accounting for the resource image added using ensureStaticImageAvailability function)
-	bool addGeoreferencedImageInfo_Internal(const GeoreferencedImageInfo& georeferencedImageInfo, uint32_t mainObjIdx);;
+	bool addGeoreferencedImageInfo_Internal(const GeoreferencedImageInfo& georeferencedImageInfo, uint32_t mainObjIdx);
 
 	uint32_t getImageIndexFromID(image_id imageID, const SIntendedSubmitInfo& intendedNextSubmit);
 
@@ -660,9 +773,9 @@ struct DrawResourcesFiller
 	 *
 	 * @param[out] outImageParams Structure to be filled with image creation parameters (format, size, etc.).
 	 * @param[out] outImageType Indicates whether the image should be fully resident or streamed.
-	 * @param[in] georeferencedImageParams Parameters describing the full image extents, viewport extents, and format.
+	 * @param[in] manager Manager for the georeferenced image
 	*/
-	void determineGeoreferencedImageCreationParams(nbl::asset::IImage::SCreationParams& outImageParams, ImageType& outImageType, const GeoreferencedImageParams& georeferencedImageParams);
+	ImageType determineGeoreferencedImageCreationParams(nbl::asset::IImage::SCreationParams& outImageParams, const GeoreferencedImageParams& params);
 
 	/**
 	 * @brief Used to implement both `drawHatch` and `drawFixedGeometryHatch` without exposing the transformation type parameter

62_CAD/Images.h

@@ -28,7 +28,6 @@ struct GeoreferencedImageParams
 	uint32_t2 imageExtents = {};
 	uint32_t2 viewportExtents = {};
 	asset::E_FORMAT format = {};
-	// TODO: Need to add other stuff later.
 };
 
 /**
@@ -205,7 +204,7 @@ class ImagesCache : public core::ResizableLRUCache<image_id, CachedImageRecord>
 struct StreamedImageCopy
 {
 	asset::E_FORMAT srcFormat;
-	core::smart_refctd_ptr<ICPUBuffer> srcBuffer; // Make it 'std::future' later?
+	smart_refctd_ptr<ICPUBuffer> srcBuffer; // Make it 'std::future' later?
 	asset::IImage::SBufferCopy region;
 };
 

62_CAD/main.cpp

@@ -49,6 +49,7 @@ static constexpr bool DebugRotatingViewProj = false;
 static constexpr bool FragmentShaderPixelInterlock = true;
 static constexpr bool LargeGeoTextureStreaming = true;
 static constexpr bool CacheAndReplay = false; // caches first frame resources (buffers and images) from DrawResourcesFiller  and replays in future frames, skiping CPU Logic
+static constexpr bool textCameraRotation = false;
 
 enum class ExampleMode
 {
@@ -64,6 +65,7 @@ enum class ExampleMode
 	CASE_9, // DTM
 	CASE_10, // testing fixed geometry and emulated fp64 corner cases
 	CASE_11, // grid DTM
+	CASE_12, // Georeferenced streamed images
 	CASE_COUNT
 };
 
@@ -80,10 +82,11 @@ constexpr std::array<float, (uint32_t)ExampleMode::CASE_COUNT> cameraExtents =
 	600.0,	// CASE_8
 	600.0,	// CASE_9
 	10.0,	// CASE_10
-	1000.0	// CASE_11
+	1000.0,	// CASE_11
+	10.0	// CASE_12
 };
 
-constexpr ExampleMode mode = ExampleMode::CASE_8;
+constexpr ExampleMode mode = ExampleMode::CASE_12;
 
 class Camera2D
 {
@@ -133,7 +136,7 @@ class Camera2D
 
 			if (ev.type == nbl::ui::SMouseEvent::EET_SCROLL)
 			{
-				m_bounds = m_bounds + float64_t2{ (double)ev.scrollEvent.verticalScroll * -0.1 * m_aspectRatio, (double)ev.scrollEvent.verticalScroll * -0.1};
+				m_bounds = m_bounds + float64_t2{ (double)ev.scrollEvent.verticalScroll * -0.0025 * m_aspectRatio, (double)ev.scrollEvent.verticalScroll * -0.0025};
 				m_bounds = float64_t2{ core::max(m_aspectRatio, m_bounds.x), core::max(1.0, m_bounds.y) };
 			}
 		}
@@ -1263,6 +1266,8 @@ class ComputerAidedDesign final : public nbl::examples::SimpleWindowedApplicatio
 
 		gridDTMHeightMap = loadImage("../../media/gridDTMHeightMap.exr");
 
+		bigTiledGrid = loadImage("../../media/tiled_grid.exr");
+
 		// set diagonals of cells to TOP_LEFT_TO_BOTTOM_RIGHT or BOTTOM_LEFT_TO_TOP_RIGHT randomly
 		{
 			// assumption is that format of the grid DTM height map is *_SRGB, I don't think we need any code to ensure that
@@ -1311,7 +1316,8 @@ class ComputerAidedDesign final : public nbl::examples::SimpleWindowedApplicatio
 
 		mouse.consumeEvents([&](const IMouseEventChannel::range_t& events) -> void
 			{
-				m_Camera.mouseProcess(events);
+				if (m_window->hasMouseFocus())
+					m_Camera.mouseProcess(events);
 			}
 		, m_logger.get());
 		keyboard.consumeEvents([&](const IKeyboardEventChannel::range_t& events) -> void
@@ -1492,17 +1498,9 @@ class ComputerAidedDesign final : public nbl::examples::SimpleWindowedApplicatio
 		projectionToNDC = m_Camera.constructViewProjection();
 
 		// TEST CAMERA ROTATION
-#if 1
-		// double rotation = 0.25 * PI<double>();
-		double rotation = abs(cos(m_timeElapsed * 0.0004)) * 0.25 * PI<double>() ;
-		float64_t2 rotationVec = float64_t2(cos(rotation), sin(rotation));
-		float64_t3x3 rotationParameter = float64_t3x3 {
-			rotationVec.x, rotationVec.y, 0.0,
-			-rotationVec.y, rotationVec.x, 0.0,
-			0.0, 0.0, 1.0
-		};
-		projectionToNDC = nbl::hlsl::mul(projectionToNDC, rotationParameter);
-#endif
+		if constexpr (textCameraRotation)
+			projectionToNDC = rotateBasedOnTime(projectionToNDC);
+
 		Globals globalData = {};
 		uint64_t baseAddress = resourcesGPUBuffer->getDeviceAddress();
 		globalData.pointers = {
@@ -3127,12 +3125,6 @@ class ComputerAidedDesign final : public nbl::examples::SimpleWindowedApplicatio
 				//printf("\n");
 			}
 
-			GeoreferencedImageParams geoRefParams = {};
-			geoRefParams.format = asset::EF_R8G8B8A8_SRGB;
-			geoRefParams.imageExtents = uint32_t2 (2048, 2048);
-			geoRefParams.viewportExtents = (m_realFrameIx <= 5u) ? uint32_t2(1280, 720) : uint32_t2(3840, 2160); // to test trigerring resize/recreation
-			// drawResourcesFiller.ensureGeoreferencedImageAvailability_AllocateIfNeeded(6996, geoRefParams, intendedNextSubmit);
-
 			LineStyleInfo lineStyle = 
 			{
 				.color = float32_t4(1.0f, 0.1f, 0.1f, 0.9f),
@@ -3686,6 +3678,44 @@ class ComputerAidedDesign final : public nbl::examples::SimpleWindowedApplicatio
 			}
 #endif
 		}
+		else if (mode == ExampleMode::CASE_12)
+		{
+			const static float64_t3 topLeftViewportH = float64_t3(-1.0, -1.0, 1.0);
+			const static float64_t3 topRightViewportH = float64_t3(1.0, -1.0, 1.0);
+			const static float64_t3 bottomLeftViewportH = float64_t3(-1.0, 1.0, 1.0);
+			const static float64_t3 bottomRightViewportH = float64_t3(1.0, 1.0, 1.0);
+
+			image_id tiledGridID = 6996;
+			static GeoreferencedImageParams tiledGridParams;
+			auto& tiledGridCreationParams = bigTiledGrid->getCreationParameters();
+			// Position at topLeft viewport
+			auto projectionToNDC = m_Camera.constructViewProjection();
+			// TEST CAMERA ROTATION
+			if constexpr (textCameraRotation)
+				projectionToNDC = rotateBasedOnTime(projectionToNDC);
+			auto inverseViewProj = nbl::hlsl::inverse(projectionToNDC);
+
+			const static auto startingTopLeft = nbl::hlsl::mul(inverseViewProj, topLeftViewportH);
+			tiledGridParams.worldspaceOBB.topLeft = startingTopLeft;
+
+			// Get 1 viewport pixel to match `startingImagePixelsPerViewportPixel` pixels of the image by choosing appropriate dirU
+			const static float64_t startingImagePixelsPerViewportPixels = 1.5;
+			const static auto startingViewportWidthVector = nbl::hlsl::mul(inverseViewProj, topRightViewportH - topLeftViewportH);
+			const static auto dirU = startingViewportWidthVector * float64_t(bigTiledGrid->getCreationParameters().extent.width) / float64_t(startingImagePixelsPerViewportPixels * m_window->getWidth());
+			tiledGridParams.worldspaceOBB.dirU = dirU;
+			tiledGridParams.worldspaceOBB.aspectRatio = 1.0;
+			tiledGridParams.imageExtents = { tiledGridCreationParams.extent.width, tiledGridCreationParams.extent.height};
+			tiledGridParams.viewportExtents = uint32_t2{ m_window->getWidth(), m_window->getHeight() };
+			tiledGridParams.format = tiledGridCreationParams.format;
+
+			static auto bigTileGridPtr = bigTiledGrid;
+			static DrawResourcesFiller::ImageLoader loader(std::move(bigTileGridPtr));
+			static DrawResourcesFiller::StreamedImageManager tiledGridManager(tiledGridID, std::move(tiledGridParams), std::move(loader));
+
+			drawResourcesFiller.ensureGeoreferencedImageAvailability_AllocateIfNeeded(tiledGridID, tiledGridManager.georeferencedImageParams, intendedNextSubmit);
+
+			drawResourcesFiller.addGeoreferencedImage(tiledGridManager, inverseViewProj, tiledGridParams.viewportExtents, intendedNextSubmit);
+		}
 	}
 
 	double getScreenToWorldRatio(const float64_t3x3& viewProjectionMatrix, uint32_t2 windowSize)
@@ -3695,6 +3725,18 @@ class ComputerAidedDesign final : public nbl::examples::SimpleWindowedApplicatio
 		return hlsl::length(float64_t2(idx_0_0, idx_1_0));
 	}
 
+	float64_t3x3 rotateBasedOnTime(const float64_t3x3& projectionMatrix)
+	{
+		double rotation = abs(cos(m_timeElapsed * 0.0004)) * 0.25 * PI<double>();
+		float64_t2 rotationVec = float64_t2(cos(rotation), sin(rotation));
+		float64_t3x3 rotationParameter = float64_t3x3{
+			rotationVec.x, rotationVec.y, 0.0,
+			-rotationVec.y, rotationVec.x, 0.0,
+			0.0, 0.0, 1.0
+		};
+		return nbl::hlsl::mul(projectionMatrix, rotationParameter);
+	}
+
 protected:
 	std::chrono::seconds timeout = std::chrono::seconds(0x7fffFFFFu);
 	clock_t::time_point start;
@@ -3758,6 +3800,7 @@ class ComputerAidedDesign final : public nbl::examples::SimpleWindowedApplicatio
 
 	std::vector<smart_refctd_ptr<ICPUImage>> sampleImages;
 	smart_refctd_ptr<ICPUImage> gridDTMHeightMap;
+	smart_refctd_ptr<ICPUImage> bigTiledGrid;
 
 	static constexpr char FirstGeneratedCharacter = ' ';
 	static constexpr char LastGeneratedCharacter = '~';