Implemeted AIO file writing (more like writing on different threads, actual AIO not yet implemented on linux)

Dandielo · Dandielo · commit 04ca2f1e07b2 · 2025-06-07T15:04:07.000+09:00
* Reenabled shader pipeline on Linux after the current set of fixes.
* Fixed bugs and issues in 'hscp_reader' tool.
* Fixed params CLI11 not being able to handle same path values like Windows. (manually stripping quotes from values)
* Fixed memory leak when creating resources from baked data.
diff --git a/source/asset_compiler.bff b/source/asset_compiler.bff
@@ -6,7 +6,11 @@
     .Name = 'Assets-Shaders-Mobile'
     .Requires = { 'assets-mobile', 'Step-Bake-Shaders' }
     .BuildOptions = {
+#if __WINDOWS__
         '-c shader_tools.dll'
+#else // __LINUX__
+        '-c libshader_tools.so'
+#endif
         '--param shader:target GLSL'
     }
 ]
@@ -16,7 +20,11 @@
     .Name = 'Assets-Shaders-Web'
     .Requires = { 'assets-web', 'Step-Bake-Shaders' }
     .BuildOptions = {
+#if __WINDOWS__
         '-c shader_tools.dll'
+#else // __LINUX__
+        '-c libshader_tools.so'
+#endif
         '--param shader:target WGSL'
     }
 ]
diff --git a/source/code/core/utils/private/native_aio.cxx b/source/code/core/utils/private/native_aio.cxx
@@ -300,7 +300,7 @@ namespace ice::native_aio
         internal.next = nullptr;
         internal.native_file_handle = file.native();
         internal.request_type = 1; // 1 = read, 2 = write
-        internal.data_location = memory.location;
+        internal.data_destination = memory.location;
         internal.data_offset = static_cast<ice::u32>(requested_read_offset.value);
         internal.data_size = static_cast<ice::u32>(requested_read_size.value);
         ice::linked_queue::push(request._port->_requests, ice::addressof(internal));
@@ -310,6 +310,40 @@ namespace ice::native_aio
         return ice::native_file::FileRequestStatus::Pending;
     }
 
+    auto aio_file_write_request(
+        ice::native_aio::AIORequest& request,
+        ice::native_file::File const& file,
+        ice::usize requested_write_offset,
+        ice::Data data
+    ) noexcept -> ice::native_file::FileRequestStatus
+    {
+        using ice::native_file::FileRequestStatus;
+
+        if (data.size == 0_B)
+        {
+            return FileRequestStatus::Completed;
+        }
+
+        if (data.size.value > ice::u32_max)
+        {
+            // Too big of a write
+            return FileRequestStatus::Error;
+        }
+
+        AIORequestInternal& internal = reinterpret_cast<AIORequestInternal&>(request);
+        internal.next = nullptr;
+        internal.native_file_handle = file.native();
+        internal.request_type = 2; // 1 = read, 2 = write
+        internal.data_location = data.location;
+        internal.data_size = ice::u32(data.size.value);
+        internal.data_offset = ice::u32(requested_write_offset.value);
+        ice::linked_queue::push(request._port->_requests, ice::addressof(internal));
+
+        // Increment the semaphore
+        sem_post(&request._port->_semaphore);
+        return ice::native_file::FileRequestStatus::Pending;
+    }
+
     bool aio_file_await_request(
         ice::native_aio::AIOPort port,
         ice::native_aio::AIOProcessLimits limits,
@@ -344,36 +378,48 @@ namespace ice::native_aio
             if (internal->request_type == 1)
             {
                 // TODO: Handle sizes / offsets above 4gb
-                ice::i32 const bytes_read = pread(
+                ssize_t const bytes_read = pread(
                     internal->native_file_handle,
-                    internal->data_location,
+                    internal->data_destination,
                     internal->data_size,
                     internal->data_offset
                 );
 
                 ICE_ASSERT_CORE(bytes_read > 0); // 0 == eof, -1 == error
                 ICE_ASSERT_CORE(bytes_read == internal->data_size);
-
                 out_size = { (ice::usize::base_type) bytes_read };
             }
             else
             {
-                // TODO: Write requests
-                ICE_ASSERT_CORE(false);
+                ssize_t const bytes_written = pwrite(
+                    internal->native_file_handle,
+                    internal->data_location,
+                    internal->data_size,
+                    internal->data_offset
+                );
+
+                ICE_ASSERT_CORE(bytes_written > 0);
+                ICE_ASSERT_CORE(bytes_written == internal->data_size);
+                out_size = { (ice::usize::base_type) bytes_written };
             }
         }
+        else
+        {
+            // Repost the semaphore, because we missed a request.
+            sem_post(&port->_semaphore);
+        }
         return out_size > 0_B;
     }
 
     void aio_complete_request(
         ice::native_aio::AIORequest const* request,
         ice::native_aio::AIORequestResult result,
-        ice::usize read_size
+        ice::usize processed_bytes
     ) noexcept
     {
         if (request != nullptr && request->_callback != nullptr)
         {
-            request->_callback(result, read_size, request->_userdata);
+            request->_callback(result, processed_bytes, request->_userdata);
         }
     }
 #else
diff --git a/source/code/core/utils/private/native_aio.hxx b/source/code/core/utils/private/native_aio.hxx
@@ -24,7 +24,11 @@ namespace ice::native_aio
         AIORequestInternal* next;
         ice::i32 native_file_handle;
         ice::u32 request_type; // 1 == read, 2 == write
-        void* data_location;
+        union
+        {
+            void* data_destination;
+            void const* data_location;
+        };
         ice::u32 data_offset;
         ice::u32 data_size;
     };
diff --git a/source/code/core/utils/private/native_file.cxx b/source/code/core/utils/private/native_file.cxx
@@ -683,8 +683,7 @@ namespace ice::native_file
         ice::Data data
     ) noexcept -> ice::native_file::FileRequestStatus
     {
-        ICE_ASSERT(false, "Missing implementation!");
-        return FileRequestStatus::Error;
+        return aio_file_write_request(request, native_file, write_offset, data);
     }
 
     auto append_file(
@@ -860,7 +859,14 @@ namespace ice::native_file
         ice::String string
     ) noexcept
     {
-        ice::path::join(path, string);
+        if (ice::string::any(path))
+        {
+            ice::path::join(path, string);
+        }
+        else if (ice::string::any(string))
+        {
+            path = string;
+        }
     }
 
 #else
diff --git a/source/code/core/utils/private/params.cxx b/source/code/core/utils/private/params.cxx
@@ -227,17 +227,39 @@ namespace ice
             }
         }
 
+#if ISP_LINUX
+        static auto f = [](std::string p) noexcept -> std::string
+        {
+            size_t const start_quote = p.find_first_of('"');
+            size_t const end_quote = p.find_last_of('"');
+            if (start_quote != std::string::npos && end_quote != std::string::npos && start_quote < end_quote)
+            {
+                return p.substr(start_quote + 1, (end_quote - start_quote) - 1);
+            }
+            return p;
+        };
+#endif
+
         // Built-In Validators
         if (ice::has_all(definition.flags, PF::ValidatePath))
         {
+#if ISP_LINUX
+            opt->transform(f);
+#endif
             opt->check(CLI::ExistingPath);
         }
         else if (ice::has_all(definition.flags, PF::ValidateFile))
         {
+#if ISP_LINUX
+            opt->transform(f);
+#endif
             opt->check(CLI::ExistingFile);
         }
         else if (ice::has_all(definition.flags, PF::ValidateDirectory))
         {
+#if ISP_LINUX
+            opt->transform(f);
+#endif
             opt->check(CLI::ExistingDirectory);
         }
         return opt;
diff --git a/source/code/systems/resource_system/private/resource_filesystem_baked.cxx b/source/code/systems/resource_system/private/resource_filesystem_baked.cxx
@@ -88,8 +88,7 @@ namespace ice
         ice::Allocator& alloc,
         ice::ResourceFormatHeader const& header,
         ice::HeapString<> origin,
-        ice::HeapString<> name,
-        ice::Memory metadata
+        ice::HeapString<> name
     ) noexcept
         : _allocator{ alloc }
         , _header{ header }
@@ -213,20 +212,13 @@ namespace ice
         ICE_ASSERT_CORE(read >= 0_B);
         ice::string::push_back(utf8_uri, { temp, header.name_size });
 
-        ice::Memory metadata = alloc.allocate(ice::usize{ header.meta_size });
-        read = ice::native_file::read_file(
-            file, {header.meta_offset}, {header.meta_size}, metadata
-        );
-        ICE_ASSERT_CORE(read >= 0_B);
-
         return ice::create_resource_object<BakedFileResource>(
             alloc,
             provider,
             alloc,
             header,
             ice::move(utf8_file_path),
-            ice::move(utf8_uri),
-            metadata
+            ice::move(utf8_uri)
         );
     }
 
diff --git a/source/code/systems/resource_system/private/resource_filesystem_baked.hxx b/source/code/systems/resource_system/private/resource_filesystem_baked.hxx
@@ -23,8 +23,7 @@ namespace ice
             ice::Allocator& alloc,
             ice::ResourceFormatHeader const& header,
             ice::HeapString<> origin,
-            ice::HeapString<> name,
-            ice::Memory metadata
+            ice::HeapString<> name
         ) noexcept;
 
         ~BakedFileResource() noexcept override;
diff --git a/source/code/systems/resource_system/private/resource_provider_filesystem.cxx b/source/code/systems/resource_system/private/resource_provider_filesystem.cxx
@@ -206,7 +206,7 @@ namespace ice
             ice::HeapString<> uri_base{ _named_allocator };
             ice::string::push_format(uri_base, "file://{}/", _virtual_hostname);
 
-            create_resource_from_baked_file(_named_allocator, *this, uri_base, filepath);
+            return create_resource_from_baked_file(_named_allocator, *this, uri_base, filepath);
         }
         else
         {
diff --git a/source/code/tools/hsc_packer/private/hsc_packer_aiostream.cxx b/source/code/tools/hsc_packer/private/hsc_packer_aiostream.cxx
@@ -42,6 +42,7 @@ struct HailstormAIOWriter
     ice::Span<ice::ResourceHandle> _resources;
 
     std::atomic_uint32_t _started_writes;
+    std::atomic_uint32_t _finished_loads;
     std::atomic_uint32_t _finished_writes;
 
     ~HailstormAIOWriter() noexcept = default;
@@ -289,6 +290,7 @@ inline auto HailstormAIOWriter::async_write_resource(ice::u32 idx, ice::usize of
 {
     _started_writes.fetch_add(1, std::memory_order_relaxed);
     ice::ResourceResult const load_result = co_await _resource_tracker.load_resource(_resources[idx]);
+    _finished_loads.fetch_add(1, std::memory_order_relaxed);
     if (load_result.resource_status == ice::ResourceStatus::Loaded)
     {
         bool const success = co_await async_write(offset, data_to_hsdata(load_result.data));
diff --git a/source/code/tools/hsc_reader/private/hsc_reader.cxx b/source/code/tools/hsc_reader/private/hsc_reader.cxx
@@ -47,7 +47,7 @@ class HailStormReaderApp final : public ice::tool::ToolApp<HailStormReaderApp>
         // Open the pack file
         if (packfile_validate() == false)
         {
-            ICE_LOG(ice::LogSeverity::Retail, LogTag_Main, "Provide input file is not a valid Hailstorm pack.");
+            ICE_LOG(ice::LogSeverity::Retail, LogTag_Main, "Provided input file is not a valid Hailstorm pack.");
             return 1;
         }
 
diff --git a/source/code/tools/hsc_reader/private/hsc_reader_app.cxx b/source/code/tools/hsc_reader/private/hsc_reader_app.cxx
@@ -29,19 +29,20 @@ bool ParamRange::param_parse_results(ParamRange& range, ice::Span<ice::String co
 
 void hscr_initialize_logging() noexcept
 {
-    static const std::tuple<ice::LogTagDefinition const&, bool*> tags[]{
-        { LogTag_Main, &Param_HideHeader.value },
-        { LogTag_InfoChunks, &Param_ShowChunks.value.set },
-        { LogTag_InfoResources, &Param_ShowResources.value.set },
-        { LogTag_InfoPaths, &Param_ShowResourcePaths.value },
+    ice::log_tag_register(LogTag_Main);
+
+    static const std::tuple<ice::LogTagDefinition const&, bool> tags[]{
+        { LogTag_InfoHeader, Param_HideHeader.value == false },
+        { LogTag_InfoChunks, Param_ShowChunks.value.set },
+        { LogTag_InfoResources, Param_ShowResources.value.set },
+        { LogTag_InfoPaths, Param_ShowResourcePaths.value },
     };
 
     for (auto const& tag : tags)
     {
         ice::log_tag_register(std::get<0>(tag));
 
         // Check if the tag can be enabled.
-        bool* temp_param = std::get<1>(tag);
-        ice::log_tag_enable(std::get<0>(tag).tag, *temp_param);
+        ice::log_tag_enable(std::get<0>(tag).tag, std::get<1>(tag));
     }
 }
diff --git a/source/code/tools/hsc_reader/private/hsc_reader_app.hxx b/source/code/tools/hsc_reader/private/hsc_reader_app.hxx
@@ -50,5 +50,5 @@ static ice::ParamInstance<bool> Param_ShowCustomValues{
     "", "--custom-vals", "Shows app custom values (if the format supports them)."
 };
 static ice::ParamInstance<bool> Param_HideHeader{
-    "", "--hide,--header", "Shows header information."
+    "", "--hide", "Hides header information.",
 };
diff --git a/source/code/tools/hsc_reader/private/hsc_reader_funcs.cxx b/source/code/tools/hsc_reader/private/hsc_reader_funcs.cxx
@@ -42,6 +42,7 @@ void hailstorm_print_headerinfo(
             (Constant_HailstormHeaderVersion_0 & 0x00ff'0000) >> 16,
             (Constant_HailstormHeaderVersion_0 & 0x0000'ff00) >> 8,
             (Constant_HailstormHeaderVersion_0 & 0x0000'00ff) >> 0,
+            0
         };
 
         HSCR_INFO(LogTag_InfoHeader, "Base-Header");
diff --git a/source/data/shaders/shaders.bff b/source/data/shaders/shaders.bff
@@ -1,8 +1,6 @@
 /// Copyright 2022 - 2025, Dandielo <dandielo@iceshard.net>
 /// SPDX-License-Identifier: MIT
 
-#if __WINDOWS__ // Only on windows for now
-
 .Project =
 [
     .Name = 'data'
@@ -22,5 +20,3 @@
     ]
 ]
 .Projects + .Project
-
-#endif
diff --git a/source/fbuild.bff b/source/fbuild.bff
@@ -44,10 +44,13 @@ Settings
 [
     Using( .Pipeline_Linux_x64 )
 
-    .PipelineName = 'x64-Clang'
+    .PipelineName = 'x64'
     .PipelineToolchain = 'clang-20.0.0'
     .PipelineAllowUnityBuilds = true
 
+    // Projects from this pipeline may be exposed as compilers for other pipelines
+    .PipelineDefinesCompilers = true
+
     .PipelineConanProfile = 'Linux-Clang'
 ]
 

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,11 @@`
`6`	`6`	`.Name = 'Assets-Shaders-Mobile'`
`7`	`7`	`.Requires = { 'assets-mobile', 'Step-Bake-Shaders' }`
`8`	`8`	`.BuildOptions = {`
	`9`	`+#if __WINDOWS__`
`9`	`10`	`'-c shader_tools.dll'`
	`11`	`+#else // __LINUX__`
	`12`	`+ '-c libshader_tools.so'`
	`13`	`+#endif`
`10`	`14`	`'--param shader:target GLSL'`
`11`	`15`	`}`
`12`	`16`	`]`
`@@ -16,7 +20,11 @@`
`16`	`20`	`.Name = 'Assets-Shaders-Web'`
`17`	`21`	`.Requires = { 'assets-web', 'Step-Bake-Shaders' }`
`18`	`22`	`.BuildOptions = {`
	`23`	`+#if __WINDOWS__`
`19`	`24`	`'-c shader_tools.dll'`
	`25`	`+#else // __LINUX__`
	`26`	`+ '-c libshader_tools.so'`
	`27`	`+#endif`
`20`	`28`	`'--param shader:target WGSL'`
`21`	`29`	`}`
`22`	`30`	`]`
Original file line number	Diff line number	Diff line change
`@@ -683,8 +683,7 @@ namespace ice::native_file`
`683`	`683`	`ice::Data data`
`684`	`684`	`) noexcept -> ice::native_file::FileRequestStatus`
`685`	`685`	`{`
`686`		`- ICE_ASSERT(false, "Missing implementation!");`
`687`		`- return FileRequestStatus::Error;`
	`686`	`+ return aio_file_write_request(request, native_file, write_offset, data);`
`688`	`687`	`}`
`689`	`688`
`690`	`689`	`auto append_file(`
`@@ -860,7 +859,14 @@ namespace ice::native_file`
`860`	`859`	`ice::String string`
`861`	`860`	`) noexcept`
`862`	`861`	`{`
`863`		`- ice::path::join(path, string);`
	`862`	`+ if (ice::string::any(path))`
	`863`	`+ {`
	`864`	`+ ice::path::join(path, string);`
	`865`	`+ }`
	`866`	`+ else if (ice::string::any(string))`
	`867`	`+ {`
	`868`	`+ path = string;`
	`869`	`+ }`
`864`	`870`	`}`
`865`	`871`
`866`	`872`	`#else`
Original file line number	Diff line number	Diff line change
`@@ -227,17 +227,39 @@ namespace ice`
`227`	`227`	`}`
`228`	`228`	`}`
`229`	`229`
	`230`	`+#if ISP_LINUX`
	`231`	`+ static auto f = [](std::string p) noexcept -> std::string`
	`232`	`+ {`
	`233`	`+ size_t const start_quote = p.find_first_of('"');`
	`234`	`+ size_t const end_quote = p.find_last_of('"');`
	`235`	`+ if (start_quote != std::string::npos && end_quote != std::string::npos && start_quote < end_quote)`
	`236`	`+ {`
	`237`	`+ return p.substr(start_quote + 1, (end_quote - start_quote) - 1);`
	`238`	`+ }`
	`239`	`+ return p;`
	`240`	`+ };`
	`241`	`+#endif`
	`242`	`+`
`230`	`243`	`// Built-In Validators`
`231`	`244`	`if (ice::has_all(definition.flags, PF::ValidatePath))`
`232`	`245`	`{`
	`246`	`+#if ISP_LINUX`
	`247`	`+ opt->transform(f);`
	`248`	`+#endif`
`233`	`249`	`opt->check(CLI::ExistingPath);`
`234`	`250`	`}`
`235`	`251`	`else if (ice::has_all(definition.flags, PF::ValidateFile))`
`236`	`252`	`{`
	`253`	`+#if ISP_LINUX`
	`254`	`+ opt->transform(f);`
	`255`	`+#endif`
`237`	`256`	`opt->check(CLI::ExistingFile);`
`238`	`257`	`}`
`239`	`258`	`else if (ice::has_all(definition.flags, PF::ValidateDirectory))`
`240`	`259`	`{`
	`260`	`+#if ISP_LINUX`
	`261`	`+ opt->transform(f);`
	`262`	`+#endif`
`241`	`263`	`opt->check(CLI::ExistingDirectory);`
`242`	`264`	`}`
`243`	`265`	`return opt;`
Original file line number	Diff line number	Diff line change
`@@ -206,7 +206,7 @@ namespace ice`
`206`	`206`	`ice::HeapString<> uri_base{ _named_allocator };`
`207`	`207`	`ice::string::push_format(uri_base, "file://{}/", _virtual_hostname);`
`208`	`208`
`209`		`- create_resource_from_baked_file(_named_allocator, *this, uri_base, filepath);`
	`209`	`+ return create_resource_from_baked_file(_named_allocator, *this, uri_base, filepath);`
`210`	`210`	`}`
`211`	`211`	`else`
`212`	`212`	`{`
Original file line number	Diff line number	Diff line change
`@@ -42,6 +42,7 @@ struct HailstormAIOWriter`
`42`	`42`	`ice::Span<ice::ResourceHandle> _resources;`
`43`	`43`
`44`	`44`	`std::atomic_uint32_t _started_writes;`
	`45`	`+ std::atomic_uint32_t _finished_loads;`
`45`	`46`	`std::atomic_uint32_t _finished_writes;`
`46`	`47`
`47`	`48`	`~HailstormAIOWriter() noexcept = default;`
`@@ -289,6 +290,7 @@ inline auto HailstormAIOWriter::async_write_resource(ice::u32 idx, ice::usize of`
`289`	`290`	`{`
`290`	`291`	`_started_writes.fetch_add(1, std::memory_order_relaxed);`
`291`	`292`	`ice::ResourceResult const load_result = co_await _resource_tracker.load_resource(_resources[idx]);`
	`293`	`+ _finished_loads.fetch_add(1, std::memory_order_relaxed);`
`292`	`294`	`if (load_result.resource_status == ice::ResourceStatus::Loaded)`
`293`	`295`	`{`
`294`	`296`	`bool const success = co_await async_write(offset, data_to_hsdata(load_result.data));`
Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,7 @@ class HailStormReaderApp final : public ice::tool::ToolApp<HailStormReaderApp>`
`47`	`47`	`// Open the pack file`
`48`	`48`	`if (packfile_validate() == false)`
`49`	`49`	`{`
`50`		`- ICE_LOG(ice::LogSeverity::Retail, LogTag_Main, "Provide input file is not a valid Hailstorm pack.");`
	`50`	`+ ICE_LOG(ice::LogSeverity::Retail, LogTag_Main, "Provided input file is not a valid Hailstorm pack.");`
`51`	`51`	`return 1;`
`52`	`52`	`}`
`53`	`53`