[iceshard] Introduced ResourceFilter interface to allow systems to filter for resources.

Dandielo · Dandielo · commit fe5904453342 · 2025-12-14T15:15:32.000+09:00
* Only meant to be used for editor / development tools.
* Removed old remains of the Metadata type. (Was replaced with ice::Config)
diff --git a/source/code/systems/asset_system/public/ice/asset_category_archive.hxx b/source/code/systems/asset_system/public/ice/asset_category_archive.hxx
@@ -12,6 +12,9 @@ namespace ice
 
     struct AssetCategoryDefinition
     {
+        constexpr auto valid() const noexcept { return ice::span::any(resource_extensions); }
+
+        ice::String name;
         ice::Span<ice::Shard const> asset_params;
         ice::Span<ice::String const> resource_extensions;
 
diff --git a/source/code/systems/resource_system/natvis/resource.natvis b/source/code/systems/resource_system/natvis/resource.natvis
@@ -3,16 +3,19 @@
 
   <!-- URI Type -->
   <Type Name="ice::URI">
-    <DisplayString Condition="_forced_scheme == 0">URI {{ path={_uri,sb} }}</DisplayString>
-    <DisplayString Condition="_forced_scheme == 1">URI {{ urn, path={_uri,s8b} }}</DisplayString>
-    <DisplayString Condition="_forced_scheme == 2">URI {{ file, path={_uri,s8b} }}</DisplayString>
+    <DisplayString Condition="_forced_scheme == 0">URI {{ urn, path={_uri,s8b} }}</DisplayString>
+    <DisplayString Condition="_forced_scheme == 1">URI {{ file, path={_uri,s8b} }}</DisplayString>
+    <DisplayString Condition="_forced_scheme == 2">URI {{ hailstorm, path={_uri,s8b} }}</DisplayString>
     <DisplayString Condition="_forced_scheme == 3">URI {{ dynlib, path={_uri,s8b} }}</DisplayString>
+    <DisplayString Condition="_forced_scheme == 4">URI {{ path={_uri,sb} }}</DisplayString>
 
     <Expand>
       <Synthetic Name="[scheme]" Condition="_forced_scheme != 0">
-        <DisplayString Condition="_forced_scheme == 1">urn</DisplayString>
-        <DisplayString Condition="_forced_scheme == 2">file</DisplayString>
+        <DisplayString Condition="_forced_scheme == 0">urn</DisplayString>
+        <DisplayString Condition="_forced_scheme == 1">file</DisplayString>
+        <DisplayString Condition="_forced_scheme == 2">hailstorm</DisplayString>
         <DisplayString Condition="_forced_scheme == 3">dynlib</DisplayString>
+        <DisplayString Condition="_forced_scheme == 4">invalid</DisplayString>
       </Synthetic>
       <Item Name="[path]">_uri,sb</Item>
     </Expand>
diff --git a/source/code/systems/resource_system/private/resource_filesystem_loose.cxx b/source/code/systems/resource_system/private/resource_filesystem_loose.cxx
@@ -56,6 +56,7 @@ namespace ice
                 ice::string::push_back(native_filepath, ISP_PATH_LITERAL(".isrm"));
             }
 
+            IPT_MESSAGE_STR(filepath);
             ice::Expected<ice::native_file::File> handle = ice::native_file::open_file(aioport, native_filepath);
             if (handle)
             {
diff --git a/source/code/systems/resource_system/private/resource_provider_filesystem.cxx b/source/code/systems/resource_system/private/resource_provider_filesystem.cxx
@@ -4,6 +4,8 @@
 #include "resource_provider_filesystem.hxx"
 #include "resource_provider_hailstorm.hxx"
 #include "resource_filesystem_baked.hxx"
+#include <ice/resource_filter.hxx>
+#include <ice/config.hxx>
 
 namespace ice
 {
@@ -52,6 +54,58 @@ namespace ice
         return ice::Scheme_File;
     }
 
+    auto FileSystemResourceProvider::filter_resource_uris(
+        ice::ResourceFilter const& filter,
+        ice::Array<ice::URI>& out_uris
+    ) noexcept -> ice::TaskExpected<ice::ucount>
+    {
+        ice::ucount collected = 0;
+        for (ice::FileSystemResource const* resource : _resources)
+        {
+            if (filter.allows_resource(resource))
+            {
+                ice::Data metadata_data{};
+                if (filter.requires_metadata())
+                {
+                    metadata_data = co_await load_resource(resource, "meta");
+
+                    if (filter.filter_thread() != nullptr)
+                    {
+                        co_await *filter.filter_thread();
+                    }
+
+                    if (metadata_data.location == nullptr || metadata_data.size == 0_B)
+                    {
+                        continue;
+                    }
+
+                    ice::Config metadata{};
+                    ice::Memory metadata_mem{};
+                    if (reinterpret_cast<char const*>(metadata_data.location)[0] == '{')
+                    {
+                        metadata = ice::config::from_json(_named_allocator, ice::string::from_data(metadata_data), metadata_mem);
+                    }
+                    else
+                    {
+                        metadata = ice::config::from_data(metadata_data);
+                    }
+
+                    if (filter.allows_metadata(metadata) == false)
+                    {
+                        _named_allocator.deallocate(metadata_mem);
+                        continue;
+                    }
+
+                    _named_allocator.deallocate(metadata_mem);
+                }
+
+                ice::array::push_back(out_uris, resource->uri());
+                collected += 1;
+            }
+        }
+        co_return collected;
+    }
+
     auto FileSystemResourceProvider::collect(
         ice::Array<ice::Resource*>& out_changes
     ) noexcept -> ice::ucount
diff --git a/source/code/systems/resource_system/private/resource_provider_filesystem.hxx b/source/code/systems/resource_system/private/resource_provider_filesystem.hxx
@@ -44,6 +44,11 @@ namespace ice
 
         auto schemeid() const noexcept -> ice::StringID override;
 
+        auto filter_resource_uris(
+            ice::ResourceFilter const& filter,
+            ice::Array<ice::URI>& out_uris
+        ) noexcept -> ice::TaskExpected<ice::ucount> override;
+
         auto collect(
             ice::Array<ice::Resource*>& out_changes
         ) noexcept -> ice::ucount override;
diff --git a/source/code/systems/resource_system/private/resource_tracker.cxx b/source/code/systems/resource_system/private/resource_tracker.cxx
@@ -3,6 +3,7 @@
 
 #include "resource_tracker.hxx"
 #include "resource_internal.hxx"
+#include <ice/resource_filter.hxx>
 
 #include <ice/sync_manual_events.hxx>
 
@@ -80,14 +81,17 @@ namespace ice
     }
 
     auto ResourceTrackerImplementation::attach_writer(
-        ice::UniquePtr<ice::ResourceWriter> provider
+        ice::UniquePtr<ice::ResourceWriter> writer
     ) noexcept -> ice::ResourceWriter*
     {
-        ice::ResourceWriter* const result = provider.get();
+        ice::ResourceWriter* const result = writer.get();
+        ice::ResourceProvider* const provider = attach_provider(ice::move(writer));
+        ICE_ASSERT_CORE(provider != nullptr);
+
         ice::multi_hashmap::insert(
             _resource_writers,
-            ice::hash(provider->schemeid()),
-            ice::move(provider)
+            ice::hash(result->schemeid()),
+            result
         );
         return result;
     }
@@ -104,12 +108,14 @@ namespace ice
 
             this->sync_provider(temp_resources, *provider);
         }
+#if 0
         for (auto const& writer : _resource_writers)
         {
             ice::array::clear(temp_resources);
 
             this->sync_provider(temp_resources, *writer);
         }
+#endif
     }
 
     auto ResourceTrackerImplementation::find_resource(
@@ -143,6 +149,41 @@ namespace ice
         return result;
     }
 
+    auto ResourceTrackerImplementation::filter_resource_provider_uris(
+        ice::ResourceProvider& provider,
+        ice::ResourceFilter const& filter,
+        ice::Array<ice::URI>& out_uris
+    ) const noexcept -> ice::TaskExpected<ice::ucount>
+    {
+        co_return co_await provider.filter_resource_uris(filter, out_uris);
+    }
+
+    auto ResourceTrackerImplementation::filter_resource_uris(
+        ice::ResourceFilter const& filter,
+        ice::Array<ice::URI>& out_uris
+    ) const noexcept -> ice::TaskExpected<ice::ucount>
+    {
+        ice::ucount collected = 0;
+        for (ice::UniquePtr<ice::ResourceProvider> const& provider : _resource_providers)
+        {
+            static constexpr ice::String const keywords[]{ "blocked", "allowed" };
+            bool const scheme_allowed = filter.allows_scheme(provider->schemeid());
+            bool const hostname_allowed = filter.allows_hostname(provider->hostname());
+            ICE_LOG_IF(
+                !scheme_allowed || !hostname_allowed,
+                LogSeverity::Debug, LogTag::Engine,
+                "Provider was filtered out. [scheme: {} ({}), hostname: {} ({})]",
+                provider->schemeid(), keywords[i32(scheme_allowed)],
+                provider->hostname(), keywords[i32(hostname_allowed)]
+            );
+            if (scheme_allowed && hostname_allowed)
+            {
+                collected += co_await filter_resource_provider_uris(*provider, filter, out_uris);
+            }
+        }
+        co_return collected;
+    }
+
     auto ResourceTrackerImplementation::set_resource(
         ice::URI const& uri,
         ice::ResourceHandle const& resource_handle
@@ -239,7 +280,7 @@ namespace ice
         while (it != nullptr)
         {
             [[maybe_unused]]
-            ice::ResourceWriter* candidate_writer = it.value().get();
+            ice::ResourceWriter* candidate_writer = it.value();
 
             // We only allow to create resources for specific writers.
             if (candidate_writer->hostname() == resource_uri.host())
diff --git a/source/code/systems/resource_system/private/resource_tracker.hxx b/source/code/systems/resource_system/private/resource_tracker.hxx
@@ -149,6 +149,17 @@ namespace ice
             ice::ResourceHandle const& resource_handle
         ) const noexcept -> ice::ResourceHandle override;
 
+        auto filter_resource_provider_uris(
+            ice::ResourceProvider& provider,
+            ice::ResourceFilter const& filter,
+            ice::Array<ice::URI>& out_uris
+        ) const noexcept -> ice::TaskExpected<ice::ucount>;
+
+        auto filter_resource_uris(
+            ice::ResourceFilter const& filter,
+            ice::Array<ice::URI>& out_uris
+        ) const noexcept -> ice::TaskExpected<ice::ucount>  override;
+
 
         auto set_resource(
             ice::URI const& uri,
@@ -217,7 +228,7 @@ namespace ice
 
         ice::HashMap<ice::Resource*> _resources;
         ice::HashMap<ice::UniquePtr<ice::ResourceProvider>, ContainerLogic::Complex> _resource_providers;
-        ice::HashMap<ice::UniquePtr<ice::ResourceWriter>, ContainerLogic::Complex> _resource_writers;
+        ice::HashMap<ice::ResourceWriter*> _resource_writers;
 
         ice::UniquePtr<ice::DevUIWidget> _devui_widget;
     };
diff --git a/source/code/systems/resource_system/private/resource_writer_filesystem.cxx b/source/code/systems/resource_system/private/resource_writer_filesystem.cxx
@@ -3,6 +3,7 @@
 
 #include "resource_writer_filesystem.hxx"
 #include "resource_filesystem_writable.hxx"
+#include <ice/resource_filter.hxx>
 
 namespace ice
 {
@@ -48,6 +49,58 @@ namespace ice
         return ice::Scheme_File;
     }
 
+    auto FileSystemResourceWriter::filter_resource_uris(
+        ice::ResourceFilter const& filter,
+        ice::Array<ice::URI>& out_uris
+    ) noexcept -> ice::TaskExpected<ice::ucount>
+    {
+        ice::ucount collected = 0;
+        for (ice::FileSystemResource const* resource : _resources)
+        {
+            if (filter.allows_resource(resource))
+            {
+                ice::Data metadata_data{};
+                if (filter.requires_metadata())
+                {
+                    metadata_data = co_await load_resource(resource, "meta");
+
+                    if (filter.filter_thread() != nullptr)
+                    {
+                        co_await *filter.filter_thread();
+                    }
+
+                    if (metadata_data.location == nullptr || metadata_data.size == 0_B)
+                    {
+                        continue;
+                    }
+
+                    ice::Config metadata{};
+                    ice::Memory metadata_mem{};
+                    if (reinterpret_cast<char const*>(metadata_data.location)[0] == '{')
+                    {
+                        metadata = ice::config::from_json(_named_allocator, ice::string::from_data(metadata_data), metadata_mem);
+                    }
+                    else
+                    {
+                        metadata = ice::config::from_data(metadata_data);
+                    }
+
+                    if (filter.allows_metadata(metadata) == false)
+                    {
+                        _named_allocator.deallocate(metadata_mem);
+                        continue;
+                    }
+
+                    _named_allocator.deallocate(metadata_mem);
+                }
+
+                ice::array::push_back(out_uris, resource->uri());
+                collected += 1;
+            }
+        }
+        co_return collected;
+    }
+
     auto FileSystemResourceWriter::collect(
         ice::Array<ice::Resource*>& out_changes
     ) noexcept -> ice::ucount
diff --git a/source/code/systems/resource_system/private/resource_writer_filesystem.hxx b/source/code/systems/resource_system/private/resource_writer_filesystem.hxx
@@ -43,6 +43,11 @@ namespace ice
 
         auto hostname() const noexcept -> ice::String override { return _virtual_hostname; }
 
+        auto filter_resource_uris(
+            ice::ResourceFilter const& filter,
+            ice::Array<ice::URI>& out_uris
+        ) noexcept -> ice::TaskExpected<ice::ucount> override;
+
         auto collect(
             ice::Array<ice::Resource*>& out_changes
         ) noexcept -> ice::ucount override;
diff --git a/source/code/systems/resource_system/public/ice/resource.hxx b/source/code/systems/resource_system/public/ice/resource.hxx
@@ -11,8 +11,6 @@
 namespace ice
 {
 
-    struct Metadata;
-
     class Resource
     {
     public:
diff --git a/source/code/systems/resource_system/public/ice/resource_filter.hxx b/source/code/systems/resource_system/public/ice/resource_filter.hxx
@@ -0,0 +1,57 @@
+#pragma once
+#include <ice/config.hxx>
+#include <ice/path_utils.hxx>
+#include <ice/resource.hxx>
+#include <ice/uri.hxx>
+
+namespace ice
+{
+
+    class ResourceFilter
+    {
+    public:
+        virtual ~ResourceFilter() noexcept = default;
+
+        virtual auto filter_thread() const noexcept -> ice::TaskScheduler* { return nullptr; }
+
+        virtual bool requires_metadata() const noexcept { return false; }
+
+        virtual bool allows_scheme(ice::StringID_Arg scheme) const noexcept { return true; }
+
+        virtual bool allows_hostname(ice::String hostname) const noexcept { return true; }
+
+        virtual bool allows_resource(
+            ice::Resource const* resource
+        ) const noexcept = 0;
+
+        virtual bool allows_metadata(
+            ice::Config const& metadata
+        ) const noexcept { return false; }
+    };
+
+    class FileResourceFilter : public ResourceFilter
+    {
+    public:
+        FileResourceFilter(ice::String extension, ice::String hostname = {}) noexcept
+            : _extension{ extension }
+            , _hostname{ hostname }
+        { }
+
+        bool allows_hostname(ice::String hostname) const noexcept override
+        {
+            return ice::string::empty(_hostname) || hostname == _hostname;
+        }
+
+        bool allows_resource(
+            ice::Resource const* resource
+        ) const noexcept override
+        {
+            return ice::string::empty(_extension) || ice::path::extension(resource->origin()) == _extension;
+        }
+
+    private:
+        ice::String _extension;
+        ice::String _hostname;
+    };
+
+} // namespace ice
diff --git a/source/code/systems/resource_system/public/ice/resource_provider.hxx b/source/code/systems/resource_system/public/ice/resource_provider.hxx
diff --git a/source/code/systems/resource_system/public/ice/resource_tracker.hxx b/source/code/systems/resource_system/public/ice/resource_tracker.hxx
diff --git a/source/code/systems/resource_system/public/ice/resource_types.hxx b/source/code/systems/resource_system/public/ice/resource_types.hxx
diff --git a/source/code/systems/resource_system/public/ice/uri.hxx b/source/code/systems/resource_system/public/ice/uri.hxx

Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,9 @@ namespace ice`
`12`	`12`
`13`	`13`	`struct AssetCategoryDefinition`
`14`	`14`	`{`
	`15`	`+ constexpr auto valid() const noexcept { return ice::span::any(resource_extensions); }`
	`16`	`+`
	`17`	`+ ice::String name;`
`15`	`18`	`ice::Span<ice::Shard const> asset_params;`
`16`	`19`	`ice::Span<ice::String const> resource_extensions;`
`17`	`20`
Original file line number	Diff line number	Diff line change
`@@ -56,6 +56,7 @@ namespace ice`
`56`	`56`	`ice::string::push_back(native_filepath, ISP_PATH_LITERAL(".isrm"));`
`57`	`57`	`}`
`58`	`58`
	`59`	`+ IPT_MESSAGE_STR(filepath);`
`59`	`60`	`ice::Expected<ice::native_file::File> handle = ice::native_file::open_file(aioport, native_filepath);`
`60`	`61`	`if (handle)`
`61`	`62`	`{`
Original file line number	Diff line number	Diff line change
`@@ -11,8 +11,6 @@`
`11`	`11`	`namespace ice`
`12`	`12`	`{`
`13`	`13`
`14`		`- struct Metadata;`
`15`		`-`
`16`	`14`	`class Resource`
`17`	`15`	`{`
`18`	`16`	`public:`