diff --git a/crates/bevy_asset/src/io/memory.rs b/crates/bevy_asset/src/io/memory.rs index ac936b9be9a10..7bf69e570231a 100644 --- a/crates/bevy_asset/src/io/memory.rs +++ b/crates/bevy_asset/src/io/memory.rs @@ -1,13 +1,16 @@ -use crate::io::{AssetReader, AssetReaderError, PathStream, Reader}; -use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; +use crate::io::{AssetReader, AssetReaderError, AssetWriter, AssetWriterError, PathStream, Reader}; +use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec, vec::Vec}; use bevy_platform::{ collections::HashMap, sync::{PoisonError, RwLock}, }; use core::{pin::Pin, task::Poll}; -use futures_io::AsyncRead; +use futures_io::{AsyncRead, AsyncWrite}; use futures_lite::{ready, Stream}; -use std::path::{Path, PathBuf}; +use std::{ + io::{Error, ErrorKind}, + path::{Path, PathBuf}, +}; use super::AsyncSeekForward; @@ -59,7 +62,9 @@ impl Dir { ); } - /// Removes the stored asset at `path` and returns the `Data` stored if found and otherwise `None`. + /// Removes the stored asset at `path`. + /// + /// Returns the [`Data`] stored if found, [`None`] otherwise. pub fn remove_asset(&self, path: &Path) -> Option { let mut dir = self.clone(); if let Some(parent) = path.parent() { @@ -91,6 +96,22 @@ impl Dir { ); } + /// Removes the stored metadata at `path`. + /// + /// Returns the [`Data`] stored if found, [`None`] otherwise. + pub fn remove_metadata(&self, path: &Path) -> Option { + let mut dir = self.clone(); + if let Some(parent) = path.parent() { + dir = self.get_or_insert_dir(parent); + } + let key: Box = path.file_name().unwrap().to_string_lossy().into(); + dir.0 + .write() + .unwrap_or_else(PoisonError::into_inner) + .metadata + .remove(&key) + } + pub fn get_or_insert_dir(&self, path: &Path) -> Dir { let mut dir = self.clone(); let mut full_path = PathBuf::new(); @@ -108,6 +129,22 @@ impl Dir { dir } + /// Removes the dir at `path`. + /// + /// Returns the [`Dir`] stored if found, [`None`] otherwise. + pub fn remove_dir(&self, path: &Path) -> Option { + let mut dir = self.clone(); + if let Some(parent) = path.parent() { + dir = self.get_or_insert_dir(parent); + } + let key: Box = path.file_name().unwrap().to_string_lossy().into(); + dir.0 + .write() + .unwrap_or_else(PoisonError::into_inner) + .dirs + .remove(&key) + } + pub fn get_dir(&self, path: &Path) -> Option { let mut dir = self.clone(); for p in path.components() { @@ -215,6 +252,14 @@ pub struct MemoryAssetReader { pub root: Dir, } +/// In-memory [`AssetWriter`] implementation. +/// +/// This is primarily intended for unit tests. +#[derive(Default, Clone)] +pub struct MemoryAssetWriter { + pub root: Dir, +} + /// Asset data stored in a [`Dir`]. #[derive(Clone, Debug)] pub struct Data { @@ -230,10 +275,13 @@ pub enum Value { } impl Data { - fn path(&self) -> &Path { + /// The path that this data was written to. + pub fn path(&self) -> &Path { &self.path } - fn value(&self) -> &[u8] { + + /// The value in bytes that was written here. + pub fn value(&self) -> &[u8] { match &self.value { Value::Vec(vec) => vec, Value::Static(value) => value, @@ -296,8 +344,8 @@ impl AsyncSeekForward for DataReader { self.bytes_read = new_pos as _; Poll::Ready(Ok(new_pos as _)) } else { - Poll::Ready(Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, + Poll::Ready(Err(Error::new( + ErrorKind::InvalidInput, "seek position is out of range", ))) } @@ -361,6 +409,183 @@ impl AssetReader for MemoryAssetReader { } } +/// A writer that writes into [`Dir`], buffering internally until flushed/closed. +struct DataWriter { + /// The dir to write to. + dir: Dir, + /// The path to write to. + path: PathBuf, + /// The current buffer of data. + /// + /// This will include data that has been flushed already. + current_data: Vec, + /// Whether to write to the data or to the meta. + is_meta_writer: bool, +} + +impl AsyncWrite for DataWriter { + fn poll_write( + self: Pin<&mut Self>, + _: &mut core::task::Context<'_>, + buf: &[u8], + ) -> Poll> { + self.get_mut().current_data.extend_from_slice(buf); + Poll::Ready(Ok(buf.len())) + } + + fn poll_flush( + self: Pin<&mut Self>, + _: &mut core::task::Context<'_>, + ) -> Poll> { + // Write the data to our fake disk. This means we will repeatedly reinsert the asset. + if self.is_meta_writer { + self.dir.insert_meta(&self.path, self.current_data.clone()); + } else { + self.dir.insert_asset(&self.path, self.current_data.clone()); + } + Poll::Ready(Ok(())) + } + + fn poll_close( + self: Pin<&mut Self>, + cx: &mut core::task::Context<'_>, + ) -> Poll> { + // A flush will just write the data to Dir, which is all we need to do for close. + self.poll_flush(cx) + } +} + +impl AssetWriter for MemoryAssetWriter { + async fn write<'a>(&'a self, path: &'a Path) -> Result, AssetWriterError> { + Ok(Box::new(DataWriter { + dir: self.root.clone(), + path: path.to_owned(), + current_data: vec![], + is_meta_writer: false, + })) + } + + async fn write_meta<'a>( + &'a self, + path: &'a Path, + ) -> Result, AssetWriterError> { + Ok(Box::new(DataWriter { + dir: self.root.clone(), + path: path.to_owned(), + current_data: vec![], + is_meta_writer: true, + })) + } + + async fn remove<'a>(&'a self, path: &'a Path) -> Result<(), AssetWriterError> { + if self.root.remove_asset(path).is_none() { + return Err(AssetWriterError::Io(Error::new( + ErrorKind::NotFound, + "no such file", + ))); + } + Ok(()) + } + + async fn remove_meta<'a>(&'a self, path: &'a Path) -> Result<(), AssetWriterError> { + self.root.remove_metadata(path); + Ok(()) + } + + async fn rename<'a>( + &'a self, + old_path: &'a Path, + new_path: &'a Path, + ) -> Result<(), AssetWriterError> { + let Some(old_asset) = self.root.get_asset(old_path) else { + return Err(AssetWriterError::Io(Error::new( + ErrorKind::NotFound, + "no such file", + ))); + }; + self.root.insert_asset(new_path, old_asset.value); + // Remove the asset after instead of before since otherwise there'd be a moment where the + // Dir is unlocked and missing both the old and new paths. This just prevents race + // conditions. + self.root.remove_asset(old_path); + Ok(()) + } + + async fn rename_meta<'a>( + &'a self, + old_path: &'a Path, + new_path: &'a Path, + ) -> Result<(), AssetWriterError> { + let Some(old_meta) = self.root.get_metadata(old_path) else { + return Err(AssetWriterError::Io(Error::new( + ErrorKind::NotFound, + "no such file", + ))); + }; + self.root.insert_meta(new_path, old_meta.value); + // Remove the meta after instead of before since otherwise there'd be a moment where the + // Dir is unlocked and missing both the old and new paths. This just prevents race + // conditions. + self.root.remove_metadata(old_path); + Ok(()) + } + + async fn create_directory<'a>(&'a self, path: &'a Path) -> Result<(), AssetWriterError> { + // Just pretend we're on a file system that doesn't consider directory re-creation a + // failure. + self.root.get_or_insert_dir(path); + Ok(()) + } + + async fn remove_directory<'a>(&'a self, path: &'a Path) -> Result<(), AssetWriterError> { + if self.root.remove_dir(path).is_none() { + return Err(AssetWriterError::Io(Error::new( + ErrorKind::NotFound, + "no such dir", + ))); + } + Ok(()) + } + + async fn remove_empty_directory<'a>(&'a self, path: &'a Path) -> Result<(), AssetWriterError> { + let Some(dir) = self.root.get_dir(path) else { + return Err(AssetWriterError::Io(Error::new( + ErrorKind::NotFound, + "no such dir", + ))); + }; + + let dir = dir.0.read().unwrap(); + if !dir.assets.is_empty() || !dir.metadata.is_empty() || !dir.dirs.is_empty() { + return Err(AssetWriterError::Io(Error::new( + ErrorKind::DirectoryNotEmpty, + "not empty", + ))); + } + + self.root.remove_dir(path); + Ok(()) + } + + async fn remove_assets_in_directory<'a>( + &'a self, + path: &'a Path, + ) -> Result<(), AssetWriterError> { + let Some(dir) = self.root.get_dir(path) else { + return Err(AssetWriterError::Io(Error::new( + ErrorKind::NotFound, + "no such dir", + ))); + }; + + let mut dir = dir.0.write().unwrap(); + dir.assets.clear(); + dir.dirs.clear(); + dir.metadata.clear(); + Ok(()) + } +} + #[cfg(test)] pub mod test { use super::Dir; diff --git a/crates/bevy_asset/src/lib.rs b/crates/bevy_asset/src/lib.rs index 0f5aa40b67528..81deeff1450f5 100644 --- a/crates/bevy_asset/src/lib.rs +++ b/crates/bevy_asset/src/lib.rs @@ -247,6 +247,12 @@ pub struct AssetPlugin { /// Most use cases should leave this set to [`None`] and enable a specific watcher feature such as `file_watcher` to enable /// watching for dev-scenarios. pub watch_for_changes_override: Option, + /// If set, will override the default "use asset processor" setting. By default "use asset + /// processor" will be `false` unless the `asset_processor` cargo feature is set. + /// + /// Most use cases should leave this set to [`None`] and enable the `asset_processor` cargo + /// feature. + pub use_asset_processor_override: Option, /// The [`AssetMode`] to use for this server. pub mode: AssetMode, /// How/If asset meta files should be checked. @@ -332,6 +338,7 @@ impl Default for AssetPlugin { file_path: Self::DEFAULT_UNPROCESSED_FILE_PATH.to_string(), processed_file_path: Self::DEFAULT_PROCESSED_FILE_PATH.to_string(), watch_for_changes_override: None, + use_asset_processor_override: None, meta_check: AssetMetaCheck::default(), unapproved_path_mode: UnapprovedPathMode::default(), } @@ -360,10 +367,9 @@ impl Plugin for AssetPlugin { embedded.register_source(&mut sources); } { - let mut watch = cfg!(feature = "watch"); - if let Some(watch_override) = self.watch_for_changes_override { - watch = watch_override; - } + let watch = self + .watch_for_changes_override + .unwrap_or(cfg!(feature = "watch")); match self.mode { AssetMode::Unprocessed => { let mut builders = app.world_mut().resource_mut::(); @@ -378,8 +384,10 @@ impl Plugin for AssetPlugin { )); } AssetMode::Processed => { - #[cfg(feature = "asset_processor")] - { + let use_asset_processor = self + .use_asset_processor_override + .unwrap_or(cfg!(feature = "asset_processor")); + if use_asset_processor { let mut builders = app.world_mut().resource_mut::(); let processor = AssetProcessor::new(&mut builders); let mut sources = builders.build_sources(false, watch); @@ -395,9 +403,7 @@ impl Plugin for AssetPlugin { )) .insert_resource(processor) .add_systems(bevy_app::Startup, AssetProcessor::start); - } - #[cfg(not(feature = "asset_processor"))] - { + } else { let mut builders = app.world_mut().resource_mut::(); let sources = builders.build_sources(false, watch); app.insert_resource(AssetServer::new_with_meta_check( @@ -705,14 +711,16 @@ mod tests { handle::Handle, io::{ gated::{GateOpener, GatedReader}, - memory::{Dir, MemoryAssetReader}, + memory::{Dir, MemoryAssetReader, MemoryAssetWriter}, AssetReader, AssetReaderError, AssetSource, AssetSourceEvent, AssetSourceId, AssetWatcher, Reader, }, loader::{AssetLoader, LoadContext}, - Asset, AssetApp, AssetEvent, AssetId, AssetLoadError, AssetLoadFailedEvent, AssetPath, - AssetPlugin, AssetServer, Assets, InvalidGenerationError, LoadState, UnapprovedPathMode, - UntypedHandle, + saver::AssetSaver, + transformer::{AssetTransformer, TransformedAsset}, + Asset, AssetApp, AssetEvent, AssetId, AssetLoadError, AssetLoadFailedEvent, AssetMode, + AssetPath, AssetPlugin, AssetServer, Assets, InvalidGenerationError, LoadState, + UnapprovedPathMode, UntypedHandle, }; use alloc::{ boxed::Box, @@ -733,8 +741,9 @@ mod tests { sync::Mutex, }; use bevy_reflect::TypePath; - use core::time::Duration; + use core::{marker::PhantomData, time::Duration}; use crossbeam_channel::Sender; + use futures_lite::AsyncWriteExt; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use thiserror::Error; @@ -2232,4 +2241,315 @@ mod tests { Some(()) }); } + + #[expect(clippy::allow_attributes, reason = "this is only sometimes unused")] + #[allow( + unused, + reason = "We only use this for asset processor tests, which are only compiled with the `multi_threaded` feature." + )] + struct AppWithProcessor { + app: App, + source_dir: Dir, + processed_dir: Dir, + } + + #[expect(clippy::allow_attributes, reason = "this is only sometimes unused")] + #[allow( + unused, + reason = "We only use this for asset processor tests, which are only compiled with the `multi_threaded` feature." + )] + fn create_app_with_asset_processor() -> AppWithProcessor { + let mut app = App::new(); + let source_dir = Dir::default(); + let processed_dir = Dir::default(); + + let source_memory_reader = MemoryAssetReader { + root: source_dir.clone(), + }; + let processed_memory_reader = MemoryAssetReader { + root: processed_dir.clone(), + }; + let processed_memory_writer = MemoryAssetWriter { + root: processed_dir.clone(), + }; + + app.register_asset_source( + AssetSourceId::Default, + AssetSource::build() + .with_reader(move || Box::new(source_memory_reader.clone())) + .with_processed_reader(move || Box::new(processed_memory_reader.clone())) + .with_processed_writer(move |_| Some(Box::new(processed_memory_writer.clone()))), + ) + .add_plugins(( + TaskPoolPlugin::default(), + AssetPlugin { + mode: AssetMode::Processed, + use_asset_processor_override: Some(true), + ..Default::default() + }, + )); + + AppWithProcessor { + app, + source_dir, + processed_dir, + } + } + + #[expect(clippy::allow_attributes, reason = "this is only sometimes unused")] + #[allow( + unused, + reason = "We only use this for asset processor tests, which are only compiled with the `multi_threaded` feature." + )] + struct CoolTextSaver; + + impl AssetSaver for CoolTextSaver { + type Asset = CoolText; + type Settings = (); + type OutputLoader = CoolTextLoader; + type Error = std::io::Error; + + async fn save( + &self, + writer: &mut crate::io::Writer, + asset: crate::saver::SavedAsset<'_, Self::Asset>, + _: &Self::Settings, + ) -> Result<(), Self::Error> { + let ron = CoolTextRon { + text: asset.text.clone(), + sub_texts: asset + .iter_labels() + .map(|label| asset.get_labeled::(label).unwrap().text.clone()) + .collect(), + dependencies: asset + .dependencies + .iter() + .map(|handle| handle.path().unwrap().path()) + .map(|path| path.to_str().unwrap().to_string()) + .collect(), + // NOTE: We can't handle embedded dependencies in any way, since we need to write to + // another file to do so. + embedded_dependencies: vec![], + }; + let ron = ron::ser::to_string(&ron).unwrap(); + writer.write_all(ron.as_bytes()).await?; + Ok(()) + } + } + + #[expect(clippy::allow_attributes, reason = "this is only sometimes unused")] + #[allow( + unused, + reason = "We only use this for asset processor tests, which are only compiled with the `multi_threaded` feature." + )] + // Note: while we allow any Fn, since closures are unnameable types, creating a processor with a + // closure cannot be used (since we need to include the name of the transformer in the meta + // file). + struct RootAssetTransformer, A: Asset>(M, PhantomData); + + trait MutateAsset: Send + Sync + 'static { + fn mutate(&self, asset: &mut A); + } + + impl, A: Asset> RootAssetTransformer { + #[expect(clippy::allow_attributes, reason = "this is only sometimes unused")] + #[allow( + unused, + reason = "We only use this for asset processor tests, which are only compiled with the `multi_threaded` feature." + )] + fn new(m: M) -> Self { + Self(m, PhantomData) + } + } + + impl, A: Asset> AssetTransformer for RootAssetTransformer { + type AssetInput = A; + type AssetOutput = A; + type Error = std::io::Error; + type Settings = (); + + async fn transform<'a>( + &'a self, + mut asset: TransformedAsset, + _settings: &'a Self::Settings, + ) -> Result, Self::Error> { + self.0.mutate(asset.get_mut()); + Ok(asset) + } + } + + #[cfg(feature = "multi_threaded")] + use crate::processor::{AssetProcessor, LoadTransformAndSave}; + + // The asset processor currently requires multi_threaded. + #[cfg(feature = "multi_threaded")] + #[test] + fn no_meta_or_default_processor_copies_asset() { + // Assets without a meta file or a default processor should still be accessible in the + // processed path. Note: This isn't exactly the desired property - we don't want the assets + // to be copied to the processed directory. We just want these assets to still be loadable + // if we no longer have the source directory. This could be done with a symlink instead of a + // copy. + + let AppWithProcessor { + mut app, + source_dir, + processed_dir, + } = create_app_with_asset_processor(); + + let path = Path::new("abc.cool.ron"); + let source_asset = r#"( + text: "abc", + dependencies: [], + embedded_dependencies: [], + sub_texts: [], +)"#; + + source_dir.insert_asset_text(path, source_asset); + + // Start the app, which also starts the asset processor. + app.update(); + + // Wait for all processing to finish. + bevy_tasks::block_on( + app.world() + .resource::() + .data() + .wait_until_finished(), + ); + + let processed_asset = processed_dir.get_asset(path).unwrap(); + let processed_asset = str::from_utf8(processed_asset.value()).unwrap(); + assert_eq!(processed_asset, source_asset); + } + + // The asset processor currently requires multi_threaded. + #[cfg(feature = "multi_threaded")] + #[test] + fn asset_processor_transforms_asset_default_processor() { + let AppWithProcessor { + mut app, + source_dir, + processed_dir, + } = create_app_with_asset_processor(); + + struct AddText; + + impl MutateAsset for AddText { + fn mutate(&self, text: &mut CoolText) { + text.text.push_str("_def"); + } + } + + type CoolTextProcessor = LoadTransformAndSave< + CoolTextLoader, + RootAssetTransformer, + CoolTextSaver, + >; + app.register_asset_loader(CoolTextLoader) + .register_asset_processor(CoolTextProcessor::new( + RootAssetTransformer::new(AddText), + CoolTextSaver, + )) + .set_default_asset_processor::("cool.ron"); + + let path = Path::new("abc.cool.ron"); + source_dir.insert_asset_text( + path, + r#"( + text: "abc", + dependencies: [], + embedded_dependencies: [], + sub_texts: [], +)"#, + ); + + // Start the app, which also starts the asset processor. + app.update(); + + // Wait for all processing to finish. + bevy_tasks::block_on( + app.world() + .resource::() + .data() + .wait_until_finished(), + ); + + let processed_asset = processed_dir.get_asset(path).unwrap(); + let processed_asset = str::from_utf8(processed_asset.value()).unwrap(); + assert_eq!( + processed_asset, + r#"(text:"abc_def",dependencies:[],embedded_dependencies:[],sub_texts:[])"# + ); + } + + // The asset processor currently requires multi_threaded. + #[cfg(feature = "multi_threaded")] + #[test] + fn asset_processor_transforms_asset_with_meta() { + let AppWithProcessor { + mut app, + source_dir, + processed_dir, + } = create_app_with_asset_processor(); + + struct AddText; + + impl MutateAsset for AddText { + fn mutate(&self, text: &mut CoolText) { + text.text.push_str("_def"); + } + } + + type CoolTextProcessor = LoadTransformAndSave< + CoolTextLoader, + RootAssetTransformer, + CoolTextSaver, + >; + app.register_asset_loader(CoolTextLoader) + .register_asset_processor(CoolTextProcessor::new( + RootAssetTransformer::new(AddText), + CoolTextSaver, + )); + + let path = Path::new("abc.cool.ron"); + source_dir.insert_asset_text( + path, + r#"( + text: "abc", + dependencies: [], + embedded_dependencies: [], + sub_texts: [], +)"#, + ); + source_dir.insert_meta_text(path, r#"( + meta_format_version: "1.0", + asset: Process( + processor: "bevy_asset::processor::process::LoadTransformAndSave, bevy_asset::tests::CoolTextSaver>", + settings: ( + loader_settings: (), + transformer_settings: (), + saver_settings: (), + ), + ), +)"#); + + // Start the app, which also starts the asset processor. + app.update(); + + // Wait for all processing to finish. + bevy_tasks::block_on( + app.world() + .resource::() + .data() + .wait_until_finished(), + ); + + let processed_asset = processed_dir.get_asset(path).unwrap(); + let processed_asset = str::from_utf8(processed_asset.value()).unwrap(); + assert_eq!( + processed_asset, + r#"(text:"abc_def",dependencies:[],embedded_dependencies:[],sub_texts:[])"# + ); + } } diff --git a/release-content/migration-guides/asset_plugin_processed_override.md b/release-content/migration-guides/asset_plugin_processed_override.md new file mode 100644 index 0000000000000..0bb7b5e7e044f --- /dev/null +++ b/release-content/migration-guides/asset_plugin_processed_override.md @@ -0,0 +1,8 @@ +--- +title: AssetPlugin now has a `use_asset_processor_override` field. +pull_requests: [21409] +--- + +The `AssetPlugin` now has a `use_asset_processor_override` field. If you were previously setting all +`AssetPlugin` fields, you should either use struct update syntax `..Default::default()`, or set the +new `use_asset_processor_override` to `None`. diff --git a/tools/example-showcase/asset-source-website.patch b/tools/example-showcase/asset-source-website.patch index c2b7bf47865a4..c24df487f9ee5 100644 --- a/tools/example-showcase/asset-source-website.patch +++ b/tools/example-showcase/asset-source-website.patch @@ -1,8 +1,8 @@ diff --git a/crates/bevy_asset/src/lib.rs b/crates/bevy_asset/src/lib.rs -index ea8caf003..1b4b4bbf9 100644 +index 81deeff145..7792d9f1ed 100644 --- a/crates/bevy_asset/src/lib.rs +++ b/crates/bevy_asset/src/lib.rs -@@ -134,7 +134,7 @@ impl Default for AssetPlugin { +@@ -335,7 +335,7 @@ fn default() -> Self { Self { mode: AssetMode::Unprocessed, @@ -10,4 +10,4 @@ index ea8caf003..1b4b4bbf9 100644 + file_path: "/assets/examples".to_string(), processed_file_path: Self::DEFAULT_PROCESSED_FILE_PATH.to_string(), watch_for_changes_override: None, - meta_check: AssetMetaCheck::default(), + use_asset_processor_override: None,