diff --git a/docs/scripting/AnimationClip.md b/docs/scripting/AnimationClip.md new file mode 100644 index 0000000000..eea62fc50e --- /dev/null +++ b/docs/scripting/AnimationClip.md @@ -0,0 +1,326 @@ +# Animation Clip + +## Overview +`AnimationClip` is the core class in the Galacean engine for storing keyframe-based animation data. It contains animation curve bindings, animation events, and timeline information, serving as the fundamental data unit of the animation system. Each `AnimationClip` is an independent animation segment that can be reused by multiple animation states. Clips can be authored in the Galacean editor, imported with animated assets, or created programmatically. + +## Core Architecture + +### Main Components +- **AnimationClip**: The main class for animation clips, storing keyframe animations. +- **AnimationClipCurveBinding**: Binds animation curves to entity component properties. +- **AnimationEvent**: Triggers callbacks at specific points in time. +- **AnimationCurve**: Stores keyframe data and interpolation information. + +## API Reference + +### Core Methods of AnimationClip + +#### Basic Properties +```typescript +// The name of the animation clip +readonly name: string + +// The length of the animation in seconds +readonly length: number + +// An array of animation events +readonly events: Readonly + +// An array of curve bindings +readonly curveBindings: Readonly +``` + +#### Animation Event Management +```typescript +// Add an animation event - created via parameters +addEvent(functionName: string, time: number, parameter: Object | undefined): void + +// Add an animation event - by passing the event object directly +addEvent(event: AnimationEvent): void + +// Clear all events +clearEvents(): void + +// Example usage +clip.addEvent("onFootStep", 0.5, { footIndex: 0 }); +clip.addEvent("onAttackHit", 0.8, { damage: 50 }); +clip.addEvent("onJumpLand", 1.1, undefined); // Explicitly pass undefined when no payload is needed +``` + +#### Curve Binding Management +```typescript +// Add a curve binding - basic version +addCurveBinding( + entityPath: string, + componentType: new (entity: Entity) => T, + propertyPath: string, + curve: AnimationCurve +): void + +// Add a curve binding - with separate read/write paths +addCurveBinding( + entityPath: string, + componentType: new (entity: Entity) => T, + setPropertyPath: string, + getPropertyPath: string, + curve: AnimationCurve +): void + +// Add a curve binding - specifying the component index +addCurveBinding( + entityPath: string, + componentType: new (entity: Entity) => T, + componentIndex: number, + propertyPath: string, + curve: AnimationCurve +): void + +// Add a curve binding - specifying the component index with separate read/write paths +addCurveBinding( + entityPath: string, + componentType: new (entity: Entity) => T, + componentIndex: number, + setPropertyPath: string, + getPropertyPath: string, + curve: AnimationCurve +): void + +// Clear all curve bindings (resets clip length to 0) +clearCurveBindings(): void +``` + +### AnimationClipCurveBinding Configuration + +#### Basic Properties +```typescript +// Relative path to the target entity +relativePath: string // e.g., "root/spine/leftArm" + +// Target component type +type: new (entity: Entity) => Component + +// Component index (for multiple components of the same type) +typeIndex: number = 0 + +// Property path (for setting the value) +property: string // Supports: "a.b", "a.b[0]", "a.b('c', 0, $value)" + +// Get property path (optional, for reading the current value) +getProperty?: string // Supports: "a.b", "a.b[0]", "a.b('c', 0)" + +// The animation curve +curve: AnimationCurve +``` + +### AnimationEvent Configuration + +#### Basic Properties +```typescript +// Trigger time in seconds +time: number + +// Function name +functionName: string + +// Parameter to pass to the function +parameter: Object +``` + +## Property Path Syntax + +### Supported Path Formats +```typescript +// 1. Simple property access +"position.x" // Access the x property of position +"rotation.y" // Access the y property of rotation +"material.baseColor" // Access the baseColor property of material + +// 2. Array index access +"materials[0].baseColor" // Access the baseColor of the first material +"bones[2].rotation.x" // Access the x rotation value of the third bone + +// 3. Method call (for setting values only) +"setPosition('x', $value)" // Call the setPosition method +"transform.setRotation('y', $value)" // Call the setRotation method of transform +"material.setFloat('metallic', $value)" // Set a float parameter of the material +``` + +### Path Parsing Rules +- **$value**: A placeholder representing the value calculated by the animation curve. +- **Dot notation**: Represents object property access. +- **Square brackets**: Represent array or index access. +- **Parentheses**: Represent a method call, with parameters separated by commas. + +## Usage Examples + +### Creating a Basic Animation Clip +```typescript +// Create an animation clip +const walkClip = new AnimationClip("walk"); + +// Create a position animation curve +const positionCurve = new AnimationCurve(); +positionCurve.addKey(0, new Vector3(0, 0, 0)); +positionCurve.addKey(1, new Vector3(0, 0, 5)); + +// Add a curve binding +walkClip.addCurveBinding( + "", // Root entity + Transform, // Transform component + "position", // position property + positionCurve // The animation curve +); +``` + +### Complex Property Animation +```typescript +// Material color animation +const colorClip = new AnimationClip("colorChange"); + +// Create a color curve +const colorCurve = new AnimationCurve(); +colorCurve.addKey(0, new Color(1, 0, 0, 1)); // Red +colorCurve.addKey(1, new Color(0, 0, 1, 1)); // Blue + +// Bind to the material's baseColor property +colorClip.addCurveBinding( + "", // Root entity + MeshRenderer, // MeshRenderer component + "material.baseColor", // material base color + colorCurve +); +``` + +### Bone Animation Binding +```typescript +// Bone rotation animation +const boneRotationClip = new AnimationClip("boneAnimation"); + +// Left arm rotation curve +const leftArmRotationCurve = new AnimationCurve(); +leftArmRotationCurve.addKey(0, new Vector3(0, 0, 0)); +leftArmRotationCurve.addKey(0.5, new Vector3(45, 0, 0)); +leftArmRotationCurve.addKey(1, new Vector3(0, 0, 0)); + +// Bind to the left arm bone +boneRotationClip.addCurveBinding( + "root/spine/leftShoulder/leftArm", // Bone path + Transform, // Transform component + "rotation", // rotation property + leftArmRotationCurve +); +``` + +### Multi-component Animation +```typescript +// Animate multiple components simultaneously +const complexClip = new AnimationClip("complex"); + +// Transform position animation +complexClip.addCurveBinding("", Transform, "position.y", jumpCurve); + +// Material transparency animation +complexClip.addCurveBinding("", MeshRenderer, "material.baseColor.a", alphaCurve); + +// Color animation of the second material (component index 1) +complexClip.addCurveBinding("", MeshRenderer, 1, "material.baseColor", colorCurve); +``` + +### Using Animation Events +```typescript +// Add footstep sound events +walkClip.addEvent("playFootstepSound", 0.25, { foot: "left" }); +walkClip.addEvent("playFootstepSound", 0.75, { foot: "right" }); + +// Add an attack hit check event +attackClip.addEvent("checkHit", 0.6, { + damage: 100, + range: 2.0, + type: "melee" +}); + +// Add a special effect event +jumpClip.addEvent("spawnEffect", 0.1, { + effectName: "dustCloud", + position: new Vector3(0, -1, 0) +}); +``` + +### Separate Read/Write Paths +```typescript +// For properties that require special handling +complexClip.addCurveBinding( + "weapon", // Weapon entity path + WeaponComponent, // Custom weapon component + "setDamage($value)", // Setter method + "getDamage()", // Getter method + damageCurve // Damage curve +); +``` + +## Animation Sampling Mechanism + +### Internal Sampling Process +```typescript +// Internal sampling method (called by the engine) +_sampleAnimation(entity: Entity, time: number): void { + // 1. Iterate through all curve bindings + // 2. Resolve target entity with entity.findByPath(binding.relativePath) + // 3. Fetch the component (typeIndex > 0 uses getComponents(...) with a shared scratch array) + // 4. Acquire a cached AnimationCurveOwner for the entity/component pair + // 5. Evaluate the curve at the specified time and apply the value via the owner +} +``` + +### Sampling Performance Optimization +- **Curve Owner Caching**: Reuses `AnimationCurveOwner` instances per entity/component pair. +- **Shared Component Scratch Array**: Reuses an internal array when fetching components to reduce allocations. +- **Automatic Length Tracking**: Clip length updates to the longest bound curve, so shorter curves are skipped once complete. + +## Best Practices + +### Curve Binding Design +1. **Path Naming Convention**: Use clear, hierarchical path names. +2. **Property Grouping**: Group related properties on a similar timeline. +3. **Avoid Redundancy**: Do not bind properties that do not change. +4. **Specify Index**: Clearly specify the index for multiple components. + +### Animation Event Optimization +1. **Precise Event Timing**: Ensure the accuracy of event timings. +2. **Lightweight Parameters**: Avoid passing heavy parameter objects. +3. **Efficient Event Handling**: Keep event handler functions efficient. +4. **Time Sorting**: Events are automatically sorted by time. + +### Performance Considerations +1. **Curve Complexity**: Avoid an excessive number of keyframes. +2. **Binding Count**: Control the number of bindings per clip. +3. **Path Depth**: Avoid overly deep entity paths. +4. **Memory Management**: Clean up unnecessary bindings promptly. + +### Compatibility Design +1. **Versioning**: Version control for animation data. +2. **Path Fault Tolerance**: Handle cases where entity paths do not exist. +3. **Component Checking**: Verify the existence of target components. +4. **Property Validation**: Ensure the validity of property paths. + +## Notes + +### Path Parsing +- Entity paths are separated by slashes, similar to file paths. +- An empty path refers to the current entity (the one containing the animation clip). +- Path lookups are real-time; caching is necessary for performance-sensitive scenarios. + +### Time Calculation +- The animation length is determined by the longest curve. +- The time range is [0, length]. +- Times outside this range are handled according to the curve's wrap mode. + +### Memory Management +- `AnimationClip` can be shared by multiple states. +- Curve owners are cached per target entity to avoid repeatedly recreating accessors. +- `clearCurveBindings()` empties bindings and resets the cached clip length to `0`. + +### Thread Safety +- The sampling process runs on the main thread. +- Avoid mutating bindings or events while a clip is actively sampled. +- Event triggers are synchronous. diff --git a/docs/scripting/AnimationCurve.md b/docs/scripting/AnimationCurve.md new file mode 100644 index 0000000000..9fdd5d9216 --- /dev/null +++ b/docs/scripting/AnimationCurve.md @@ -0,0 +1,328 @@ +# Animation Curve + +## Overview +`AnimationCurve` is an abstract base class in the Galacean engine that stores a collection of keyframes. It is used to define interpolation curves for how properties change over time. It supports multiple data types and interpolation methods, forming the foundation for smooth transitions and complex animation effects in the animation system. + +## Core Architecture + +### Type System +`AnimationCurve` is generic over the engine's `KeyframeValueType` union. Supported value categories include: +- **Scalars**: `number` +- **Vectors**: `Vector2`, `Vector3`, `Vector4` +- **Colors & rotations**: `Color`, `Quaternion` +- **Geometry**: `Rect` +- **Collections**: `number[]`, `Float32Array` +- **Discrete values**: `string`, `boolean` +- **Asset references**: `ReferResource` + +When you animate array or typed-array data, every keyframe must use the same element count; interpolation runs per index. + +### Concrete Implementation Classes +| Class | Value type | Interpolation support | Notes | +| --- | --- | --- | --- | +| `AnimationFloatCurve` | `number` | Linear, CubicSpine, Hermite, Step | Default choice for scalar properties. | +| `AnimationVector2Curve` | `Vector2` | Linear, CubicSpine, Hermite, Step | Interpolates component-wise. | +| `AnimationVector3Curve` | `Vector3` | Linear, CubicSpine, Hermite, Step | Works for positions, scales, etc. | +| `AnimationVector4Curve` | `Vector4` | Linear, CubicSpine, Hermite, Step | Useful for homogeneous data. | +| `AnimationColorCurve` | `Color` | Linear, CubicSpine, Hermite, Step | Internally treats colors as `Vector4`. | +| `AnimationQuaternionCurve` | `Quaternion` | Linear, CubicSpine, Hermite, Step | Manages quaternion keyframes without gimbal lock. | +| `AnimationArrayCurve` | `number[]` | Linear, CubicSpine, Hermite, Step | Arrays must keep a fixed length across keys. | +| `AnimationFloatArrayCurve` | `Float32Array` | Linear, CubicSpine, Hermite, Step | Allocates internal buffers based on the first key. | +| `AnimationRectCurve` | `Rect` | Step only | Suited for rectangle swaps; smoothing is not supported. | +| `AnimationBoolCurve` | `boolean` | Step only | For on/off style toggles. | +| `AnimationStringCurve` | `string` | Step only | Useful for state labels or shader keywords. | +| `AnimationRefCurve` | `ReferResource` | Step only | Switches resource references such as textures or prefabs. | + +> `interpolation` automatically clamps to `InterpolationType.Step` on step-only curves. Attempting to assign a smoothing mode logs a warning. + +## API Reference + +### AnimationCurve Base Class Methods + +#### Basic Properties +```typescript +// Sorted array of keyframes (earliest → latest) +keys: Keyframe[]; + +// The duration of the curve (equal to the last key's time, or 0 when empty) +get length(): number; + +// Interpolation mode. Defaults to Linear unless the curve only supports Step. +get interpolation(): InterpolationType; +set interpolation(value: InterpolationType); +``` + +- `keys` can be mutated directly if you need fine-grained control, but prefer `addKey`/`removeKey` to keep the array ordered. +- `length` updates whenever you add or remove keys; removing the last key collapses it back to `0`. +- On step-only curves (`AnimationBoolCurve`, `AnimationStringCurve`, `AnimationRectCurve`, `AnimationRefCurve`) any non-Step assignment is coerced back to `Step` with a warning. + +#### Keyframe Management +```typescript +// Insert a keyframe. The curve keeps the array ordered by time. +addKey(key: Keyframe): void; + +// Remove the keyframe at the given index. +removeKey(index: number): void; + +// Sample the curve at the specified second. +evaluate(time: number): V; +``` + +- `addKey` accepts any object that satisfies the `Keyframe` shape (a constructed `Keyframe` instance or a plain object typed as one). If the new key's time is the largest so far, `length` grows automatically. +- `evaluate` clamps outside the range of defined keys: times before the first key return the first value, and times past the last key return the last value. + +### Keyframe + +#### Keyframe Structure +```typescript +class Keyframe> { + time: number; + value: V; + inTangent?: T; + outTangent?: T; +} + +type TangentType = + V extends number ? number : + V extends Vector2 ? Vector2 : + V extends Vector3 ? Vector3 : + V extends Vector4 | Color | Quaternion | Rect ? Vector4 : + V extends number[] | Float32Array ? V : + V extends ReferResource ? ReferResource : + never; +``` + +- Tangents are only honoured by `InterpolationType.Hermite` and `InterpolationType.CubicSpine`. They are ignored for `Linear` and `Step` curves. +- For array-based curves the tangent must mirror the value's shape (same element count). Undefined tangents default to the value itself for Hermite calculations. + +### InterpolationType + +#### Interpolation Modes +```typescript +enum InterpolationType { + Linear, // Linear interpolation + CubicSpine, // Cubic spline interpolation + Step, // Step interpolation (no transition) + Hermite // Hermite interpolation +} +``` + +## Specific Curve Types + +All concrete curves expose the same public API; the differences are in the value type and whether smooth interpolation is supported. + +- **Smooth-capable curves** (`AnimationFloatCurve`, `AnimationVector2Curve`, `AnimationVector3Curve`, `AnimationVector4Curve`, `AnimationColorCurve`, `AnimationQuaternionCurve`, `AnimationArrayCurve`, `AnimationFloatArrayCurve`) honour linear, Hermite, and cubic spline interpolation. Hermite tangents must match the value type (scalar, vector, or array). +- **Step-only curves** (`AnimationBoolCurve`, `AnimationStringCurve`, `AnimationRectCurve`, `AnimationRefCurve`) always use discrete transitions. Their `interpolation` property is locked to `Step`. +- **Quaternion curves** expect normalized quaternions. Provide keys in radians (`Quaternion.rotationX/Y/Z`) to avoid gimbal issues. +- **Array and typed-array curves** interpolate each index independently; ensure all keys share the same length. `AnimationFloatArrayCurve` automatically resizes its internal buffers based on the first keyframe and any reference value bound through an `AnimationClip`. + +## Usage Examples + +### Creating a Basic Animation Curve +```typescript +// Float animation - transparency change +const alphaCurve = new AnimationFloatCurve(); +alphaCurve.interpolation = InterpolationType.Linear; + +const addFloatKey = (time: number, value: number) => { + const key = new Keyframe(); + key.time = time; + key.value = value; + alphaCurve.addKey(key); +}; + +addFloatKey(0, 1.0); // Start fully opaque +addFloatKey(0.5, 0.0); // Fully transparent in the middle +addFloatKey(1.0, 1.0); // End fully opaque + +// Get the value at 0.3 seconds +const alphaValue = alphaCurve.evaluate(0.3); // Approximately 0.4 +``` + +`Keyframe` is exported from `@galacean/engine`; creating an instance keeps the compiler aware of optional tangent fields. + +### Vector Animation Curve +```typescript +// Position animation - bounce effect +const jumpCurve = new AnimationVector3Curve(); +jumpCurve.interpolation = InterpolationType.Hermite; + +const addVec3Key = (time: number, value: Vector3) => { + const key = new Keyframe(); + key.time = time; + key.value = value; + jumpCurve.addKey(key); +}; + +// Add keyframes +addVec3Key(0, new Vector3(0, 0, 0)); // Starting position +addVec3Key(0.5, new Vector3(0, 5, 0)); // Peak of the jump +addVec3Key(1.0, new Vector3(0, 0, 0)); // Landing position + +// Set tangents to control the curve shape +const startKey = jumpCurve.keys[0]; +startKey.outTangent = new Vector3(0, 15, 0); // Out-tangent for take-off + +const endKey = jumpCurve.keys[2]; +endKey.inTangent = new Vector3(0, -15, 0); // In-tangent for landing +``` + +### Rotation Animation Curve +```typescript +// Quaternion rotation animation +const rotationCurve = new AnimationQuaternionCurve(); + +// Rotate 360 degrees around the Y-axis +const startRotation = Quaternion.rotationY(0); +const midRotation = Quaternion.rotationY(Math.PI); +const endRotation = Quaternion.rotationY(Math.PI * 2); + +const addQuatKey = (time: number, value: Quaternion) => { + const key = new Keyframe(); + key.time = time; + key.value = value; + rotationCurve.addKey(key); +}; + +addQuatKey(0, startRotation); +addQuatKey(1, midRotation); +addQuatKey(2, endRotation); +``` + +### Color Animation Curve +```typescript +// Color gradient animation +const colorCurve = new AnimationColorCurve(); + +// From red to blue, then to green +const addColorKey = (time: number, value: Color) => { + const key = new Keyframe(); + key.time = time; + key.value = value; + colorCurve.addKey(key); +}; + +addColorKey(0, new Color(1, 0, 0, 1)); // Red +addColorKey(1, new Color(0, 0, 1, 1)); // Blue +addColorKey(2, new Color(0, 1, 0, 1)); // Green +``` + +### Step Animation (Discrete Values) +```typescript +// Material switching animation +const materialIndexCurve = new AnimationFloatCurve(); +materialIndexCurve.interpolation = InterpolationType.Step; + +const addMaterialKey = (time: number, value: number) => { + const key = new Keyframe(); + key.time = time; + key.value = value; + materialIndexCurve.addKey(key); +}; + +addMaterialKey(0, 0); // Use material 0 +addMaterialKey(1, 1); // Switch to material 1 +addMaterialKey(2, 2); // Switch to material 2 + +// Step interpolation does not produce intermediate values +const matIndex = materialIndexCurve.evaluate(0.9); // Still 0 +``` + +### Controlling Complex Curve Shapes +```typescript +// Ease-in/ease-out effect +const easeCurve = new AnimationFloatCurve(); +easeCurve.interpolation = InterpolationType.Hermite; + +// Start keyframe +const startFrame = new Keyframe(); +startFrame.time = 0; +startFrame.value = 0; +startFrame.outTangent = 0; // Start smoothly + +// End keyframe +const endFrame = new Keyframe(); +endFrame.time = 1; +endFrame.value = 1; +endFrame.inTangent = 0; // End smoothly + +easeCurve.addKey(startFrame); +easeCurve.addKey(endFrame); +``` + +## Interpolation Algorithms Explained + +### Linear Interpolation +```typescript +// Formula: result = start + (end - start) * t +// t is the interpolation factor between 0 and 1 +const result = start + (end - start) * t; +``` + +### Hermite Interpolation +```typescript +// Cubic interpolation that uses tangents to control the curve shape +// Considers the inTangent and outTangent of the keyframes +const t2 = t * t; +const t3 = t2 * t; +const a = 2.0 * t3 - 3.0 * t2 + 1.0; +const b = t3 - 2.0 * t2 + t; +const c = t3 - t2; +const d = -2.0 * t3 + 3.0 * t2; + +result = a * p0 + b * outTangent * duration + c * inTangent * duration + d * p1; +``` + +### Step Interpolation +```typescript +// Directly returns the value of the current keyframe, with no smooth transition +result = currentKeyframe.value; +``` + +## Best Practices + +### Keyframe Design +1. **Keyframe Count**: Avoid too many keyframes, as it affects performance. +2. **Time Distribution**: Keyframe timings should be meaningful, not randomly distributed. +3. **Value Range**: Ensure keyframe values are within a reasonable range. +4. **Boundary Handling**: Pay attention to the behavior at the start and end of the curve. + +### Choosing an Interpolation Mode +1. **Linear**: Suitable for most common animations. +2. **Hermite**: Suitable for animations that require precise control over the curve shape. +3. **Step**: Use for discrete switches (material indices, booleans, asset swaps). Step-only curves lock to this mode automatically. +4. **CubicSpine**: Suitable for complex curves that require smooth continuity. + +### Performance Optimization +1. **Curve Reuse**: Reuse similar animation curves. +2. **Keyframe Optimization**: Remove redundant keyframes. +3. **Pre-calculation**: Cache values that are repeatedly calculated. +4. **Memory Management**: Release unnecessary curves promptly. + +### Tangent Control Techniques +1. **Smooth Transition**: Use continuous tangent values. +2. **Abrupt Change Effect**: Set discontinuous tangents. +3. **Ease-in/Ease-out**: Use zero tangents at the start and end. +4. **Elastic Effect**: Use tangents that overshoot the target value. + +## Notes + +### Numerical Precision +- Floating-point calculations may have precision issues. +- Time values should be within a reasonable range. +- Avoid extremely small or large tangent values. + +### Memory Management +- Vector and array curves keep reusable buffers for evaluation; changing the value length forces reallocation. Keep lengths consistent. +- Remove unused keyframes and curves to reduce serialized clip size. +- Reference-type curves (`AnimationRefCurve`) hold onto asset pointers—clear them if the asset can be unloaded. + +### Performance Considerations +- `evaluate` reuses an internal cache (`_evaluateData`) so you can call it every frame without additional allocations. +- Hermite and cubic spline interpolation do more math than linear or step curves. Prefer Linear unless you need the extra control. +- Keep keyframe counts as low as you can; fewer keys mean faster lookups and smaller serialized clips. + +### Compatibility +- Different types of curves have different characteristics. +- Some interpolation modes are not supported by all types. +- Tangent calculations may differ between types. diff --git a/docs/scripting/Animator.md b/docs/scripting/Animator.md new file mode 100644 index 0000000000..d6e37dfe14 --- /dev/null +++ b/docs/scripting/Animator.md @@ -0,0 +1,460 @@ +# Animator System + +## Overview +`Animator` is the runtime animation controller component in the Galacean engine. It plays animation clips through an `AnimatorController`, evaluates state machines, blends layers, and exposes parameter-driven control to gameplay code. Each Animator instance lives on an entity and can share controller assets with other animators. + +## Core Architecture +- **Animator**: Component that evaluates animation layers every frame and writes animated values back to components. +- **AnimatorController** (`ReferResource`): Asset containing animation parameters and layers. +- **AnimatorControllerLayer**: Holds an `AnimatorStateMachine` plus blending settings for that layer. +- **AnimatorStateMachine**: State graph that references `AnimatorState` objects and transition collections. +- **AnimatorState**: Wraps an `AnimationClip`, exposes per‑state playback settings, transitions, and `StateMachineScript` hooks. +- **AnimatorStateTransition**: Edge in the state machine; can be created from states, entry, or AnyState collections. +- **AnimatorControllerParameter**: Named parameter (float, int, bool, trigger, string, etc.) stored on the controller and driven per Animator instance. + +## Animator API + +### Key Properties +```typescript +animator.animatorController: AnimatorController | undefined; +animator.layers: Readonly; +animator.parameters: Readonly; +animator.speed: number; // Default 1.0 +animator.cullingMode: AnimatorCullingMode; // Default AnimatorCullingMode.None +``` +- Setting `animatorController` registers change flags; the Animator automatically resets when the asset changes or when the component is enabled. +- `layers` and `parameters` proxy through to the controller for quick inspection. +- `speed` scales playback globally for all layers. +- `cullingMode` determines whether evaluation is skipped when all controlled renderers are culled. + +### Playback & Queries +```typescript +animator.play(stateName: string, layerIndex: number = -1, normalizedTimeOffset: number = 0): void; +animator.crossFade(stateName: string, normalizedDuration: number, layerIndex: number = -1, normalizedTimeOffset: number = 0): void; +animator.crossFadeInFixedDuration(stateName: string, fixedDuration: number, layerIndex: number = -1, normalizedTimeOffset: number = 0): void; +animator.update(deltaTime: number): void; // Usually driven by the engine +``` +- `layerIndex = -1` tells the animator to search all layers for the first matching state name. +- `crossFade` interprets the duration as a normalized ratio of the destination clip length; `crossFadeInFixedDuration` uses seconds. +- When `play` or `crossFade*` is called from script, the Animator internally plays the first frame with `deltaTime = 0` to avoid skipping. + +#### State & Layer Lookup +```typescript +animator.getCurrentAnimatorState(layerIndex: number): AnimatorState | undefined; +animator.findAnimatorState(stateName: string, layerIndex: number = -1): AnimatorState | undefined; +animator.findLayerByName(name: string): AnimatorControllerLayer | undefined; +animator.getParameter(name: string): AnimatorControllerParameter | null; +``` + +### Parameter Control +```typescript +animator.getParameterValue(name: string): AnimatorControllerParameterValue | undefined; +animator.setParameterValue(name: string, value: AnimatorControllerParameterValue): void; +animator.activateTriggerParameter(name: string): void; +animator.deactivateTriggerParameter(name: string): void; +``` +- Triggers are just boolean parameters flagged as trigger; `activateTriggerParameter` writes `true`, `deactivateTriggerParameter` resets it to `false`. +- Use `setParameterValue` for all numeric, boolean, string, or reference parameters. + +## AnimatorController API +```typescript +const controller = new AnimatorController(engine); +controller.addParameter(name: string, defaultValue?: AnimatorControllerParameterValue): AnimatorControllerParameter; +controller.addTriggerParameter(name: string): AnimatorControllerParameter; +controller.removeParameter(name: string): void; +controller.clearParameters(): void; +controller.getParameter(name: string): AnimatorControllerParameter | null; + +const layer = new AnimatorControllerLayer('UpperBody'); +controller.addLayer(layer); +controller.removeLayer(index: number): void; +controller.clearLayers(): void; +controller.findLayerByName(name: string): AnimatorControllerLayer | undefined; +``` +- `AnimatorControllerLayer` exposes `weight`, `blendingMode`, `mask`, and its embedded `stateMachine`. +- Adding or removing layers/parameters invalidates change flags so active Animators reload state. + +## AnimatorStateMachine & AnimatorState +```typescript +// State machine +const state = layer.stateMachine.addState('idle'); +layer.stateMachine.defaultState = state; +layer.stateMachine.removeState(state); +layer.stateMachine.findStateByName('walk'); +layer.stateMachine.makeUniqueStateName('idle'); +layer.stateMachine.addEntryStateTransition(targetState); +layer.stateMachine.addAnyStateTransition(targetState); +layer.stateMachine.removeEntryStateTransition(transition); +layer.stateMachine.clearAnyStateTransitions(); + +// AnimatorState +state.clip = idleClip; +state.speed = 1.0; +state.wrapMode = WrapMode.Loop; +state.clipStartTime = 0; +state.clipEndTime = 1; +const transition = state.addTransition(otherState); +state.addExitTransition(0.9); +state.removeTransition(transition); +state.clearTransitions(); +state.addStateMachineScript(AttackScript); +``` +- `addTransition` and `addEntryStateTransition`/`addAnyStateTransition` accept either an existing `AnimatorStateTransition` instance or a destination `AnimatorState`. +- `clipStartTime`/`clipEndTime` restrict the normalized playback range inside the clip. +- `StateMachineScript` subclasses receive `onStateEnter`, `onStateUpdate`, `onStateExit` callbacks. + +## Usage Example +```typescript +import { Animator, AnimatorController, AnimatorControllerLayer, AnimatorLayerBlendingMode } from '@galacean/engine'; + +// Setup controller +const controller = new AnimatorController(engine); +const baseLayer = new AnimatorControllerLayer('Base'); +controller.addLayer(baseLayer); + +const idle = baseLayer.stateMachine.addState('idle'); +const walk = baseLayer.stateMachine.addState('walk'); +idle.clip = idleClip; +walk.clip = walkClip; +baseLayer.stateMachine.defaultState = idle; + +const speedParam = controller.addParameter('speed', 0); +const idleToWalk = idle.addTransition(walk); +idleToWalk.addCondition(AnimatorConditionMode.Greater, speedParam.name, 0.1); + +const upperLayer = new AnimatorControllerLayer('Upper'); +upperLayer.blendingMode = AnimatorLayerBlendingMode.Override; +controller.addLayer(upperLayer); +const shoot = upperLayer.stateMachine.addState('shoot'); +shoot.clip = shootClip; + +// Attach to entity +const animator = entity.addComponent(Animator); +animator.animatorController = controller; +animator.play('idle'); + +// Drive parameters at runtime +animator.setParameterValue('speed', inputValue); +if (pressedFire) { + animator.activateTriggerParameter('fire'); +} +``` + +## Best Practices +- **Organize layers**: Keep base locomotion in layer 0 and additive/override adjustments in separate layers with appropriate masks. +- **Reuse clips**: Share `AnimationClip` instances across states to reduce memory usage. +- **Update parameters sparingly**: Avoid writing parameter values every frame if nothing has changed. +- **Define default states**: Always configure `stateMachine.defaultState` so the Animator has a deterministic starting point. +- **Reset triggers**: After handling one-shot triggers in gameplay code, call `deactivateTriggerParameter` if the transition logic doesn’t auto-reset it. + +## Notes +- Animators are evaluated on the main thread; `update` is called automatically by the engine, but manual invocation is available for custom sequencing. +- Culling: when `cullingMode` is `AnimatorCullingMode.Complete`, the Animator skips evaluation while all controlled renderers are culled. +- `AnimatorController` and `AnimationClip` assets are reference-counted; releasing them when unused avoids leaks. +- `StateMachineScript` instances are created per state and persisted until removed; dispose of them if you dynamically unload states. + +## Advanced Animation Features + +### AnimatorStateMachine Deep Dive + +The `AnimatorStateMachine` is the core state management system that controls animation flow through states and transitions: + +```typescript +import { + AnimatorStateMachine, + AnimatorState, + AnimatorStateTransition, + AnimatorConditionMode +} from "@galacean/engine"; + +// Create and configure state machine +const stateMachine = new AnimatorStateMachine(); + +// Add states +const idleState = stateMachine.addState("idle"); +const walkState = stateMachine.addState("walk"); +const runState = stateMachine.addState("run"); +const jumpState = stateMachine.addState("jump"); + +// Set default state (auto-plays when controller starts) +stateMachine.defaultState = idleState; + +// Configure state properties +idleState.clip = idleClip; +idleState.speed = 1.0; +idleState.wrapMode = WrapMode.Loop; + +walkState.clip = walkClip; +walkState.speed = 1.2; // Play 20% faster +walkState.wrapMode = WrapMode.Loop; +``` + +#### State Transitions with Conditions + +```typescript +// Create transition from idle to walk +const idleToWalk = new AnimatorStateTransition(); +idleToWalk.destinationState = walkState; +idleToWalk.duration = 0.3; // 0.3 second blend duration +idleToWalk.exitTime = 0.8; // Start transition at 80% of idle animation +idleToWalk.hasExitTime = true; // Require exit time condition +idleToWalk.offset = 0.1; // Start walk animation 10% in + +// Add parameter-based conditions +idleToWalk.addCondition("speed", AnimatorConditionMode.Greater, 0.1); +idleToWalk.addCondition("isGrounded", AnimatorConditionMode.If, true); + +// Add transition to state +idleState.addTransition(idleToWalk); + +// Complex multi-condition transition +const walkToRun = new AnimatorStateTransition(); +walkToRun.destinationState = runState; +walkToRun.duration = 0.2; +walkToRun.hasExitTime = false; // Immediate transition when conditions met + +// Multiple conditions (ALL must be true) +walkToRun.addCondition("speed", AnimatorConditionMode.Greater, 0.5); +walkToRun.addCondition("stamina", AnimatorConditionMode.Greater, 20); +walkToRun.addCondition("canRun", AnimatorConditionMode.If, true); + +walkState.addTransition(walkToRun); +``` + +#### Exit Transitions and Any State + +```typescript +// Exit transition (returns to entry point) +const jumpToExit = jumpState.addExitTransition(); +jumpToExit.duration = 0.1; +jumpToExit.addCondition("jumpFinished", AnimatorConditionMode.If, true); + +// Any State transitions (can trigger from any current state) +const anyStateToJump = stateMachine.addAnyStateTransition(jumpState); +anyStateToJump.duration = 0.05; // Quick transition for responsive jump +anyStateToJump.addCondition("jumpTrigger", AnimatorConditionMode.If, true); + +// Entry state transitions (auto-play specific states) +stateMachine.addEntryStateTransition(idleState); +``` + +### AnimatorController Advanced Configuration + +#### Multi-Layer Animation System + +```typescript +const controller = new AnimatorController(engine); + +// Base layer - full body locomotion +const baseLayer = new AnimatorControllerLayer("Base"); +baseLayer.weight = 1.0; +baseLayer.blendingMode = AnimatorLayerBlendingMode.Override; +controller.addLayer(baseLayer); + +// Upper body layer - additive arm animations +const upperBodyLayer = new AnimatorControllerLayer("UpperBody"); +upperBodyLayer.weight = 0.8; +upperBodyLayer.blendingMode = AnimatorLayerBlendingMode.Additive; +controller.addLayer(upperBodyLayer); + +// Face layer - override facial expressions +const faceLayer = new AnimatorControllerLayer("Face"); +faceLayer.weight = 1.0; +faceLayer.blendingMode = AnimatorLayerBlendingMode.Override; +controller.addLayer(faceLayer); +``` + +#### Layer Masking System + +```typescript +// Create layer mask to isolate specific bones +const upperBodyMask = AnimatorLayerMask.createByEntity(characterEntity); + +// Enable/disable specific bone paths +upperBodyMask.setPathMaskActive("Root/Spine/Spine1", true, true); // Include children +upperBodyMask.setPathMaskActive("Root/Spine/Spine1/LeftArm", true, true); +upperBodyMask.setPathMaskActive("Root/Spine/Spine1/RightArm", true, true); +upperBodyMask.setPathMaskActive("Root/Hips", false, true); // Exclude hips and children + +// Apply mask to layer +upperBodyLayer.mask = upperBodyMask; +``` + +#### Parameter Management System + +```typescript +// Add different parameter types +const speedParam = controller.addParameter("speed", 0.0); // Float +const isGroundedParam = controller.addParameter("isGrounded", true); // Boolean +const jumpTrigger = controller.addTriggerParameter("jump"); // Trigger +const stateIndex = controller.addParameter("currentState", 0); // Integer + +// Runtime parameter control +animator.setParameterValue("speed", inputMagnitude); +animator.setParameterValue("isGrounded", isOnGround); +animator.setParameterValue("currentState", 2); + +// Trigger parameters (auto-reset after use) +if (jumpPressed) { + animator.activateTriggerParameter("jump"); +} + +// Parameter queries +const currentSpeed = animator.getParameterValue("speed"); +const canJump = animator.getParameterValue("isGrounded"); +``` + +### Animation Events System + +Animation events allow you to trigger script functions at specific times during animation playback: + +```typescript +// Add events to animation clips +const walkClip = walkState.clip; + +// Method 1: Direct event creation +const footstepEvent = new AnimationEvent(); +footstepEvent.functionName = "onFootstep"; +footstepEvent.time = 0.3; // 30% through animation +footstepEvent.parameter = { foot: "left", volume: 0.8 }; +walkClip.addEvent(footstepEvent); + +// Method 2: Simplified event creation +walkClip.addEvent("onFootstep", 0.7, { foot: "right", volume: 0.8 }); +walkClip.addEvent("onAnimationLoop", walkClip.length, null); + +// Event handler script +class CharacterController extends Script { + onFootstep(eventData: any): void { + console.log(`Footstep: ${eventData.foot} foot, volume: ${eventData.volume}`); + // Play footstep sound, spawn dust particles, etc. + this.playFootstepSound(eventData.foot, eventData.volume); + } + + onAnimationLoop(): void { + console.log("Walk animation completed one loop"); + // Update step counter, check for animation changes, etc. + } + + private playFootstepSound(foot: string, volume: number): void { + // Implementation for playing footstep audio + } +} +``` + +#### Event Management + +```typescript +// Clear all events from a clip +walkClip.clearEvents(); + +// Events are automatically sorted by time when added +walkClip.addEvent("earlyEvent", 0.1, null); +walkClip.addEvent("lateEvent", 0.9, null); +walkClip.addEvent("middleEvent", 0.5, null); +// Events will fire in chronological order: earlyEvent -> middleEvent -> lateEvent + +// Events with parameters +const attackClip = attackState.clip; +attackClip.addEvent("onAttackStart", 0.0, { attackType: "slash" }); +attackClip.addEvent("onAttackHit", 0.6, { damage: 50, hitType: "critical" }); +attackClip.addEvent("onAttackEnd", 1.0, { cooldown: 2.0 }); +``` + +### StateMachineScript System + +StateMachineScript provides lifecycle callbacks for animation states: + +```typescript +// Custom state machine script +class CombatStateScript extends StateMachineScript { + private weaponTrail: TrailRenderer; + private hasTriggeredHit = false; + + onStateEnter(animator: Animator, state: AnimatorState, layerIndex: number): void { + console.log(`Entering combat state: ${state.name}`); + + // Enable weapon trail effect + this.weaponTrail = animator.entity.getComponent(TrailRenderer); + if (this.weaponTrail) { + this.weaponTrail.enabled = true; + } + + // Set combat parameters + animator.setParameterValue("inCombat", true); + this.hasTriggeredHit = false; + } + + onStateUpdate(animator: Animator, state: AnimatorState, layerIndex: number): void { + // Called every frame while in this state + const stateInfo = animator.getCurrentStateInfo(layerIndex); + const normalizedTime = stateInfo.normalizedTime; + + // Trigger attack hit detection at 60% through animation + if (normalizedTime >= 0.6 && !this.hasTriggeredHit) { + this.triggerAttackHit(); + this.hasTriggeredHit = true; + } + + // Dynamic parameter updates based on animation progress + const attackIntensity = Math.sin(normalizedTime * Math.PI); + animator.setParameterValue("attackIntensity", attackIntensity); + } + + onStateExit(animator: Animator, state: AnimatorState, layerIndex: number): void { + console.log(`Exiting combat state: ${state.name}`); + + // Disable weapon trail effect + if (this.weaponTrail) { + this.weaponTrail.enabled = false; + } + + // Reset combat parameters + animator.setParameterValue("inCombat", false); + animator.setParameterValue("attackIntensity", 0); + } + + private triggerAttackHit(): void { + // Implementation for attack hit detection + console.log("Attack hit triggered!"); + } +} + +// Add script to animation state +const attackState = stateMachine.addState("attack"); +attackState.addStateMachineScript(CombatStateScript); +``` + +#### Multiple Scripts per State + +```typescript +// Add multiple scripts to handle different aspects +class EffectsScript extends StateMachineScript { + onStateEnter(animator: Animator, state: AnimatorState, layerIndex: number): void { + // Handle visual effects + this.startParticleEffects(); + } + + onStateExit(animator: Animator, state: AnimatorState, layerIndex: number): void { + // Clean up effects + this.stopParticleEffects(); + } + + private startParticleEffects(): void { /* ... */ } + private stopParticleEffects(): void { /* ... */ } +} + +class AudioScript extends StateMachineScript { + onStateEnter(animator: Animator, state: AnimatorState, layerIndex: number): void { + // Handle audio + this.playStateAudio(); + } + + private playStateAudio(): void { /* ... */ } +} + +// Add both scripts to the same state +attackState.addStateMachineScript(CombatStateScript); +attackState.addStateMachineScript(EffectsScript); +attackState.addStateMachineScript(AudioScript); +``` diff --git a/docs/scripting/AssetManagement.md b/docs/scripting/AssetManagement.md new file mode 100644 index 0000000000..c755a6d406 --- /dev/null +++ b/docs/scripting/AssetManagement.md @@ -0,0 +1,202 @@ +# Asset Management + +Galacean groups all runtime asset loading, caching, and recovery features behind the `ResourceManager`. Each engine instance owns a single manager (`engine.resourceManager`) that knows how to locate registered loaders, queue requests, expose loading progress, and cooperate with the reference-counted asset system built on `ReferResource`. + +## ResourceManager basics + +### Getting the manager +```ts +import { WebGLEngine } from "@galacean/engine"; + +const engine = await WebGLEngine.create({ canvas: "canvas" }); +const resourceManager = engine.resourceManager; +``` + +### Loading signatures +`ResourceManager.load` accepts several forms: +```ts +resourceManager.load("textures/diffuse.png"); // string URL +resourceManager.load({ url: "models/robot.gltf", type: AssetType.GLTF }); +resourceManager.load(["textures/albedo.png", "models/robot.gltf"]); +resourceManager.load([ + { url: "env/posx.hdr" }, + { url: "env/negx.hdr" }, +]); +``` +- When you pass a `LoadItem`, you must provide either `url` **or** `urls` (the latter is used by cube textures). The manager infers `type` from the URL extension if you omit it. +- Per-item overrides (`retryCount`, `retryInterval`, `timeout`, `params`) fall back to manager defaults (`resourceManager.retryCount`, `retryInterval`, and `timeout`). +- Relative paths are resolved against `resourceManager.baseUrl` when it is not `null`. + +All overloads return an `AssetPromise`, which is Promise-like and works with `await`, but also exposes progress and cancellation helpers. + +## AssetPromise in practice +```ts +const promise = resourceManager.load(["textures/diffuse.png", "models/robot.gltf"]); +promise + .onProgress( + (loaded, total) => console.log(`batch: ${loaded}/${total}`), + (url, loaded, total) => console.log(`${url}: ${loaded}/${total}`) + ) + .then(([texture, gltf]) => { /* use assets */ }) + .catch(console.error); + +// Cancel the entire batch if it is no longer needed. +promise.cancel(); +``` +- `AssetPromise` implements the standard `then/catch/finally` chain, so `await resourceManager.load(...)` works as expected. +- Use `.onProgress` for both aggregate and per-item progress callbacks. +- `.cancel()` invokes loader-specific cancellation if the request is still pending. + +`ResourceManager.cancelNotLoaded()` builds on the same mechanism. Call it with no arguments to abort every queued load, with one URL, or with an array of URLs to cancel selectively. + +## Caching & retrieval +Most loaders opt into caching by default. When a cached asset is requested again, the manager resolves immediately without reissuing network requests. + +```ts +const texture = await resourceManager.load("textures/wood.png"); +const fromCache = resourceManager.getFromCache("textures/wood.png"); +``` +- Loaders created with `@resourceLoader(..., useCache = false)` (built-in examples: `AssetType.Text`, `AssetType.JSON`, `AssetType.Buffer`) always bypass the cache. +- `resourceManager.findResourcesByType(Constructor)` returns all currently tracked `ReferResource` instances of that type. +- `resourceManager.getAssetPath(instanceId)` lets you map a resource back to the URL that produced it. + +## Retry, timeout, and base URL defaults +```ts +resourceManager.retryCount = 2; // per-item default, can be overridden on LoadItem +resourceManager.retryInterval = 500; // ms between retries +resourceManager.timeout = 10_000; // ms before the request aborts +resourceManager.baseUrl = "https://cdn.example.com/assets/"; +``` +The underlying `request` helper honors these values and exposes per-request overrides via `LoadItem` or a custom loader. + +## Reference counting & garbage collection +All loadable engine assets derive from `ReferResource` and participate in reference counting. Components increment counts automatically when they reference resources; counts drop when components release or destroy the reference. Call `resourceManager.gc()` to try releasing every referable whose `refCount` has fallen to zero. + +```ts +resourceManager.gc(); // normal sweep +resourceManager.gc(); // respects isGCIgnored flags + +texture.isGCIgnored = true; // opt-out of automatic GC when required +``` +Manual ownership (for example, when you create procedural textures) should call `addRef`/`removeRef` on the resource or `destroy()` it when finished. + +## Graphics context recovery +`GraphicsResource` derivatives (textures, buffers, render targets) automatically register themselves so the manager can: +- Flag the content as lost when WebGL reports a context loss. +- Rebuild GPU objects when the context is restored. + +For custom GPU objects or resources populated from URLs at runtime, create a `ContentRestorer` and register it: +```ts +class CustomTextureRestorer extends ContentRestorer { + constructor(texture: Texture2D, private url: string) { + super(texture); + } + restoreContent() { + return request(this.url).then((image) => { + this.resource.setImageSource(image); + this.resource.generateMipmaps(); + return this.resource; + }); + } +} + +resourceManager.addContentRestorer(new CustomTextureRestorer(texture, url)); +``` +`ContentRestorer.restoreContent` should return an `AssetPromise` or void; the manager awaits all promises before resuming rendering after a context restore. + +## Custom loaders +Register new loaders with the `@resourceLoader` decorator so the manager can resolve asset types and extensions. +```ts +@resourceLoader("FBX", ["fbx"], true) +export class FBXLoader extends Loader { + load(item: LoadItem, resourceManager: ResourceManager): AssetPromise { + return new AssetPromise((resolve, reject) => { + // implement fetch + parse, call resolve/reject when done + }); + } +} +``` +- The first argument is the asset `type` string. You can reference this string in `LoadItem.type`. +- The extension list seeds the type inference table used by `ResourceManager.load("file.ext")`. +- The optional `useCache` flag controls whether successfully loaded assets are stored in the cache. + +All loaders receive the resolved `LoadItem` (with retry/timeout already merged) and the manager instance so they can queue sub-loads or register referable resources. + +## AssetType reference +Galacean ships the following asset types (see `AssetType` enum): + +| Type | Description | Cached?* | +| --- | --- | --- | +| `Text` | Plain text files | No | +| `JSON` | Parsed JSON | No | +| `Buffer` | Binary `ArrayBuffer` | No | +| `Texture2D`, `TextureCube`, `KTX`, `KTXCube`, `KTX2` | Texture assets | Yes | +| `GLTF` | glTF or glb packages | Yes | +| `AnimationClip`, `AnimatorController` | Animation data | Yes | +| `Prefab`, `Scene`, `Project` | Editor-authored assets | Yes | +| `Mesh`, `PrimitiveMesh` | Mesh resources | Yes | +| `Material`, `Shader`, `ShaderChunk` | Rendering resources | Yes | +| `Sprite`, `SpriteAtlas` | 2D sprite data | Yes | +| `Env`, `HDR` | Environment lighting data | Yes | +| `Font`, `SourceFont` | Font data | Yes | +| `Audio` | Audio clips (mp3/ogg/wav) | Yes | +| `PhysicsMaterial` | Physics material presets | Yes | + +*Caching is determined by the loader. Built-in Text/JSON/Buffer loaders opt out of caching; all other shipped loaders set `useCache = true`. + +## Usage patterns + +### Loading with progress & cancellation +```ts +const loadTask = resourceManager.load([ + { url: "textures/ground.png" }, + { url: "models/outdoor.glb", retryCount: 3 } +]); + +const cancel = () => loadTask.cancel(); + +loadTask.onProgress( + (loaded, total) => updateOverallProgress(loaded / total), + (url, loaded, total) => updateRow(url, loaded / total) +); + +const [texture, glb] = await loadTask; +``` + +### Loading cube textures with `urls` +```ts +const cubeTexture = await resourceManager.load({ + type: AssetType.TextureCube, + urls: ["px.png", "nx.png", "py.png", "ny.png", "pz.png", "nz.png"], +}); +``` + +### Retrieving cached assets safely +```ts +let cached = resourceManager.getFromCache("textures/ground.png"); +if (!cached) { + cached = await resourceManager.load("textures/ground.png"); +} +``` + +### Releasing memory when leaving a scene +```ts +// Drop entity references (which decreases ref counts) +scene.destroy(); + +// Sweep referable assets whose refCount hit 0 +resourceManager.gc(); +``` + +## Best practices +- Prefer `await resourceManager.load()` or chain `AssetPromise` to take advantage of progress callbacks. +- Call `resourceManager.cancelNotLoaded()` when changing scenes or aborting downloads to avoid wasted bandwidth. +- Use manager-level defaults (`retryCount`, `retryInterval`, `timeout`) to centralize network policy. +- Keep long-lived shared assets (`isGCIgnored = true`) to prevent them from being reclaimed during GC sweeps. +- Register custom `ContentRestorer`s when you construct GPU resources manually so the engine can rebuild them after context loss. +- When extending the loader suite, always use the decorator so type inference and caching stay consistent. + +## Notes +- `ResourceManager.load` understands editor-specific virtual paths and sub-packaged assets; API consumers typically only need to provide URLs. +- Sub-asset addressing (e.g., `model.gltf?q=meshes[0]`) is handled internally for glTF and prefab loaders—no extra parsing is required in user code. +- All API surface documented here is available in `packages/core/src/asset` and used throughout the engine; there is no separate runtime service to enable. diff --git a/docs/scripting/AudioSystem.md b/docs/scripting/AudioSystem.md new file mode 100644 index 0000000000..90ae7d8826 --- /dev/null +++ b/docs/scripting/AudioSystem.md @@ -0,0 +1,475 @@ +# Audio System + +## System Overview + +The Audio System provides basic audio playback capabilities for the Galacean 3D engine using the Web Audio API. It supports audio clip loading, playback control, volume adjustment, and automatic lifecycle management through component-based architecture. + +## Core Architecture + +### AudioManager (Context Management) +```typescript +// Get audio context (automatically initialized) +const context = AudioManager.getContext(); + +// Check if audio context is running (requires user interaction) +if (AudioManager.isAudioContextRunning()) { + console.log("Audio is ready to play"); +} else { + console.warn("Audio requires user interaction to start"); +} + +// Get master gain node for global volume control +const masterGain = AudioManager.getGainNode(); +masterGain.gain.value = 0.5; // 50% global volume +``` + +### AudioClip (Audio Asset) +```typescript +// Create audio clip from loaded audio buffer +const audioClip = new AudioClip(engine, "backgroundMusic"); +audioClip.setAudioSource(audioBuffer); // Set decoded audio data + +// Access audio properties +console.log(`Duration: ${audioClip.duration} seconds`); +console.log(`Sample Rate: ${audioClip.sampleRate} Hz`); +console.log(`Channels: ${audioClip.channels}`); +``` + +### AudioSource (Component-Based Playback) +```typescript +// Add AudioSource component to entity +const audioEntity = engine.sceneManager.activeScene.createEntity("AudioPlayer"); +const audioSource = audioEntity.addComponent(AudioSource); + +// Configure audio source +audioSource.clip = audioClip; +audioSource.volume = 0.8; +audioSource.loop = true; +audioSource.playOnEnabled = true; + +// Manual playback control +audioSource.play(); // Start playback +audioSource.pause(); // Pause playback +audioSource.stop(); // Stop and reset playback +``` + +## Audio Clip Management + +### Loading and Creating Audio Clips +```typescript +// Load audio file and create clip +async function loadAudioClip(engine: Engine, url: string): Promise { + // Fetch audio file + const response = await fetch(url); + const arrayBuffer = await response.arrayBuffer(); + + // Decode audio data + const context = AudioManager.getContext(); + const audioBuffer = await context.decodeAudioData(arrayBuffer); + + // Create clip + const clip = new AudioClip(engine, url.split('/').pop()); + clip.setAudioSource(audioBuffer); + + return clip; +} + +// Usage +const musicClip = await loadAudioClip(engine, "/assets/music/background.mp3"); +const sfxClip = await loadAudioClip(engine, "/assets/sfx/explosion.wav"); +``` + +### Audio Clip Properties +```typescript +// Access audio information +const clip = audioSource.clip; +if (clip) { + console.log(`Audio: ${clip.name}`); + console.log(`Duration: ${clip.duration}s`); + console.log(`Sample Rate: ${clip.sampleRate}Hz`); + console.log(`Channels: ${clip.channels} (${clip.channels === 1 ? 'Mono' : 'Stereo'})`); +} + +// Resource management (automatic with reference counting) +audioSource.clip = newClip; // Old clip reference count decreases +audioSource.clip = null; // Release clip reference +``` + +## AudioSource Component API + +### Basic Properties +```typescript +// Clip assignment +audioSource.clip: AudioClip | null + +// Auto-play configuration +audioSource.playOnEnabled: boolean = true + +// Playback state (read-only) +audioSource.isPlaying: boolean + +// Playback position in seconds (read-only) +audioSource.time: number +``` + +### Volume Control +```typescript +// Volume control (0.0 to 1.0) +audioSource.volume: number = 1.0 + +// Mute/unmute functionality +audioSource.mute: boolean +// When muted, volume becomes 0; when unmuted, volume is restored + +// Example usage +audioSource.volume = 0.5; // 50% volume +audioSource.mute = true; // Mute audio +audioSource.mute = false; // Unmute and restore previous volume +``` + +### Playback Control +```typescript +// Playback rate adjustment (speed control) +audioSource.playbackRate: number = 1.0 + +// Loop configuration +audioSource.loop: boolean = false + +// Example usage +audioSource.playbackRate = 0.8; // 80% speed (slower) +audioSource.playbackRate = 1.5; // 150% speed (faster) +audioSource.loop = true; // Enable looping +``` + +### Playback Methods +```typescript +// Start playback +audioSource.play(): void + +// Pause playback (preserves position) +audioSource.pause(): void + +// Stop playback (resets position to start) +audioSource.stop(): void + +// Example usage +if (!audioSource.isPlaying) { + audioSource.play(); +} + +// Pause at current position +audioSource.pause(); +console.log(`Paused at ${audioSource.time} seconds`); + +// Resume from paused position +audioSource.play(); + +// Reset to beginning +audioSource.stop(); +console.log(`Position reset: ${audioSource.time} seconds`); // 0 +``` + +## Practical Examples + +### Background Music System +```typescript +class MusicManager { + private audioSource: AudioSource; + private musicClips: Map = new Map(); + private currentTrack: string | null = null; + + constructor(engine: Engine) { + const musicEntity = engine.sceneManager.activeScene.createEntity("MusicManager"); + this.audioSource = musicEntity.addComponent(AudioSource); + this.audioSource.loop = true; + this.audioSource.volume = 0.7; + this.audioSource.playOnEnabled = false; + } + + async loadTrack(name: string, url: string): Promise { + const clip = await this.loadAudioClip(url); + this.musicClips.set(name, clip); + } + + playTrack(name: string): void { + const clip = this.musicClips.get(name); + if (clip) { + this.audioSource.stop(); + this.audioSource.clip = clip; + this.audioSource.play(); + this.currentTrack = name; + } + } + + setVolume(volume: number): void { + this.audioSource.volume = Math.max(0, Math.min(1, volume)); + } + + fadeOut(duration: number): void { + const startVolume = this.audioSource.volume; + const fadeStep = startVolume / (duration * 60); // Assuming 60 FPS + + const fadeInterval = setInterval(() => { + this.audioSource.volume = Math.max(0, this.audioSource.volume - fadeStep); + + if (this.audioSource.volume <= 0) { + this.audioSource.stop(); + clearInterval(fadeInterval); + } + }, 1000 / 60); + } + + private async loadAudioClip(url: string): Promise { + const response = await fetch(url); + const arrayBuffer = await response.arrayBuffer(); + const audioBuffer = await AudioManager.getContext().decodeAudioData(arrayBuffer); + + const clip = new AudioClip(this.audioSource.entity.engine, url); + clip.setAudioSource(audioBuffer); + return clip; + } +} + +// Usage +const musicManager = new MusicManager(engine); +await musicManager.loadTrack("menu", "/audio/menu-theme.mp3"); +await musicManager.loadTrack("gameplay", "/audio/game-theme.mp3"); + +musicManager.playTrack("menu"); +``` + +### Sound Effects Pool +```typescript +class SFXManager { + private sfxPool: AudioSource[] = []; + private sfxClips: Map = new Map(); + private poolSize = 8; + + constructor(engine: Engine) { + const scene = engine.sceneManager.activeScene; + + // Create pool of AudioSource components + for (let i = 0; i < this.poolSize; i++) { + const sfxEntity = scene.createEntity(`SFX_${i}`); + const audioSource = sfxEntity.addComponent(AudioSource); + audioSource.playOnEnabled = false; + audioSource.loop = false; + this.sfxPool.push(audioSource); + } + } + + async loadSFX(name: string, url: string): Promise { + const response = await fetch(url); + const arrayBuffer = await response.arrayBuffer(); + const audioBuffer = await AudioManager.getContext().decodeAudioData(arrayBuffer); + + const clip = new AudioClip(this.sfxPool[0].entity.engine, name); + clip.setAudioSource(audioBuffer); + this.sfxClips.set(name, clip); + } + + playSFX(name: string, volume: number = 1.0): void { + const clip = this.sfxClips.get(name); + if (!clip) return; + + // Find available AudioSource in pool + const availableSource = this.sfxPool.find(source => !source.isPlaying); + if (availableSource) { + availableSource.clip = clip; + availableSource.volume = volume; + availableSource.play(); + } + } + + stopAllSFX(): void { + this.sfxPool.forEach(source => { + if (source.isPlaying) { + source.stop(); + } + }); + } +} + +// Usage +const sfxManager = new SFXManager(engine); +await sfxManager.loadSFX("jump", "/audio/sfx/jump.wav"); +await sfxManager.loadSFX("explosion", "/audio/sfx/explosion.wav"); + +// Play sound effects +sfxManager.playSFX("jump", 0.8); +sfxManager.playSFX("explosion", 1.0); +``` + +### Audio Settings Controller +```typescript +class AudioSettings { + private musicSources: AudioSource[] = []; + private sfxSources: AudioSource[] = []; + + registerMusicSource(source: AudioSource): void { + this.musicSources.push(source); + } + + registerSFXSource(source: AudioSource): void { + this.sfxSources.push(source); + } + + setMasterVolume(volume: number): void { + const masterGain = AudioManager.getGainNode(); + masterGain.gain.setValueAtTime(volume, AudioManager.getContext().currentTime); + } + + setMusicVolume(volume: number): void { + this.musicSources.forEach(source => { + source.volume = volume; + }); + } + + setSFXVolume(volume: number): void { + this.sfxSources.forEach(source => { + source.volume = volume; + }); + } + + muteAll(muted: boolean): void { + [...this.musicSources, ...this.sfxSources].forEach(source => { + source.mute = muted; + }); + } +} + +// Usage +const audioSettings = new AudioSettings(); +audioSettings.setMasterVolume(0.8); +audioSettings.setMusicVolume(0.6); +audioSettings.setSFXVolume(0.9); +``` + +## Best Practices + +### Performance Optimization +- **Audio Pool**: Use object pooling for frequent sound effects to avoid creating/destroying AudioSource components +- **Clip Reuse**: Share AudioClip instances between multiple AudioSource components +- **Context Management**: Always check `AudioManager.isAudioContextRunning()` before playing audio +- **Resource Cleanup**: Set `audioSource.clip = null` when no longer needed to release references + +### User Interaction Requirements +```typescript +// Handle browser's autoplay policy +function initializeAudio(): void { + if (!AudioManager.isAudioContextRunning()) { + // Display "Click to Enable Audio" prompt to user + document.addEventListener('click', resumeAudio, { once: true }); + } +} + +function resumeAudio(): void { + const context = AudioManager.getContext(); + if (context.state === 'suspended') { + context.resume().then(() => { + console.log('Audio context resumed'); + }); + } +} +``` + +### Error Handling +```typescript +// Robust audio loading with error handling +async function safeLoadAudio(engine: Engine, url: string): Promise { + try { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to load audio: ${response.status} ${response.statusText}`); + } + + const arrayBuffer = await response.arrayBuffer(); + const audioBuffer = await AudioManager.getContext().decodeAudioData(arrayBuffer); + + const clip = new AudioClip(engine, url); + clip.setAudioSource(audioBuffer); + return clip; + + } catch (error) { + console.error(`Audio loading failed for ${url}:`, error); + return null; + } +} + +// Safe audio playback +function safePlay(audioSource: AudioSource): void { + if (!audioSource.clip) { + console.warn("Cannot play: no audio clip assigned"); + return; + } + + if (!AudioManager.isAudioContextRunning()) { + console.warn("Cannot play: audio context not running (requires user interaction)"); + return; + } + + audioSource.play(); +} +``` + +## API Reference + +```apidoc +AudioManager: + Static Methods: + getContext(): AudioContext + - Returns the shared Web Audio API context. + getGainNode(): GainNode + - Returns the master gain node for global volume control. + isAudioContextRunning(): boolean + - Checks if audio context is in 'running' state. + +AudioClip: + Properties: + name: string + - The name identifier for this audio clip. + duration: number + - Duration of the audio clip in seconds. + sampleRate: number + - Sample rate of the audio data in Hz. + channels: number + - Number of audio channels (1 = mono, 2 = stereo). + + Methods: + setAudioSource(audioBuffer: AudioBuffer): void + - Sets the decoded audio data for this clip. + +AudioSource: + Properties: + clip: AudioClip | null + - The audio clip to play. @defaultValue `null` + playOnEnabled: boolean + - If true, audio plays automatically when component is enabled. @defaultValue `true` + isPlaying: boolean + - Whether audio is currently playing (read-only). + volume: number + - Volume level from 0.0 to 1.0. @defaultValue `1.0` + playbackRate: number + - Playback speed multiplier. @defaultValue `1.0` + loop: boolean + - Whether audio should loop when it reaches the end. @defaultValue `false` + mute: boolean + - Mute state (preserves volume setting). + time: number + - Current playback position in seconds (read-only). + + Methods: + play(): void + - Starts or resumes audio playback. + pause(): void + - Pauses audio playback, preserving current position. + stop(): void + - Stops audio playback and resets position to start. +``` + +## Limitations + +- **No 3D Spatial Audio**: The current audio system does not support 3D positioning or spatial audio effects +- **Browser Autoplay Policy**: Audio playback requires user interaction to start due to browser autoplay restrictions +- **Web Audio API Only**: Audio system is built on Web Audio API and requires modern browser support +- **No Audio Streaming**: All audio must be fully loaded before playback (no streaming support) \ No newline at end of file diff --git a/docs/scripting/Collider.md b/docs/scripting/Collider.md new file mode 100644 index 0000000000..ae48d31240 --- /dev/null +++ b/docs/scripting/Collider.md @@ -0,0 +1,384 @@ +# Collider + +## Overview +Collider is the base class for all colliders in the Galacean Engine, used to represent collision boundaries of entities in the physics world. It inherits from Component and can be attached to Entity, providing collision detection and physics simulation functionality for entities. The Collider system supports both static and dynamic types, with configurable shapes and materials. + +## Core Architecture + +### Class Hierarchy +``` +Component + ↓ +Collider (Abstract Base Class) + ├── StaticCollider (Static Collider) + └── DynamicCollider (Dynamic Collider) +``` + +### Main Components +- **Collider**: Base class for all colliders, managing shapes and collision layers +- **StaticCollider**: Static collider for non-moving objects +- **DynamicCollider**: Dynamic collider for objects that can move and be affected by forces +- **ColliderShape**: Abstract base class for collision shapes +- **PhysicsMaterial**: Physics material defining friction and elasticity properties + +## API Reference + +### Collider Base Class + +#### Basic Properties +```typescript +// Collision shapes array +readonly shapes: Readonly + +// Collision layer setting (single layer only) +collisionLayer: Layer +``` + +#### Shape Management +```typescript +// Add collision shape +addShape(shape: ColliderShape): void + +// Remove collision shape +removeShape(shape: ColliderShape): void + +// Clear all shapes +clearShapes(): void +``` + +### StaticCollider Static Collider + +#### Features +```typescript +// Static collider - never moves, used for immovable objects like walls, floors +// Usage scenario +const staticCollider = entity.addComponent(StaticCollider); +const boxShape = new BoxColliderShape(); +boxShape.size = new Vector3(1, 1, 1); +staticCollider.addShape(boxShape); +``` + +### DynamicCollider Dynamic Collider + +#### Physics Properties +```typescript +// Damping properties +linearDamping: number; // Linear damping +angularDamping: number; // Angular damping + +// Velocity properties +linearVelocity: Vector3; // Linear velocity +angularVelocity: Vector3; // Angular velocity + +// Mass properties +mass: number; // Mass +centerOfMass: Vector3; // Center of mass +inertiaTensor: Vector3; // Inertia tensor +``` + +#### Motion Control +```typescript +// Apply force and torque +applyForce(force: Vector3): void +applyTorque(torque: Vector3): void + +// Kinematic control - multiple overloads +move(position: Vector3): void +move(rotation: Quaternion): void +move(position: Vector3, rotation: Quaternion): void + +// Sleep control +sleep(): void +wakeUp(): void +isSleeping(): boolean +``` + +#### Physics Constraints +```typescript +// Gravity and kinematic mode +useGravity: boolean // Whether affected by gravity +isKinematic: boolean // Whether in kinematic mode + +// Motion constraints +constraints: DynamicColliderConstraints + +// Collision detection mode +collisionDetectionMode: CollisionDetectionMode +``` + +### ColliderShape Collision Shape Base Class + +#### Basic Properties +```typescript +// Shape properties +readonly id: number // Unique identifier +readonly collider: Collider // Owner collider + +// Transform properties +position: Vector3 // Local position +rotation: Vector3 // Local rotation (degrees) + +// Physics properties +material: PhysicsMaterial // Physics material +isTrigger: boolean // Whether it's a trigger +contactOffset: number // Contact offset. @defaultValue `0.02` +``` + +#### Distance Query +```typescript +// Get closest distance and position to a point +getClosestPoint(point: Vector3, outClosestPoint: Vector3): number +``` + +### Specific Shape Types + +#### BoxColliderShape Box Collider +```typescript +class BoxColliderShape extends ColliderShape { + size: Vector3 // Box dimensions +} + +// Create box collider +const boxShape = new BoxColliderShape(); +boxShape.size = new Vector3(2, 1, 1); // Width, height, depth +``` + +#### SphereColliderShape Sphere Collider +```typescript +class SphereColliderShape extends ColliderShape { + radius: number // Sphere radius +} + +// Create sphere collider +const sphereShape = new SphereColliderShape(); +sphereShape.radius = 0.5; +``` + +#### CapsuleColliderShape Capsule Collider +```typescript +class CapsuleColliderShape extends ColliderShape { + radius: number // Radius + height: number // Total height + upAxis: ColliderShapeUpAxis // Up axis direction +} +``` + +#### PlaneColliderShape Plane Collider +```typescript +class PlaneColliderShape extends ColliderShape { + // Infinite plane with normal pointing in positive Y direction +} +``` + +### PhysicsMaterial Physics Material + +#### Friction Properties +```typescript +// Friction coefficients +staticFriction: number // Static friction coefficient. @defaultValue `0.6` +dynamicFriction: number // Dynamic friction coefficient. @defaultValue `0.6` + +// Elasticity coefficient +bounciness: number // Bounciness coefficient (0-1). @defaultValue `0` +``` + +#### Combine Modes +```typescript +// Friction and bounce combine modes +frictionCombine: PhysicsMaterialCombineMode // @defaultValue `PhysicsMaterialCombineMode.Average` +bounceCombine: PhysicsMaterialCombineMode // @defaultValue `PhysicsMaterialCombineMode.Average` +``` + +## Enum Types + +### CollisionDetectionMode Collision Detection Mode +```typescript +enum CollisionDetectionMode { + Discrete, // Discrete detection + Continuous, // Continuous detection (static) + ContinuousDynamic, // Continuous detection (dynamic) + ContinuousSpeculative // Speculative continuous detection +} +``` + +### DynamicColliderConstraints Dynamic Constraints +```typescript +enum DynamicColliderConstraints { + None = 0, // No constraints + FreezePositionX = 1, // Freeze X axis translation + FreezePositionY = 2, // Freeze Y axis translation + FreezePositionZ = 4, // Freeze Z axis translation + FreezeRotationX = 8, // Freeze X axis rotation + FreezeRotationY = 16, // Freeze Y axis rotation + FreezeRotationZ = 32 // Freeze Z axis rotation +} +``` + +### PhysicsMaterialCombineMode Material Combine Mode +```typescript +enum PhysicsMaterialCombineMode { + Average, // Average value + Minimum, // Minimum value + Maximum, // Maximum value + Multiply // Product +} +``` + +## Usage Examples + +### Creating Static Collider +```typescript +// Ground collider +const groundEntity = rootEntity.createChild("Ground"); +const staticCollider = groundEntity.addComponent(StaticCollider); + +// Add plane shape +const planeShape = new PlaneColliderShape(); +planeShape.material.staticFriction = 0.8; +planeShape.material.dynamicFriction = 0.6; +staticCollider.addShape(planeShape); +``` + +### Creating Dynamic Collider +```typescript +// Movable ball +const ballEntity = rootEntity.createChild("Ball"); +ballEntity.transform.setPosition(0, 5, 0); + +const dynamicCollider = ballEntity.addComponent(DynamicCollider); + +// Configure physics properties +dynamicCollider.mass = 1.0; +dynamicCollider.useGravity = true; +dynamicCollider.linearDamping = 0.1; +dynamicCollider.angularDamping = 0.05; + +// Add sphere shape +const sphereShape = new SphereColliderShape(); +sphereShape.radius = 0.5; +sphereShape.material.bounciness = 0.8; +dynamicCollider.addShape(sphereShape); +``` + +### Compound Shape Collider +```typescript +// Complex-shaped vehicle +const vehicleEntity = rootEntity.createChild("Vehicle"); +const vehicleCollider = vehicleEntity.addComponent(DynamicCollider); + +// Vehicle body +const bodyShape = new BoxColliderShape(); +bodyShape.size = new Vector3(2, 0.8, 4); +bodyShape.position = new Vector3(0, 0.4, 0); + +// Vehicle roof +const roofShape = new BoxColliderShape(); +roofShape.size = new Vector3(1.5, 0.6, 2); +roofShape.position = new Vector3(0, 1.1, -0.5); + +vehicleCollider.addShape(bodyShape); +vehicleCollider.addShape(roofShape); +``` + +### Trigger Collider +```typescript +// Checkpoint trigger +const checkpointEntity = rootEntity.createChild("Checkpoint"); +const triggerCollider = checkpointEntity.addComponent(StaticCollider); + +const triggerShape = new BoxColliderShape(); +triggerShape.size = new Vector3(2, 3, 1); +triggerShape.isTrigger = true; // Set as trigger +triggerCollider.addShape(triggerShape); +``` + +### Applying Forces and Control +```typescript +// Forward propulsion +const forwardForce = new Vector3(0, 0, 100); +dynamicCollider.applyForce(forwardForce); + +// Upward jump +const jumpForce = new Vector3(0, 500, 0); +dynamicCollider.applyForce(jumpForce); + +// Rotation torque +const torque = new Vector3(10, 0, 0); +dynamicCollider.applyTorque(torque); + +// Kinematic control +if (dynamicCollider.isKinematic) { + const targetPosition = new Vector3(10, 0, 0); + dynamicCollider.move(targetPosition); +} +``` + +### Advanced Physics Configuration +```typescript +// Continuous collision detection for high-speed objects +dynamicCollider.collisionDetectionMode = CollisionDetectionMode.ContinuousDynamic; + +// Constrain motion axes +dynamicCollider.constraints = + DynamicColliderConstraints.FreezeRotationX | + DynamicColliderConstraints.FreezeRotationZ; + +// Custom center of mass and inertia +dynamicCollider.automaticCenterOfMass = false; +dynamicCollider.centerOfMass = new Vector3(0, -0.2, 0); + +dynamicCollider.automaticInertiaTensor = false; +dynamicCollider.inertiaTensor = new Vector3(1, 2, 1); +``` + +### Collision Layer Configuration +```typescript +// Set collision layer +collider.collisionLayer = Layer.Layer1; + +// Note: Only single layer can be set, not multiple layers +// collider.collisionLayer = Layer.Layer1 | Layer.Layer2; // ❌ Error +``` + +## Best Practices + +### Performance Optimization +1. **Shape Selection**: Prioritize simple shapes (sphere, box, capsule) +2. **Static Optimization**: Use StaticCollider for non-moving objects +3. **Sleep Mechanism**: Properly set sleepThreshold to let stationary objects sleep +4. **Collision Layers**: Use collision layers to reduce unnecessary collision calculations + +### Physics Realism +1. **Mass Setting**: Set reasonable mass based on actual objects +2. **Friction Materials**: Use different friction and elasticity parameters for different materials +3. **Damping Configuration**: Set appropriate linear and angular damping to simulate air resistance +4. **Gravity Setting**: Adjust gravity influence based on game requirements + +### Collision Detection +1. **Detection Mode**: Use continuous collision detection for high-speed objects +2. **Triggers**: Use for area detection in game logic +3. **Constraint Usage**: Limit unnecessary degrees of freedom in motion +4. **Contact Offset**: Adjust contactOffset to avoid penetration + +### Debugging and Monitoring +1. **Physics Visualization**: Enable visual debugging of physics shapes +2. **Performance Monitoring**: Monitor physics simulation performance overhead +3. **Parameter Debugging**: Real-time adjustment of physics parameters to observe effects +4. **Exception Handling**: Handle abnormal situations in physics simulation + +## Important Notes + +### Design Limitations +- Collision layers only support single layer setting, cannot belong to multiple layers simultaneously +- Material cannot be null, each shape must have a valid physics material +- Shape local transforms are relative to the entity containing the collider + +### Performance Considerations +- Complex shapes increase collision detection overhead +- Too many dynamic colliders affect physics simulation performance +- Continuous collision detection consumes more performance than discrete detection + +### Usage Constraints +- Kinematic objects are not affected by forces and gravity, can only be controlled through move method +- Triggers do not participate in physics collisions, only used for enter and exit event detection +- Automatic center of mass and inertia tensor calculations depend on shape geometric properties diff --git a/docs/scripting/InputManager.md b/docs/scripting/InputManager.md new file mode 100644 index 0000000000..0f6fa259fc --- /dev/null +++ b/docs/scripting/InputManager.md @@ -0,0 +1,184 @@ +# Input Manager System + +Galacean's `InputManager` centralizes keyboard, pointer (mouse, touch, pen), and wheel input. It converts browser events into frame-based state so gameplay logic can query a consistent view each update. The manager lives on the engine instance and is updated automatically every frame before scripts run. + +## Getting the Manager and Configuring Targets + +```ts +import { PointerButton, WebGLEngine } from '@galacean/engine'; + +const engine = await WebGLEngine.create({ + canvas, + input: { + pointerTarget: document, // capture drags outside the canvas + keyboardTarget: window, + wheelTarget: document + } +}); + +const { inputManager } = engine; +``` + +By default the engine listens on the canvas for pointer/wheel and on `window` for keyboard. When running inside an `OffscreenCanvas`, the manager is not initialized (all queries return defaults, `pointers` is empty, `wheelDelta` is `null`). Feature-detect by checking `inputManager.pointers.length` or `inputManager.wheelDelta`. + +## Keyboard Input + +Keyboard state is tracked with three queries. Each method accepts an optional `Keys` enum; omit the argument to ask "any key". + +| Method | Meaning | +| --- | --- | +| `isKeyHeldDown(key?)` | Key is currently pressed. Without arguments, returns true if *any* key is held. | +| `isKeyDown(key?)` | Key transitioned to pressed during the current frame. | +| `isKeyUp(key?)` | Key transitioned to released during the current frame. | + +```ts +import { Keys, Script } from '@galacean/engine'; +import { Vector3 } from '@galacean/engine-math'; + +class MovementScript extends Script { + private readonly move = new Vector3(); + + onUpdate(deltaTime: number): void { + const input = this.engine.inputManager; + this.move.set(0, 0, 0); + + if (input.isKeyHeldDown(Keys.KeyW)) this.move.z += 1; + if (input.isKeyHeldDown(Keys.KeyS)) this.move.z -= 1; + if (input.isKeyHeldDown(Keys.KeyD)) this.move.x += 1; + if (input.isKeyHeldDown(Keys.KeyA)) this.move.x -= 1; + + if (this.move.length() > 0) { + this.move.normalize(); + this.entity.transform.translate(this.move.scale(4 * deltaTime)); + } + + if (input.isKeyDown(Keys.Space)) this.jump(); + } +} +``` + +On macOS browsers the system suppresses `keyup` while either `Meta` key is held. When `MetaLeft`/`MetaRight` is released the manager clears every recorded key, so guard combination logic accordingly. + +## Pointer Input + +`inputManager.pointers` exposes the active pointers for the current frame (mouse, touches, pen contacts). The list is kept in ascending logical id order, and entries are removed the frame after their `phase` becomes `PointerPhase.Leave`. + +Key fields on `Pointer`: + +| Property | Description | +| --- | --- | +| `id` | Stable index (0-based) assigned by the manager. | +| `phase` | `PointerPhase.Down`, `Move`, `Stationary`, `Up`, or `Leave`. | +| `position` | Canvas-space pixel coordinates (already DPI corrected). | +| `deltaPosition` | Movement since the previous frame (0,0 when stationary). | +| `button` | Button involved in the last event for this pointer. | +| `pressedButtons` | Bit mask of all buttons currently held (`PointerButton`). | + +Pointer button helpers mirror the keyboard trio: + +```ts +import { PointerButton, PointerPhase, Script } from '@galacean/engine'; +import { Vector3 } from '@galacean/engine-math'; + +class OrbitCamera extends Script { + private yaw = 0; + private pitch = 15; + private readonly offset = new Vector3(0, 3, 8); + + onUpdate(): void { + const input = this.engine.inputManager; + const pointers = input.pointers; + + if (input.isPointerHeldDown(PointerButton.Primary) && pointers.length > 0) { + const pointer = pointers[0]; + this.yaw -= pointer.deltaPosition.x * 0.2; + this.pitch = Math.max(-80, Math.min(80, this.pitch - pointer.deltaPosition.y * 0.2)); + } + + if (input.isPointerDown(PointerButton.Secondary)) { + this.resetCamera(); + } + + this.applyTransform(); + } +} +``` + +Set `inputManager.multiPointerEnabled = false` to force a single logical pointer (touch gestures collapse to index 0). Leave it enabled to process multi-touch. + +## Pointer Callbacks on Scripts + +Pointer interactions are delivered to scripts attached to entities with colliders. Events fire **before** `onUpdate` each frame. + +```ts +import { PointerButton, PointerEventData, PointerPhase, Script } from '@galacean/engine'; + +class ButtonHandler extends Script { + private pressed = false; + + onPointerDown(event: PointerEventData): void { + if (event.pointer.button === PointerButton.Primary) this.pressed = true; + } + + onPointerUp(event: PointerEventData): void { + if (this.pressed && event.pointer.phase === PointerPhase.Up) { + this.activate(); + } + this.pressed = false; + } +} +``` + +`PointerEventData` supplies the originating `Pointer` plus `worldPosition` from whichever emitter performed the raycast. The engine automatically adds a physics-based emitter when physics is enabled. You can extend the system by registering your own emitter: + +```ts +import { + Pointer, + PointerEventData, + PointerEventEmitter, + PointerManager, + registerPointerEventEmitter +} from '@galacean/engine'; + +@registerPointerEventEmitter() +class UiCanvasEmitter extends PointerEventEmitter { + protected _init(): void {} + processRaycast(): void { /* custom hit testing */ } + processDrag(): void {} + processDown(pointer: Pointer): void {} + processUp(pointer: Pointer): void {} + processLeave(pointer: Pointer): void {} + dispose(): void {} +} +``` + +## Wheel Input + +`inputManager.wheelDelta` aggregates all wheel events received in the previous frame and returns a `Vector3` (x, y, z). The vector is reset to `(0, 0, 0)` every update; the property is `null` if the manager is uninitialized. + +```ts +const { wheelDelta } = engine.inputManager; +if (wheelDelta && wheelDelta.y !== 0) { + camera.zoom(Math.sign(wheelDelta.y)); +} +``` + +## Best Practices and Notes + +- Avoid caching `Pointer` references across frames; always read from `inputManager.pointers` so recycled instances are handled correctly. +- Query input once per update and propagate the result to gameplay systems to keep deterministic ordering. +- Clamp pointer-driven rotations and normalise movement vectors to account for large deltas on high-DPI devices. +- For drag gestures, use both `deltaPosition` and `pressedButtons` to disambiguate multi-button mice. +- When UI needs to react without colliders, register a custom emitter that translates pointer hits to your own event system. +- Remember that `PointerPhase.Stationary` indicates no movement even if buttons remain pressed. + +## Quick Reference + +- `engine.inputManager.pointers: Readonly` +- `engine.inputManager.multiPointerEnabled: boolean` +- `engine.inputManager.wheelDelta: Readonly | null` +- `engine.inputManager.isKeyHeldDown/Down/Up(key?: Keys)` +- `engine.inputManager.isPointerHeldDown/Down/Up(button?: PointerButton)` +- `Pointer` fields: `id`, `phase`, `position`, `deltaPosition`, `button`, `pressedButtons` +- Script callbacks: `onPointerDown`, `onPointerUp`, `onPointerClick`, `onPointerEnter`, `onPointerExit`, `onPointerBeginDrag`, `onPointerDrag`, `onPointerEndDrag`, `onPointerDrop` +- Advanced: extend pointer picking via `@registerPointerEventEmitter` diff --git a/docs/scripting/Joint.md b/docs/scripting/Joint.md new file mode 100644 index 0000000000..2d4e7df2ad --- /dev/null +++ b/docs/scripting/Joint.md @@ -0,0 +1,193 @@ +# Joint System + +Galacean joints provide high-level constraints that keep two colliders in a particular spatial relationship. Every joint component lives on the "primary" entity, automatically adds a `DynamicCollider` to it, and connects that body to another collider (or to world space). Joints require a physics backend (such as `@galacean/engine-physics-physx`) because they are solved by the native physics layer. + +## Quick Start + +```ts +import { + BoxColliderShape, + DynamicCollider, + FixedJoint, + HingeJoint, + JointLimits, + JointMotor, + SpringJoint, + WebGLEngine +} from '@galacean/engine'; +import { PhysXPhysics } from '@galacean/engine-physics-physx'; + +const engine = await WebGLEngine.create({ + canvas, + physics: new PhysXPhysics() +}); + +// Primary body (the host entity automatically receives a DynamicCollider) +const base = engine.sceneManager.activeScene.createRootEntity('Base'); +const baseCollider = base.addComponent(DynamicCollider); +baseCollider.addShape(new BoxColliderShape()); + +// Connected body +const target = engine.sceneManager.activeScene.createRootEntity('Target'); +const targetCollider = target.addComponent(DynamicCollider); +targetCollider.addShape(new BoxColliderShape()); + +// FixedJoint keeps both rigid bodies welded together +const fixedJoint = base.addComponent(FixedJoint); +fixedJoint.connectedCollider = targetCollider; +fixedJoint.breakForce = 2_000; +fixedJoint.breakTorque = 2_000; +``` + +Set `connectedCollider = null` to attach the primary body to an immovable point in world space (useful for hanging props). + +## Joint Base Class + +| Property | Type | Notes | +| --- | --- | --- | +| `connectedCollider` | `Collider | null` | Target collider. `null` means the anchors are interpreted in world space. | +| `anchor` | `Vector3` | Local offset on the primary body. Mutating the vector updates the joint immediately. | +| `connectedAnchor` | `Vector3` | Local offset on the connected collider or world position if `connectedCollider` is `null`. | +| `automaticConnectedAnchor` | `boolean` | When `true` (default) the engine keeps `connectedAnchor` aligned with `anchor` in world space. Set to `false` before editing `connectedAnchor` manually. | +| `massScale` / `connectedMassScale` | `number` | Multiplies the effective mass of each body during constraint solving (default `1`). | +| `inertiaScale` / `connectedInertiaScale` | `number` | Multiplies the rotational inertia contribution of each body (default `1`). | +| `breakForce` | `number` | Maximum linear force before the joint breaks. Defaults to `Infinity`. | +| `breakTorque` | `number` | Maximum angular force before the joint breaks. Defaults to `Infinity`. | + +### Common configuration + +```ts +const joint = base.addComponent(FixedJoint); + +// Automatic anchors keep both bodies aligned initially +joint.anchor.setValue(0, 0.5, 0); +joint.automaticConnectedAnchor = true; + +// Switch to manual mode when you need exact offsets +joint.automaticConnectedAnchor = false; +joint.connectedAnchor.setValue(0, -0.5, 0); + +// Soften the constraint by scaling mass and inertia +joint.massScale = 0.5; +joint.connectedMassScale = 2.0; +``` + +## FixedJoint + +`FixedJoint` exposes only the base class properties. It prevents all relative motion, effectively gluing the two colliders together. Typical uses include: + +- Assembling complex rigid bodies from multiple parts +- Attaching dynamic props to a static world anchor +- Creating destructible welds via `breakForce` / `breakTorque` + +## HingeJoint + +`HingeJoint` behaves like a door hinge or wheel axle. Additional members: + +| Property | Type | Notes | +| --- | --- | --- | +| `axis` | `Vector3` | Local axis on the primary body. The setter normalizes the vector. | +| `angle` | `number` (read-only) | Current angle in degrees relative to the initial pose. Updated each frame while enabled. | +| `velocity` | `number` (read-only) | Angular velocity in degrees per second. | +| `useLimits` | `boolean` | Enables hard or soft limits defined by `limits`. | +| `useMotor` | `boolean` | Enables the drive defined by `motor`. | +| `useSpring` | `boolean` | When `true`, limits are treated as a spring (`JointLimits.stiffness` / `damping`). | +| `motor` | `JointMotor | null` | Drive settings (lazy-updated; reuse instances when possible). | +| `limits` | `JointLimits | null` | Angle range and optional spring coefficients. | + +```ts +const hinged = base.addComponent(HingeJoint); +hinged.connectedCollider = targetCollider; + +hinged.axis.setValue(0, 1, 0); // rotate around Y +hinged.useLimits = true; + +const limits = new JointLimits(); +limits.min = -45; +limits.max = 60; +limits.contactDistance = 2; // degrees before the stop engages +limits.stiffness = 200; // used only when useSpring = true +limits.damping = 20; + +hinged.limits = limits; +hinged.useSpring = true; // enables soft limits + +const motor = new JointMotor(); +motor.targetVelocity = 90; // deg/s +motor.forceLimit = 800; // torque limit +motor.gearRatio = 1; +motor.freeSpin = false; + +hinged.motor = motor; +hinged.useMotor = true; +``` + +Call `hinged.angle`/`hinged.velocity` to read the live motion, e.g. for UI displays or control loops. + +## SpringJoint + +`SpringJoint` maintains a distance band between two anchor points and applies spring forces when the band is violated. + +| Property | Type | Notes | +| --- | --- | --- | +| `minDistance` | `number` | Minimum allowed distance (metres). | +| `maxDistance` | `number` | Maximum allowed distance (metres). | +| `tolerance` | `number` | Additional slack before forces are applied. Default `0.25`. | +| `stiffness` | `number` | Spring constant. Larger values pull harder. | +| `damping` | `number` | Damping ratio. Larger values reduce oscillation. | + +```ts +const spring = base.addComponent(SpringJoint); +spring.connectedCollider = targetCollider; + +spring.minDistance = 1.0; +spring.maxDistance = 3.0; +spring.tolerance = 0.15; +spring.stiffness = 600; +spring.damping = 40; +``` + +Set `minDistance === maxDistance` to approximate a distance constraint. + +## JointLimits + +`JointLimits` is a mutable data object that notifies listeners when any property changes. + +| Property | Notes | +| --- | --- | +| `min` / `max` | Angular bounds in degrees. Setters clamp each other so `min ≤ max`. | +| `contactDistance` | Optional margin (degrees). Defaults to `min(0.1, 0.49 * (max - min))` when left at `-1`. Only used when `useSpring` is `false`. | +| `stiffness` / `damping` | Spring coefficients used when `HingeJoint.useSpring` is `true`. | + +```ts +const limits = new JointLimits(); +limits.min = -30; +limits.max = 45; +limits.contactDistance = 1.5; +limits.stiffness = 150; +limits.damping = 10; + +hinged.limits = limits; +``` + +## JointMotor + +`JointMotor` describes the drive applied by a `HingeJoint` when `useMotor = true`. + +| Property | Notes | +| --- | --- | +| `targetVelocity` | Desired angular speed in degrees per second. | +| `forceLimit` | Maximum torque applied by the drive (defaults to `Number.MAX_VALUE`). | +| `gearRatio` | Multiplies the target velocity and required torque (default `1`). | +| `freeSpin` | When `true`, the motor only accelerates; it will not actively brake. | + +Motors and limits can be reused across multiple joints—each instance carries an internal `UpdateFlagManager`, so changing a property automatically pushes the new value to every joint referencing it. + +## Best Practices + +- Always add appropriate collider shapes to both bodies before attaching a joint; the solver needs accurate mass and inertia data. +- Keep hinge axes normalized—`HingeJoint` normalizes internally, but supplying a unit vector avoids needless allocations. +- For manual anchor editing, disable `automaticConnectedAnchor` first so the system does not overwrite your values on the next frame. +- Use `breakForce` / `breakTorque` together to simulate destructible links. When a threshold is exceeded the physics backend destroys the joint; keep track of the component if you need to respawn or play break effects. +- When connecting to the world (no `connectedCollider`), treat `connectedAnchor` as world coordinates in metres. +- Reuse `JointLimits` and `JointMotor` instances when multiple hinges share the same tuning to reduce garbage and keep configurations consistent. diff --git a/docs/scripting/Loader.md b/docs/scripting/Loader.md new file mode 100644 index 0000000000..947738bc29 --- /dev/null +++ b/docs/scripting/Loader.md @@ -0,0 +1,572 @@ +# Loader System + +Galacean's Loader system is the cornerstone of resource loading architecture, providing a unified, extensible framework for loading different asset types. The system supports custom loader development, resource caching, content restoration, and type-safe serialization while maintaining high performance across web and mobile platforms. + +## Overview + +The Loader system provides comprehensive asset loading capabilities: + +- **Extensible Architecture**: Plugin-based design for custom asset types and formats +- **Type Safety**: Generic-based loader system with compile-time type checking +- **Caching Strategy**: Intelligent resource caching with memory management +- **Format Support**: Built-in loaders for textures, models, audio, fonts, and more +- **Content Restoration**: Automatic recovery of loaded resources after context loss +- **Serialization**: Class registration system for runtime type resolution +- **Performance**: Optimized loading pipelines with parallel processing support + +The system integrates seamlessly with ResourceManager and supports both eager and lazy loading patterns. + +## Quick Start + +```ts +import { WebGLEngine, Loader, AssetType, Texture2D, GLTFResource } from "@galacean/engine"; + +const engine = await WebGLEngine.create({ canvas: "canvas" }); +const resourceManager = engine.resourceManager; + +// Load single asset +const texture = await resourceManager.load({ + url: "textures/brick.jpg", + type: AssetType.Texture2D +}); + +// Load multiple assets +const [model, normalMap] = await resourceManager.load([ + { url: "models/character.gltf", type: AssetType.GLTF }, + { url: "textures/character-normal.png", type: AssetType.Texture2D } +]); + +// Load with custom parameters +const compressedTexture = await resourceManager.load({ + url: "textures/terrain.ktx2", + type: AssetType.KTX2, + params: { + format: "RGBA_ASTC_4x4", + generateMipmaps: true + } +}); + +// Check loading progress +const promise = resourceManager.load({ url: "large-model.gltf", type: AssetType.GLTF }); +promise.onProgress((progress) => { + console.log(`Loading: ${(progress * 100).toFixed(1)}%`); +}); +const gltfResource = await promise; +``` + +## Built-in Loaders + +### Texture Loaders + +```ts +// Standard image formats +const diffuse = await resourceManager.load({ + url: "diffuse.jpg", + type: AssetType.Texture2D, + params: { + wrapModeU: TextureWrapMode.Repeat, + wrapModeV: TextureWrapMode.Repeat, + generateMipmaps: true + } +}); + +// Compressed texture formats +const ktxTexture = await resourceManager.load({ + url: "compressed.ktx", + type: AssetType.KTX +}); + +const ktx2Texture = await resourceManager.load({ + url: "compressed.ktx2", + type: AssetType.KTX2, + params: { + targetFormat: "ETC1_RGB" + } +}); + +// Cube textures for skyboxes +const cubeTexture = await resourceManager.load({ + url: "skybox.hdr", + type: AssetType.HDR +}); +``` + +### Model Loaders + +```ts +// GLTF/GLB models with full scene support +const gltfResource = await resourceManager.load({ + url: "models/scene.gltf", + type: AssetType.GLTF +}); + +// Access GLTF content +const { defaultSceneRoot, scenes, animations, materials } = gltfResource; +rootEntity.addChild(defaultSceneRoot); + +// Primitive meshes +const sphereMesh = await resourceManager.load({ + url: "primitive://sphere", + type: AssetType.Mesh, + params: { radius: 1, segments: 32 } +}); + +// Custom mesh data +const bufferMesh = await resourceManager.load({ + url: "meshes/custom.mesh", + type: AssetType.Mesh +}); +``` + +### Material and Shader Loaders + +```ts +// Material definitions +const material = await resourceManager.load({ + url: "materials/metal.material", + type: AssetType.Material +}); + +// Shader files +const shader = await resourceManager.load({ + url: "shaders/custom.shader", + type: AssetType.Shader +}); + +// Note: ShaderChunk loading is handled internally by the shader system +// and is not directly exposed through AssetType +``` + +### Audio and Font Loaders + +```ts +// Audio assets +const audioClip = await resourceManager.load({ + url: "audio/music.mp3", + type: AssetType.Audio +}); + +// Font resources +const font = await resourceManager.load({ + url: "fonts/roboto.ttf", + type: AssetType.Font +}); + +// Source fonts for text rendering +const sourceFont = await resourceManager.load({ + url: "fonts/arial.json", + type: AssetType.SourceFont +}); +``` + +## Custom Loader Development + +### Basic Loader Implementation + +```ts +import { Loader, AssetPromise, LoadItem, ResourceManager, AssetType } from "@galacean/engine"; + +// Define custom asset type +export class CustomAsset { + constructor(public data: any, public metadata: object) {} +} + +// Implement custom loader +@resourceLoader(AssetType.Buffer, ["custom"]) +export class CustomAssetLoader extends Loader { + constructor() { + super(true); // Enable caching + } + + load(item: LoadItem, resourceManager: ResourceManager): AssetPromise { + return new AssetPromise((resolve, reject, setProgress) => { + const request = resourceManager.request(item.url, { + ...item, + type: "arraybuffer" + }); + + request.onProgress = setProgress; + + request.then((buffer) => { + try { + // Custom parsing logic + const data = this.parseCustomFormat(buffer); + const metadata = this.extractMetadata(buffer); + + const asset = new CustomAsset(data, metadata); + resolve(asset); + } catch (error) { + reject(error); + } + }).catch(reject); + }); + } + + private parseCustomFormat(buffer: ArrayBuffer): any { + // Implement custom format parsing + const view = new DataView(buffer); + // ... parsing logic + return {}; + } + + private extractMetadata(buffer: ArrayBuffer): object { + // Extract metadata from buffer + return {}; + } +} +``` + +### Advanced Loader with Dependencies + +```ts +@resourceLoader(AssetType.Prefab, ["prefab"]) +export class PrefabLoader extends Loader { + constructor() { + super(true); + } + + load(item: LoadItem, resourceManager: ResourceManager): AssetPromise { + return new AssetPromise(async (resolve, reject, setProgress) => { + try { + // Load prefab definition + const prefabData = await resourceManager.request(item.url, { + ...item, + type: "json" + }); + + setProgress(0.3); + + // Load dependent assets + const dependencies = this.extractDependencies(prefabData); + const dependentAssets = await Promise.all( + dependencies.map(dep => resourceManager.load(dep)) + ); + + setProgress(0.8); + + // Create prefab resource + const prefabResource = new PrefabResource(item.url); + prefabResource.data = prefabData; + prefabResource.dependencies = dependentAssets; + + setProgress(1.0); + resolve(prefabResource); + } catch (error) { + reject(error); + } + }); + } + + private extractDependencies(prefabData: any): LoadItem[] { + // Extract asset dependencies from prefab data + const dependencies: LoadItem[] = []; + // ... dependency extraction logic + return dependencies; + } +} +``` + +## Loading Strategies + +### Preloading Strategy + +```ts +// Preload critical assets +const criticalAssets = [ + { url: "textures/ui-atlas.png", type: AssetType.Texture2D }, + { url: "models/player.gltf", type: AssetType.GLTF }, + { url: "audio/bgm.mp3", type: AssetType.Audio } +]; + +// Load with progress tracking +const totalAssets = criticalAssets.length; +let loadedCount = 0; + +const promises = criticalAssets.map(asset => { + const promise = resourceManager.load(asset); + promise.then(() => { + loadedCount++; + console.log(`Loaded ${loadedCount}/${totalAssets} assets`); + }); + return promise; +}); + +await Promise.all(promises); +console.log("All critical assets loaded"); +``` + +### Lazy Loading Strategy + +```ts +class AssetManager { + private assetCache = new Map>(); + + async loadOnDemand(url: string, type: AssetType): Promise { + if (!this.assetCache.has(url)) { + const promise = resourceManager.load({ url, type }); + this.assetCache.set(url, promise); + } + return this.assetCache.get(url)!; + } + + async loadLevel(levelId: string) { + // Load level-specific assets lazily + const levelConfig = await this.loadOnDemand(`levels/${levelId}.json`, AssetType.JSON); + const levelAssets = await Promise.all( + levelConfig.assets.map(asset => this.loadOnDemand(asset.url, asset.type)) + ); + return { config: levelConfig, assets: levelAssets }; + } +} +``` + +### Memory Management + +```ts +// Resource cleanup when no longer needed +class ResourceLifecycleManager { + private resourceRefs = new Map(); + + addReference(url: string): void { + const count = this.resourceRefs.get(url) || 0; + this.resourceRefs.set(url, count + 1); + } + + removeReference(url: string): void { + const count = this.resourceRefs.get(url) || 0; + if (count <= 1) { + // Last reference, cleanup resource + resourceManager.cancelNotLoaded(url); + resourceManager.gc(); // Trigger garbage collection + this.resourceRefs.delete(url); + } else { + this.resourceRefs.set(url, count - 1); + } + } + + cleanup(): void { + // Force cleanup of all managed resources + resourceManager.gc(); + this.resourceRefs.clear(); + } +} +``` + +## Error Handling and Recovery + +### Retry Logic + +```ts +class RobustLoader { + async loadWithRetry( + item: LoadItem, + maxRetries: number = 3, + retryDelay: number = 1000 + ): Promise { + let lastError: Error; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await resourceManager.load(item); + } catch (error) { + lastError = error as Error; + + if (attempt < maxRetries) { + console.warn(`Load attempt ${attempt + 1} failed, retrying in ${retryDelay}ms...`); + await this.delay(retryDelay); + retryDelay *= 2; // Exponential backoff + } + } + } + + throw new Error(`Failed to load ${item.url} after ${maxRetries + 1} attempts: ${lastError.message}`); + } + + private delay(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } +} +``` + +### Fallback Resources + +```ts +class FallbackResourceManager { + private fallbacks = new Map(); + + registerFallback(resourceType: string, fallbackUrl: string): void { + this.fallbacks.set(resourceType, fallbackUrl); + } + + async loadWithFallback(item: LoadItem): Promise { + try { + return await resourceManager.load(item); + } catch (error) { + const fallbackUrl = this.fallbacks.get(item.type.toString()); + if (fallbackUrl) { + console.warn(`Failed to load ${item.url}, using fallback: ${fallbackUrl}`); + return await resourceManager.load({ ...item, url: fallbackUrl }); + } + throw error; + } + } +} + +// Setup fallbacks +const fallbackManager = new FallbackResourceManager(); +fallbackManager.registerFallback(AssetType.Texture2D.toString(), "textures/missing.png"); +fallbackManager.registerFallback(AssetType.GLTF.toString(), "models/error.gltf"); +``` + +## Performance Optimization + +### Loading Profiling + +```ts +class LoadingProfiler { + private stats = new Map(); + + async profiledLoad(item: LoadItem): Promise { + const startTime = performance.now(); + + try { + const result = await resourceManager.load(item); + const loadTime = performance.now() - startTime; + + this.updateStats(item.type.toString(), loadTime); + console.log(`Loaded ${item.url} in ${loadTime.toFixed(2)}ms`); + + return result; + } catch (error) { + const failTime = performance.now() - startTime; + console.error(`Failed to load ${item.url} after ${failTime.toFixed(2)}ms:`, error); + throw error; + } + } + + private updateStats(type: string, loadTime: number): void { + const current = this.stats.get(type) || { count: 0, totalTime: 0, avgTime: 0 }; + current.count++; + current.totalTime += loadTime; + current.avgTime = current.totalTime / current.count; + this.stats.set(type, current); + } + + getStats(): Map { + return new Map(this.stats); + } +} +``` + +### Concurrent Loading Limits + +```ts +class ConcurrentLoadManager { + private concurrentLimit = 6; // Browser typical limit + private activeLoads = 0; + private pendingQueue: (() => void)[] = []; + + async load(item: LoadItem): Promise { + return new Promise((resolve, reject) => { + const loadFunction = async () => { + this.activeLoads++; + try { + const result = await resourceManager.load(item); + resolve(result); + } catch (error) { + reject(error); + } finally { + this.activeLoads--; + this.processQueue(); + } + }; + + if (this.activeLoads < this.concurrentLimit) { + loadFunction(); + } else { + this.pendingQueue.push(loadFunction); + } + }); + } + + private processQueue(): void { + if (this.pendingQueue.length > 0 && this.activeLoads < this.concurrentLimit) { + const nextLoad = this.pendingQueue.shift()!; + nextLoad(); + } + } +} +``` + +## API Reference + +```apidoc +Loader: + Properties: + useCache: boolean + - Controls whether loaded resources are cached for reuse. + + Methods: + constructor(useCache: boolean) + - Creates a new loader instance with caching configuration. + initialize(engine: Engine, configuration: EngineConfiguration): Promise + - Optional initialization method called during engine setup. + load(item: LoadItem, resourceManager: ResourceManager): AssetPromise + - Main loading method that must be implemented by subclasses. + +LoadItem: + Properties: + url: string + - URL or path to the resource to load. + type: AssetType + - Type identifier for the resource format. + params?: any + - Optional parameters specific to the loader type. + +AssetPromise: + Properties: + onProgress: (progress: number) => void + - Callback for loading progress updates (0.0 to 1.0). + + Methods: + then(onResolve: (value: T) => void, onReject?: (error: Error) => void): Promise + - Standard Promise.then() implementation. + catch(onReject: (error: Error) => void): Promise + - Standard Promise.catch() implementation. + +ResourceManager: + Methods: + load(item: LoadItem | LoadItem[]): AssetPromise | AssetPromise + - Loads single or multiple resources with type safety. + request(url: string, config?: RequestConfig): Promise + - Low-level request method for custom loading logic. + cancelNotLoaded(url: string): void + - Cancels pending loads for the specified URL. + gc(): void + - Triggers garbage collection of unused resources. +``` + +## Best Practices + +### Type Safety +- Always specify generic types when loading resources +- Use AssetType constants rather than strings for type identification +- Implement proper error handling for loading failures + +### Performance +- Enable caching for reusable resources +- Use concurrent loading limits to prevent browser connection saturation +- Implement resource pooling for frequently loaded/unloaded assets +- Profile loading times to identify bottlenecks + +### Error Resilience +- Implement retry logic for network-dependent loads +- Provide fallback resources for critical assets +- Use loading progress indicators for user experience +- Log loading failures for debugging and analytics + +### Memory Management +- Call resourceManager.gc() when appropriate to free unused resources +- Use weak references for cached resources when possible +- Monitor memory usage patterns during resource-intensive operations +- Implement cleanup strategies for level transitions diff --git a/docs/scripting/Math.md b/docs/scripting/Math.md new file mode 100644 index 0000000000..61f940142a --- /dev/null +++ b/docs/scripting/Math.md @@ -0,0 +1,1030 @@ +# Math Library - LLM Documentation + +## System Overview + +The Math Library provides comprehensive mathematical foundations for the Galacean 3D engine, including vector arithmetic, matrix transformations, quaternion operations, bounding volume calculations, collision detection utilities, and specialized mathematical functions optimized for real-time 3D graphics and game development. + +## Core Architecture + +### Vector Mathematics (Vector2, Vector3, Vector4) + +```typescript +// Vector3 creation and basic operations +const position = new Vector3(10, 5, 0); +const velocity = new Vector3(2, -1, 0); +const direction = new Vector3(); + +// Static operations (recommended for performance) +Vector3.add(position, velocity, position); // Add velocity to position +Vector3.normalize(direction, direction); // Normalize direction vector +const distance = Vector3.distance(position, target); + +// Instance operations (fluent API) +position.add(velocity).normalize().scale(speed); + +// Component access +console.log(`Position: x=${position.x}, y=${position.y}, z=${position.z}`); +position.set(0, 10, 5); // Direct component assignment +``` + +### Matrix Transformations (4x4 Matrices) + +```typescript +// Matrix creation and transformation +const transform = new Matrix(); +const scale = new Vector3(2, 2, 2); +const rotation = new Quaternion(); +const translation = new Vector3(10, 0, 5); + +// Compose transformation matrix +Matrix.affineTransformation(scale, rotation, translation, transform); + +// Individual transformations +const scaleMatrix = new Matrix(); +Matrix.scaling(scale, scaleMatrix); + +const rotationMatrix = new Matrix(); +Matrix.rotationQuaternion(rotation, rotationMatrix); + +const translationMatrix = new Matrix(); +Matrix.translation(translation, translationMatrix); + +// Matrix multiplication (order matters: T * R * S) +Matrix.multiply(translationMatrix, rotationMatrix, transform); +Matrix.multiply(transform, scaleMatrix, transform); +``` + +### Quaternion Rotations + +```typescript +// Quaternion creation and rotation +const rotation = new Quaternion(); +const axis = new Vector3(0, 1, 0); // Y-axis +const angle = Math.PI / 4; // 45 degrees + +// Create rotation from axis-angle +Quaternion.rotationAxisAngle(axis, angle, rotation); + +// Create rotation from Euler angles +Quaternion.rotationEuler(0, Math.PI / 2, 0, rotation); // 90° Y rotation + +// Apply rotation to vector +const point = new Vector3(1, 0, 0); +Vector3.transformByQuat(point, rotation, point); + +// Quaternion interpolation for smooth rotation +const targetRotation = new Quaternion(); +const interpolatedRotation = new Quaternion(); +Quaternion.slerp(rotation, targetRotation, 0.5, interpolatedRotation); +``` + +## Vector Operations + +### Vector3 Comprehensive Usage + +```typescript +class Transform3D { + private position = new Vector3(); + private velocity = new Vector3(); + private acceleration = new Vector3(); + + updateMovement(deltaTime: number) { + // Physics integration using vector operations + const deltaVelocity = Vector3.scale(this.acceleration, deltaTime, new Vector3()); + Vector3.add(this.velocity, deltaVelocity, this.velocity); + + const deltaPosition = Vector3.scale(this.velocity, deltaTime, new Vector3()); + Vector3.add(this.position, deltaPosition, this.position); + + // Apply damping + this.velocity.scale(0.99); + } + + lookAt(target: Vector3) { + const direction = new Vector3(); + Vector3.subtract(target, this.position, direction); + direction.normalize(); + + // Create rotation from direction vector + const rotation = new Quaternion(); + this.createLookRotation(direction, Vector3.up, rotation); + return rotation; + } + + private createLookRotation(forward: Vector3, up: Vector3, out: Quaternion) { + const right = new Vector3(); + Vector3.cross(up, forward, right); + right.normalize(); + + Vector3.cross(forward, right, up); + + // Convert rotation matrix to quaternion (simplified) + Quaternion.rotationMatrix3x3(this.createRotationMatrix(right, up, forward), out); + } +} +``` + +### Vector2 for 2D Operations + +```typescript +// Vector2 for UI, textures, and 2D math +const screenPos = new Vector2(800, 600); +const texCoord = new Vector2(0.5, 0.5); +const uiScale = new Vector2(1.2, 1.2); + +// 2D transformations +Vector2.multiply(texCoord, uiScale, texCoord); +const length = texCoord.length(); +const normalizedCoord = Vector2.normalize(texCoord, new Vector2()); + +// 2D distance and interpolation +const distance2D = Vector2.distance(screenPos, targetPos); +Vector2.lerp(currentPos, targetPos, 0.1, currentPos); // Smooth movement +``` + +### Vector4 for Homogeneous Coordinates + +```typescript +// Vector4 for 4D transformations and color +const homogeneousPoint = new Vector4(10, 5, 0, 1); // Position with w=1 +const colorVector = new Vector4(1, 0.5, 0.2, 0.8); // RGBA color + +// Transform point through matrix +const transformedPoint = new Vector4(); +Vector4.transformByMatrix(homogeneousPoint, worldMatrix, transformedPoint); + +// Perspective division (convert back to 3D) +if (transformedPoint.w !== 0) { + const projectedPoint = new Vector3( + transformedPoint.x / transformedPoint.w, + transformedPoint.y / transformedPoint.w, + transformedPoint.z / transformedPoint.w + ); +} +``` + +## Matrix Mathematics + +### Matrix Construction and Composition + +```typescript +class MatrixBuilder { + static createTRS(translation: Vector3, rotation: Quaternion, scale: Vector3): Matrix { + const result = new Matrix(); + Matrix.affineTransformation(scale, rotation, translation, result); + return result; + } + + static createLookAt(eye: Vector3, target: Vector3, up: Vector3): Matrix { + const viewMatrix = new Matrix(); + Matrix.lookAt(eye, target, up, viewMatrix); + return viewMatrix; + } + + static createPerspective(fov: number, aspect: number, near: number, far: number): Matrix { + const projMatrix = new Matrix(); + Matrix.perspective(fov, aspect, near, far, projMatrix); + return projMatrix; + } + + static createOrthographic(left: number, right: number, bottom: number, top: number, near: number, far: number): Matrix { + const orthoMatrix = new Matrix(); + Matrix.ortho(left, right, bottom, top, near, far, orthoMatrix); + return orthoMatrix; + } +} + +// Usage in camera system +class Camera3D { + private viewMatrix = new Matrix(); + private projectionMatrix = new Matrix(); + private viewProjectionMatrix = new Matrix(); + + updateMatrices(eye: Vector3, target: Vector3, up: Vector3, fov: number, aspect: number, near: number, far: number) { + // Update view matrix + Matrix.lookAt(eye, target, up, this.viewMatrix); + + // Update projection matrix + Matrix.perspective(fov, aspect, near, far, this.projectionMatrix); + + // Combine view and projection + Matrix.multiply(this.projectionMatrix, this.viewMatrix, this.viewProjectionMatrix); + } + + worldToScreen(worldPos: Vector3, screenSize: Vector2): Vector2 { + // Transform world position to clip space + const clipPos = new Vector4(); + Vector3.transformToVec4(worldPos, this.viewProjectionMatrix, clipPos); + + // Perspective division + if (clipPos.w !== 0) { + const ndcX = clipPos.x / clipPos.w; + const ndcY = clipPos.y / clipPos.w; + + // Convert to screen coordinates + return new Vector2( + (ndcX + 1) * 0.5 * screenSize.x, + (1 - ndcY) * 0.5 * screenSize.y + ); + } + + return new Vector2(); + } +} +``` + +### Matrix Decomposition and Analysis + +```typescript +class MatrixAnalyzer { + static decomposeTransform(matrix: Matrix): { translation: Vector3, rotation: Quaternion, scale: Vector3 } { + const translation = new Vector3(); + const rotation = new Quaternion(); + const scale = new Vector3(); + + // Decompose transformation matrix + const success = matrix.decompose(translation, rotation, scale); + + if (!success) { + console.warn("Matrix decomposition failed - matrix may be singular"); + rotation.identity(); + } + + return { translation, rotation, scale }; + } + + static getTransformComponents(matrix: Matrix) { + const translation = new Vector3(); + const rotation = new Quaternion(); + const scale = new Vector3(); + + // Extract individual components + matrix.getTranslation(translation); + matrix.getRotation(rotation); + matrix.getScaling(scale); + + return { + translation, + rotation, + scale, + determinant: matrix.determinant(), + isInvertible: Math.abs(matrix.determinant()) > MathUtil.zeroTolerance + }; + } + + static createInverseTransform(matrix: Matrix): Matrix | null { + const inverse = new Matrix(); + Matrix.invert(matrix, inverse); + + // Check if inversion was successful + if (Math.abs(matrix.determinant()) < MathUtil.zeroTolerance) { + console.error("Cannot invert singular matrix"); + return null; + } + + return inverse; + } +} +``` + +## Bounding Volume Mathematics + +### BoundingBox (AABB) Operations + +```typescript +class BoundingVolumeSystem { + static createAABB(points: Vector3[]): BoundingBox { + const aabb = new BoundingBox(); + BoundingBox.fromPoints(points, aabb); + return aabb; + } + + static createAABBFromSphere(sphere: BoundingSphere): BoundingBox { + const aabb = new BoundingBox(); + BoundingBox.fromSphere(sphere, aabb); + return aabb; + } + + static transformAABB(aabb: BoundingBox, transform: Matrix): BoundingBox { + const transformed = new BoundingBox(); + BoundingBox.transform(aabb, transform, transformed); + return transformed; + } + + static mergeAABBs(aabb1: BoundingBox, aabb2: BoundingBox): BoundingBox { + const merged = new BoundingBox(); + BoundingBox.merge(aabb1, aabb2, merged); + return merged; + } + + // AABB analysis and queries + static analyzeAABB(aabb: BoundingBox) { + const center = new Vector3(); + const extent = new Vector3(); + const corners = aabb.getCorners(); + + aabb.getCenter(center); + aabb.getExtent(extent); + + return { + center, + extent, + corners, + volume: extent.x * extent.y * extent.z * 8, // 8 = 2^3 for full box + surfaceArea: 2 * (extent.x * extent.y + extent.y * extent.z + extent.z * extent.x) * 4 + }; + } +} + +// Usage in collision detection +class CollisionSystem { + static aabbOverlap(aabb1: BoundingBox, aabb2: BoundingBox): boolean { + return ( + aabb1.min.x <= aabb2.max.x && aabb1.max.x >= aabb2.min.x && + aabb1.min.y <= aabb2.max.y && aabb1.max.y >= aabb2.min.y && + aabb1.min.z <= aabb2.max.z && aabb1.max.z >= aabb2.min.z + ); + } + + static aabbContainsPoint(aabb: BoundingBox, point: Vector3): boolean { + return ( + point.x >= aabb.min.x && point.x <= aabb.max.x && + point.y >= aabb.min.y && point.y <= aabb.max.y && + point.z >= aabb.min.z && point.z <= aabb.max.z + ); + } + + static closestPointOnAABB(aabb: BoundingBox, point: Vector3): Vector3 { + const closest = new Vector3(); + + closest.x = Math.max(aabb.min.x, Math.min(point.x, aabb.max.x)); + closest.y = Math.max(aabb.min.y, Math.min(point.y, aabb.max.y)); + closest.z = Math.max(aabb.min.z, Math.min(point.z, aabb.max.z)); + + return closest; + } +} +``` + +### BoundingSphere Operations + +```typescript +class SphereCollision { + static createBoundingSphere(center: Vector3, radius: number): BoundingSphere { + const sphere = new BoundingSphere(); + sphere.center.copyFrom(center); + sphere.radius = radius; + return sphere; + } + + static sphereFromPoints(points: Vector3[]): BoundingSphere { + if (points.length === 0) return new BoundingSphere(); + + // Simple approach: find center and max distance + const center = new Vector3(); + + // Calculate centroid + points.forEach(point => center.add(point)); + center.scale(1 / points.length); + + // Find maximum distance from center + let maxRadiusSquared = 0; + points.forEach(point => { + const distSquared = Vector3.distanceSquared(center, point); + maxRadiusSquared = Math.max(maxRadiusSquared, distSquared); + }); + + const sphere = new BoundingSphere(); + sphere.center.copyFrom(center); + sphere.radius = Math.sqrt(maxRadiusSquared); + return sphere; + } + + static sphereOverlap(sphere1: BoundingSphere, sphere2: BoundingSphere): boolean { + const distance = Vector3.distance(sphere1.center, sphere2.center); + return distance <= (sphere1.radius + sphere2.radius); + } + + static sphereContainsPoint(sphere: BoundingSphere, point: Vector3): boolean { + const distance = Vector3.distance(sphere.center, point); + return distance <= sphere.radius; + } +} +``` + +## Advanced Mathematical Operations + +### Interpolation and Animation Math + +```typescript +class InterpolationMath { + // Linear interpolation with different easing functions + static lerp(start: number, end: number, t: number): number { + return start + (end - start) * t; + } + + static smoothstep(start: number, end: number, t: number): number { + t = Math.max(0, Math.min(1, (t - start) / (end - start))); + return t * t * (3 - 2 * t); + } + + static sineInOut(start: number, end: number, t: number): number { + return start + (end - start) * (1 - Math.cos(t * Math.PI)) * 0.5; + } + + // Vector interpolation with custom easing + static lerpVector3(start: Vector3, end: Vector3, t: number, easingFunc?: (t: number) => number): Vector3 { + const eased = easingFunc ? easingFunc(t) : t; + const result = new Vector3(); + Vector3.lerp(start, end, eased, result); + return result; + } + + // Quaternion spherical linear interpolation + static slerpQuaternion(start: Quaternion, end: Quaternion, t: number): Quaternion { + const result = new Quaternion(); + Quaternion.slerp(start, end, t, result); + return result; + } + + // Matrix interpolation (decompose -> interpolate -> compose) + static lerpMatrix(start: Matrix, end: Matrix, t: number): Matrix { + // Decompose both matrices + const startTrans = new Vector3(), startRot = new Quaternion(), startScale = new Vector3(); + const endTrans = new Vector3(), endRot = new Quaternion(), endScale = new Vector3(); + + start.decompose(startTrans, startRot, startScale); + end.decompose(endTrans, endRot, endScale); + + // Interpolate components + const resultTrans = new Vector3(); + const resultRot = new Quaternion(); + const resultScale = new Vector3(); + + Vector3.lerp(startTrans, endTrans, t, resultTrans); + Quaternion.slerp(startRot, endRot, t, resultRot); + Vector3.lerp(startScale, endScale, t, resultScale); + + // Compose result matrix + const result = new Matrix(); + Matrix.affineTransformation(resultScale, resultRot, resultTrans, result); + return result; + } +} +``` + +### Ray Mathematics and Intersection + +```typescript +class RayMath { + static createRay(origin: Vector3, direction: Vector3): Ray { + const ray = new Ray(); + ray.origin.copyFrom(origin); + ray.direction.copyFrom(direction); + ray.direction.normalize(); + return ray; + } + + static rayPlaneIntersection(ray: Ray, plane: Plane): { hit: boolean, distance: number, point: Vector3 } { + const denom = Vector3.dot(plane.normal, ray.direction); + + if (Math.abs(denom) < MathUtil.zeroTolerance) { + return { hit: false, distance: 0, point: new Vector3() }; + } + + const distance = -(Vector3.dot(plane.normal, ray.origin) + plane.distance) / denom; + + if (distance < 0) { + return { hit: false, distance: 0, point: new Vector3() }; + } + + const point = new Vector3(); + const direction = Vector3.scale(ray.direction, distance, new Vector3()); + Vector3.add(ray.origin, direction, point); + + return { hit: true, distance, point }; + } + + static raySphereIntersection(ray: Ray, sphere: BoundingSphere): { hit: boolean, distance: number, point: Vector3 } { + const oc = new Vector3(); + Vector3.subtract(ray.origin, sphere.center, oc); + + const a = Vector3.dot(ray.direction, ray.direction); + const b = 2.0 * Vector3.dot(oc, ray.direction); + const c = Vector3.dot(oc, oc) - sphere.radius * sphere.radius; + + const discriminant = b * b - 4 * a * c; + + if (discriminant < 0) { + return { hit: false, distance: 0, point: new Vector3() }; + } + + const distance = (-b - Math.sqrt(discriminant)) / (2.0 * a); + + if (distance < 0) { + return { hit: false, distance: 0, point: new Vector3() }; + } + + const point = new Vector3(); + const direction = Vector3.scale(ray.direction, distance, new Vector3()); + Vector3.add(ray.origin, direction, point); + + return { hit: true, distance, point }; + } +} +``` + +### Color Mathematics + +```typescript +class ColorMath { + static rgbToHsv(color: Color): { h: number, s: number, v: number } { + const r = color.r; + const g = color.g; + const b = color.b; + + const max = Math.max(r, g, b); + const min = Math.min(r, g, b); + const diff = max - min; + + let h = 0; + const s = max === 0 ? 0 : diff / max; + const v = max; + + if (diff !== 0) { + switch (max) { + case r: h = ((g - b) / diff + (g < b ? 6 : 0)) / 6; break; + case g: h = ((b - r) / diff + 2) / 6; break; + case b: h = ((r - g) / diff + 4) / 6; break; + } + } + + return { h, s, v }; + } + + static hsvToRgb(h: number, s: number, v: number): Color { + const c = v * s; + const x = c * (1 - Math.abs((h * 6) % 2 - 1)); + const m = v - c; + + let r = 0, g = 0, b = 0; + + if (h < 1/6) { r = c; g = x; b = 0; } + else if (h < 2/6) { r = x; g = c; b = 0; } + else if (h < 3/6) { r = 0; g = c; b = x; } + else if (h < 4/6) { r = 0; g = x; b = c; } + else if (h < 5/6) { r = x; g = 0; b = c; } + else { r = c; g = 0; b = x; } + + return new Color(r + m, g + m, b + m, 1); + } + + static colorLerp(start: Color, end: Color, t: number): Color { + return new Color( + this.lerp(start.r, end.r, t), + this.lerp(start.g, end.g, t), + this.lerp(start.b, end.b, t), + this.lerp(start.a, end.a, t) + ); + } + + static gamma(color: Color, gamma: number): Color { + return new Color( + Math.pow(color.r, gamma), + Math.pow(color.g, gamma), + Math.pow(color.b, gamma), + color.a + ); + } +} +``` + +## Mathematical Utilities and Helpers + +### MathUtil Functions + +```typescript +class ExtendedMathUtil { + // Angle conversions + static degreesToRadians(degrees: number): number { + return degrees * (Math.PI / 180); + } + + static radiansToDegrees(radians: number): number { + return radians * (180 / Math.PI); + } + + // Range mapping + static remap(value: number, fromMin: number, fromMax: number, toMin: number, toMax: number): number { + return toMin + (value - fromMin) * (toMax - toMin) / (fromMax - fromMin); + } + + // Clamping and wrapping + static clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); + } + + static wrap(value: number, min: number, max: number): number { + const range = max - min; + if (range <= 0) return min; + + let result = value; + while (result < min) result += range; + while (result >= max) result -= range; + return result; + } + + // Floating point comparisons + static approximately(a: number, b: number, epsilon: number = MathUtil.zeroTolerance): boolean { + return Math.abs(a - b) < epsilon; + } + + // Smooth interpolation functions + static smoothStep(edge0: number, edge1: number, x: number): number { + const t = this.clamp((x - edge0) / (edge1 - edge0), 0, 1); + return t * t * (3 - 2 * t); + } + + static smootherStep(edge0: number, edge1: number, x: number): number { + const t = this.clamp((x - edge0) / (edge1 - edge0), 0, 1); + return t * t * t * (t * (t * 6 - 15) + 10); + } + + // Random number generation + static randomRange(min: number, max: number): number { + return min + Math.random() * (max - min); + } + + static randomVector3(min: Vector3, max: Vector3): Vector3 { + return new Vector3( + this.randomRange(min.x, max.x), + this.randomRange(min.y, max.y), + this.randomRange(min.z, max.z) + ); + } +} +``` + +### Performance-Optimized Math Operations + +```typescript +class FastMath { + // Fast inverse square root (for normalization) + static fastInverseSqrt(number: number): number { + const threehalfs = 1.5; + let x2 = number * 0.5; + let y = number; + + // Convert to integer for bit manipulation + const buffer = new ArrayBuffer(4); + const floatView = new Float32Array(buffer); + const intView = new Int32Array(buffer); + + floatView[0] = y; + intView[0] = 0x5f3759df - (intView[0] >> 1); + y = floatView[0]; + + y = y * (threehalfs - (x2 * y * y)); + y = y * (threehalfs - (x2 * y * y)); // Second iteration for higher precision + + return y; + } + + // Batch vector operations for performance + static normalizeVectorArray(vectors: Vector3[]): void { + for (let i = 0; i < vectors.length; i++) { + vectors[i].normalize(); + } + } + + static transformPointArray(points: Vector3[], matrix: Matrix, output: Vector3[]): void { + for (let i = 0; i < points.length; i++) { + Vector3.transformToVec3(points[i], matrix, output[i] || (output[i] = new Vector3())); + } + } + + // SIMD-friendly operations (when available) + static addVectorArrays(a: Vector3[], b: Vector3[], output: Vector3[]): void { + const length = Math.min(a.length, b.length); + for (let i = 0; i < length; i++) { + Vector3.add(a[i], b[i], output[i] || (output[i] = new Vector3())); + } + } +} +``` + +## Spatial Mathematics and Algorithms + +### Spatial Partitioning Support + +```typescript +class SpatialMath { + // Octree subdivision helpers + static subdivideAABB(aabb: BoundingBox): BoundingBox[] { + const center = new Vector3(); + aabb.getCenter(center); + + const children: BoundingBox[] = []; + + // Create 8 child AABBs + for (let x = 0; x < 2; x++) { + for (let y = 0; y < 2; y++) { + for (let z = 0; z < 2; z++) { + const childMin = new Vector3( + x === 0 ? aabb.min.x : center.x, + y === 0 ? aabb.min.y : center.y, + z === 0 ? aabb.min.z : center.z + ); + + const childMax = new Vector3( + x === 0 ? center.x : aabb.max.x, + y === 0 ? center.y : aabb.max.y, + z === 0 ? center.z : aabb.max.z + ); + + children.push(new BoundingBox(childMin, childMax)); + } + } + } + + return children; + } + + // Morton code calculation for spatial indexing + static mortonCode3D(x: number, y: number, z: number): number { + x = Math.floor(x) & 0x3ff; + y = Math.floor(y) & 0x3ff; + z = Math.floor(z) & 0x3ff; + + x = (x | (x << 16)) & 0x030000ff; + x = (x | (x << 8)) & 0x0300f00f; + x = (x | (x << 4)) & 0x030c30c3; + x = (x | (x << 2)) & 0x09249249; + + y = (y | (y << 16)) & 0x030000ff; + y = (y | (y << 8)) & 0x0300f00f; + y = (y | (y << 4)) & 0x030c30c3; + y = (y | (y << 2)) & 0x09249249; + + z = (z | (z << 16)) & 0x030000ff; + z = (z | (z << 8)) & 0x0300f00f; + z = (z | (z << 4)) & 0x030c30c3; + z = (z | (z << 2)) & 0x09249249; + + return x | (y << 1) | (z << 2); + } + + // Frustum-AABB intersection test + static frustumAABBIntersection(frustum: BoundingFrustum, aabb: BoundingBox): boolean { + // Get AABB corners + const corners = aabb.getCorners(); + + // Test against each frustum plane + const planes = frustum.getPlanes(); + for (const plane of planes) { + let insideCount = 0; + + for (const corner of corners) { + const distance = Vector3.dot(plane.normal, corner) + plane.distance; + if (distance >= 0) { + insideCount++; + } + } + + // If all corners are outside this plane, no intersection + if (insideCount === 0) { + return false; + } + } + + return true; + } +} +``` + +### Animation and Curve Mathematics + +```typescript +class AnimationMath { + // Bezier curve evaluation + static evaluateCubicBezier(t: number, p0: number, p1: number, p2: number, p3: number): number { + const u = 1 - t; + const u2 = u * u; + const u3 = u2 * u; + const t2 = t * t; + const t3 = t2 * t; + + return u3 * p0 + 3 * u2 * t * p1 + 3 * u * t2 * p2 + t3 * p3; + } + + static evaluateCubicBezierVector3(t: number, p0: Vector3, p1: Vector3, p2: Vector3, p3: Vector3): Vector3 { + return new Vector3( + this.evaluateCubicBezier(t, p0.x, p1.x, p2.x, p3.x), + this.evaluateCubicBezier(t, p0.y, p1.y, p2.y, p3.y), + this.evaluateCubicBezier(t, p0.z, p1.z, p2.z, p3.z) + ); + } + + // Catmull-Rom spline for smooth curves through points + static evaluateCatmullRom(t: number, p0: number, p1: number, p2: number, p3: number): number { + const t2 = t * t; + const t3 = t2 * t; + + return 0.5 * ( + 2 * p1 + + (-p0 + p2) * t + + (2 * p0 - 5 * p1 + 4 * p2 - p3) * t2 + + (-p0 + 3 * p1 - 3 * p2 + p3) * t3 + ); + } + + // Easing functions for animation + static easeInQuart(t: number): number { + return t * t * t * t; + } + + static easeOutQuart(t: number): number { + return 1 - Math.pow(1 - t, 4); + } + + static easeInOutQuart(t: number): number { + return t < 0.5 ? 8 * t * t * t * t : 1 - Math.pow(-2 * t + 2, 4) / 2; + } + + static elasticOut(t: number): number { + const c4 = (2 * Math.PI) / 3; + + return t === 0 ? 0 : t === 1 ? 1 : + Math.pow(2, -10 * t) * Math.sin((t * 10 - 0.75) * c4) + 1; + } +} +``` + +## Integration with Engine Systems + +### Transform System Integration + +```typescript +class MathTransform { + private worldMatrix = new Matrix(); + private localMatrix = new Matrix(); + private position = new Vector3(); + private rotation = new Quaternion(); + private scale = new Vector3(1, 1, 1); + private dirty = true; + + updateWorldMatrix(parentMatrix?: Matrix): Matrix { + if (this.dirty) { + // Compose local transformation matrix + Matrix.affineTransformation(this.scale, this.rotation, this.position, this.localMatrix); + this.dirty = false; + } + + if (parentMatrix) { + // Combine with parent transformation + Matrix.multiply(parentMatrix, this.localMatrix, this.worldMatrix); + } else { + this.worldMatrix.copyFrom(this.localMatrix); + } + + return this.worldMatrix; + } + + setPosition(x: number, y: number, z: number): void { + this.position.set(x, y, z); + this.dirty = true; + } + + setRotationEuler(x: number, y: number, z: number): void { + Quaternion.rotationEuler(x, y, z, this.rotation); + this.dirty = true; + } + + setScale(x: number, y: number, z: number): void { + this.scale.set(x, y, z); + this.dirty = true; + } + + lookAt(target: Vector3, up: Vector3 = Vector3.up): void { + const forward = new Vector3(); + Vector3.subtract(target, this.position, forward); + forward.normalize(); + + const right = new Vector3(); + Vector3.cross(up, forward, right); + right.normalize(); + + const newUp = new Vector3(); + Vector3.cross(forward, right, newUp); + + // Create rotation matrix and convert to quaternion + const rotMatrix = new Matrix3x3(); + rotMatrix.elements[0] = right.x; rotMatrix.elements[1] = right.y; rotMatrix.elements[2] = right.z; + rotMatrix.elements[3] = newUp.x; rotMatrix.elements[4] = newUp.y; rotMatrix.elements[5] = newUp.z; + rotMatrix.elements[6] = forward.x; rotMatrix.elements[7] = forward.y; rotMatrix.elements[8] = forward.z; + + Quaternion.rotationMatrix3x3(rotMatrix, this.rotation); + this.dirty = true; + } +} +``` + +### Physics Integration Helpers + +```typescript +class PhysicsMath { + // Convert between physics and rendering coordinate systems + static worldToPhysics(worldPos: Vector3): Vector3 { + // Galacean typically uses right-handed Y-up, physics might be different + return new Vector3(worldPos.x, worldPos.y, worldPos.z); + } + + static physicsToWorld(physicsPos: Vector3): Vector3 { + return new Vector3(physicsPos.x, physicsPos.y, physicsPos.z); + } + + // Inertia tensor calculations + static calculateBoxInertia(mass: number, extents: Vector3): Matrix3x3 { + const inertia = new Matrix3x3(); + const e = inertia.elements; + + const x2 = extents.x * extents.x; + const y2 = extents.y * extents.y; + const z2 = extents.z * extents.z; + + e[0] = mass * (y2 + z2) / 12; + e[4] = mass * (x2 + z2) / 12; + e[8] = mass * (x2 + y2) / 12; + + return inertia; + } + + static calculateSphereInertia(mass: number, radius: number): Matrix3x3 { + const inertia = new Matrix3x3(); + const e = inertia.elements; + const value = 0.4 * mass * radius * radius; + + e[0] = value; + e[4] = value; + e[8] = value; + + return inertia; + } +} +``` + +## Best Practices + +1. **Performance Optimization**: Use static methods for mathematical operations to avoid object allocation +2. **Output Parameters**: Prefer output parameter pattern to minimize garbage collection +3. **Precision Handling**: Use MathUtil.zeroTolerance for floating-point comparisons +4. **Memory Management**: Reuse Vector3/Matrix objects in hot code paths +5. **Coordinate Systems**: Understand and consistently use right-handed Y-up coordinate system +6. **Matrix Order**: Remember transformation order matters: T * R * S (Translation * Rotation * Scale) +7. **Quaternion Normalization**: Keep quaternions normalized for stable rotations +8. **Bounding Volume Efficiency**: Choose appropriate bounding volume types for different use cases + +## Common Patterns and Solutions + +### Efficient Vector Pooling + +```typescript +class VectorPool { + private static vector3Pool: Vector3[] = []; + private static vector3InUse = new Set(); + + static getVector3(): Vector3 { + let vector = this.vector3Pool.pop(); + if (!vector) { + vector = new Vector3(); + } + this.vector3InUse.add(vector); + return vector; + } + + static releaseVector3(vector: Vector3): void { + if (this.vector3InUse.has(vector)) { + vector.set(0, 0, 0); + this.vector3InUse.delete(vector); + this.vector3Pool.push(vector); + } + } + + static withVector3(callback: (v: Vector3) => T): T { + const vector = this.getVector3(); + try { + return callback(vector); + } finally { + this.releaseVector3(vector); + } + } +} + +// Usage +const distance = VectorPool.withVector3(temp => { + Vector3.subtract(pointA, pointB, temp); + return temp.length(); +}); +``` + +This comprehensive Math Library provides robust mathematical foundations for complex 3D applications with optimized vector operations, matrix transformations, quaternion mathematics, bounding volume calculations, collision detection utilities, and advanced mathematical algorithms essential for real-time 3D graphics and game development. diff --git a/docs/scripting/ParticleSystem.md b/docs/scripting/ParticleSystem.md new file mode 100644 index 0000000000..d98eeefff4 --- /dev/null +++ b/docs/scripting/ParticleSystem.md @@ -0,0 +1,387 @@ +# Particle System - LLM Documentation + +## Overview + +The Galacean Particle System is a sophisticated modular particle effect engine designed for high-performance 3D particle rendering. It implements a comprehensive lifecycle management system with circular buffer architecture, supporting complex particle behaviors through a modular design pattern. + +## Core Architecture + +### ParticleGenerator - Core Engine +```typescript +// Central particle lifecycle management through circular buffer +class ParticleGenerator { + _currentParticleCount = 0; + _firstNewElement = 0; // Start of new particles + _firstActiveElement = 0; // Start of active particles + _firstFreeElement = 0; // Start of free slots + _firstRetiredElement = 0; // Start of retired particles + + // Core lifecycle control + get isAlive(): boolean; + play(withChildren?: boolean): void; + stop(withChildren?: boolean, stopMode?: ParticleStopMode): void; + emit(count: number): void; +} +``` + +**Key Features:** +- Circular buffer particle management for optimal memory usage +- Deterministic particle lifecycle with four distinct states +- Hierarchical control for nested particle systems +- Built-in emission control and timing management + +### ParticleRenderer - Visual Output +```typescript +// Particle rendering with multiple display modes +class ParticleRenderer extends Renderer { + readonly generator: ParticleGenerator; + renderMode: ParticleRenderMode; // Billboard, StretchBillboard, Mesh, etc. + velocityScale = 0; // Velocity-based scaling + lengthScale = 2; // Length scaling for stretch modes + pivot = new Vector3(); // Rotation pivot point +} +``` + +**Render Modes:** +- `Billboard`: Always faces camera +- `StretchBillboard`: Stretches based on velocity +- `Mesh`: Uses custom geometry +- `HorizontalBillboard`: Rotates around Y-axis only +- `VerticalBillboard`: Rotates around X-axis only + +## Module System + +### MainModule - Core Configuration +```typescript +// Primary particle system parameters +class MainModule { + // Timing and control + duration = 5.0; // System duration in seconds + isLoop = true; // Loop playback + startDelay: ParticleCompositeCurve; // Initial delay + simulationSpeed = 1.0; // Playback speed multiplier + + // Initial particle properties + startRotation3D = false; // Enable 3D rotation + startRotationX/Y/Z: ParticleCompositeCurve; + startColor: ParticleCompositeGradient; + startLifetime: ParticleCompositeCurve; + startSpeed: ParticleCompositeCurve; + startSize3D = false; // Enable per-axis sizing + startSizeX/Y/Z: ParticleCompositeCurve; + + // Physics and space + gravityModifier: ParticleCompositeCurve; + simulationSpace: ParticleSimulationSpace; // Local/World + scalingMode: ParticleScaleMode; // Hierarchy/Local/World +} +``` + +### EmissionModule - Particle Spawning +```typescript +// Controls when and how particles are created +class EmissionModule { + // Continuous emission + rateOverTime: ParticleCompositeCurve = 10; // Particles per second + rateOverDistance: ParticleCompositeCurve = 0; // Particles per unit moved + + // Burst emission + bursts: Burst[] = []; // Timed burst events + + // Core emission logic + addBurst(burst: Burst): void; + _emit(lastPlayTime: number, playTime: number): void; +} +``` + +**Burst Configuration:** +```typescript +interface Burst { + time: number; // When to emit (in seconds) + count: number; // Number of particles + cycles: number; // Repeat count (-1 = infinite) + interval: number; // Time between cycles + probability: number; // Chance to emit (0-1) +} +``` + +## Animation System + +### ParticleCompositeCurve - Value Animation +```typescript +// Core animation curve supporting multiple modes +class ParticleCompositeCurve { + mode: ParticleCurveMode; // Constant, TwoConstants, Curve, TwoCurves + + // Constant values + constantMin: number; + constantMax: number; + + // Curve animation + curveMin: ParticleCurve; + curveMax: ParticleCurve; + + // Evaluation + evaluate(time: number, lerpFactor: number): number; +} +``` + +**Curve Modes:** +- `Constant`: Single fixed value +- `TwoConstants`: Random between min/max +- `Curve`: Animated curve over time +- `TwoCurves`: Random between two curves + +### Over-Lifetime Modules + +#### VelocityOverLifetimeModule +```typescript +// Particle velocity animation over lifetime +class VelocityOverLifetimeModule { + velocityX/Y/Z: ParticleCompositeCurve; // Per-axis velocity + space: ParticleSimulationSpace; // Local/World coordinates + + // Shader integration for GPU computation + _updateShaderData(shaderData: ShaderData): void; +} +``` + +#### SizeOverLifetimeModule +```typescript +// Particle size animation over lifetime +class SizeOverLifetimeModule { + separateAxes = false; // Enable per-axis control + sizeX/Y/Z: ParticleCompositeCurve; + + // Unified size control (uses sizeX) + get size(): ParticleCompositeCurve; + set size(value: ParticleCompositeCurve); +} +``` + +#### ColorOverLifetimeModule +```typescript +// Color and alpha animation over lifetime +class ColorOverLifetimeModule { + color: ParticleCompositeGradient; // Color gradient with alpha + + // Supports gradient modes: Gradient, TwoGradients + // Automatically handles color/alpha key interpolation +} +``` + +#### RotationOverLifetimeModule +```typescript +// Particle rotation animation over lifetime +class RotationOverLifetimeModule { + separateAxes = false; // Enable 3D rotation + rotationX/Y/Z: ParticleCompositeCurve; // Per-axis rotation in degrees + + // Automatic degree-to-radian conversion for shaders + // Supports both constant and curve-based rotation +} +``` + +### TextureSheetAnimationModule +```typescript +// Sprite atlas animation for texture-based effects +class TextureSheetAnimationModule { + frameOverTime: ParticleCompositeCurve; // Frame animation curve + type: TextureSheetAnimationType; // WholeSheet, SingleRow + cycleCount = 1; // Animation cycles + tiling: Vector2; // Atlas dimensions (columns, rows) + + // Animation Types: + // WholeSheet: Animate across entire atlas + // SingleRow: Animate single row only +} +``` + +## Shader Integration + +### GPU Computation +The particle system leverages shader macros and properties for high-performance GPU computation: + +```typescript +// Shader macro management +_enableMacro(shaderData: ShaderData, oldMacro: ShaderMacro, newMacro: ShaderMacro): ShaderMacro; + +// Common shader properties +static readonly _positionScale = ShaderProperty.getByName("renderer_PositionScale"); +static readonly _worldPosition = ShaderProperty.getByName("renderer_WorldPosition"); +static readonly _gravity = ShaderProperty.getByName("renderer_Gravity"); + +// Module-specific properties +static readonly _frameMaxCurveProperty = ShaderProperty.getByName("renderer_TSAFrameMaxCurve"); +static readonly _maxCurveXProperty = ShaderProperty.getByName("renderer_SOLMaxCurveX"); +``` + +### Performance Optimization +- **Instance Rendering**: Efficient GPU instancing for thousands of particles +- **Culling**: Automatic bounds calculation for frustum culling +- **Memory Management**: Circular buffer prevents allocation/deallocation overhead +- **Shader Variants**: Dynamic macro compilation for optimal performance + +## Random System + +### Deterministic Randomization +```typescript +// Seeded random generators for reproducible effects +class MainModule { + readonly _startSpeedRand = new Rand(0, ParticleRandomSubSeeds.StartSpeed); + readonly _startLifeTimeRand = new Rand(0, ParticleRandomSubSeeds.StartLifetime); + readonly _startColorRand = new Rand(0, ParticleRandomSubSeeds.StartColor); + + _resetRandomSeed(randomSeed: number): void; +} +``` + +**Random Sub-Seeds:** +- `StartSpeed`, `StartLifetime`, `StartColor`, `StartSize`, `StartRotation` +- `TextureSheetAnimation`, `ColorOverLifetime`, `RotationOverLifetime` +- `VelocityOverLifetime`, `GravityModifier` + +## Simulation Spaces + +### Coordinate System Control +```typescript +enum ParticleSimulationSpace { + Local = 0, // Particles move relative to emitter + World = 1 // Particles move in world coordinates +} + +enum ParticleScaleMode { + Hierarchy = 0, // Use full transform hierarchy scale + Local = 1, // Use local transform scale only + World = 2 // Position scales with world, size stays constant +} +``` + +**Space Implications:** +- **Local**: Particles follow emitter transform, good for attached effects +- **World**: Particles independent of emitter, good for explosions/debris + +## Usage Patterns + +### Basic Particle System +```typescript +// Create particle system entity +const particleEntity = rootEntity.createChild("ParticleSystem"); +const particleRenderer = particleEntity.addComponent(ParticleRenderer); +const generator = particleRenderer.generator; + +// Configure main properties +const main = generator.main; +main.startLifetime.constant = 2.0; +main.startSpeed.constant = 5.0; +main.startColor.constant = new Color(1, 0.5, 0, 1); + +// Setup emission +const emission = generator.emission; +emission.rateOverTime.constant = 50; + +// Add size animation +const sizeOverLifetime = generator.sizeOverLifetime; +sizeOverLifetime.enabled = true; +sizeOverLifetime.size = new ParticleCompositeCurve( + new ParticleCurve( + new CurveKey(0, 0.1), // Start small + new CurveKey(1, 1.0) // Grow over lifetime + ) +); + +// Start playback +generator.play(); +``` + +### Advanced Effect with Multiple Modules +```typescript +// Fire effect with complex animation +const fireGenerator = particleRenderer.generator; + +// Main configuration +fireGenerator.main.startLifetime = new ParticleCompositeCurve(1.5, 3.0); // Random lifetime +fireGenerator.main.startSpeed = new ParticleCompositeCurve(2, 8); +fireGenerator.main.startSize = new ParticleCompositeCurve(0.5, 2.0); + +// Upward velocity with random spread +const velocity = fireGenerator.velocityOverLifetime; +velocity.enabled = true; +velocity.velocityY = new ParticleCompositeCurve(3, 6); + +// Size growth then shrink +const size = fireGenerator.sizeOverLifetime; +size.enabled = true; +size.size = new ParticleCompositeCurve( + new ParticleCurve( + new CurveKey(0, 0.2), + new CurveKey(0.3, 1.0), + new CurveKey(1, 0.1) + ) +); + +// Color from orange to red to black +const color = fireGenerator.colorOverLifetime; +color.enabled = true; +color.color = new ParticleCompositeGradient( + new ParticleGradient( + [ + new GradientColorKey(0, new Color(1, 0.8, 0.2)), // Orange + new GradientColorKey(0.5, new Color(1, 0.2, 0)), // Red + new GradientColorKey(1, new Color(0.2, 0, 0)) // Dark red + ], + [ + new GradientAlphaKey(0, 0.8), + new GradientAlphaKey(1, 0) // Fade out + ] + ) +); + +// Rotation animation +const rotation = fireGenerator.rotationOverLifetime; +rotation.enabled = true; +rotation.rotationZ = new ParticleCompositeCurve(-90, 90); // Random spin +``` + +### Texture Atlas Animation +```typescript +// Animated sprite effect +const spriteGenerator = particleRenderer.generator; + +// Setup texture sheet animation +const textureAnim = spriteGenerator.textureSheetAnimation; +textureAnim.enabled = true; +textureAnim.tiling = new Vector2(4, 4); // 4x4 atlas +textureAnim.frameOverTime = new ParticleCompositeCurve( + new ParticleCurve( + new CurveKey(0, 0), // Start at first frame + new CurveKey(1, 15) // End at last frame (16 frames total) + ) +); +textureAnim.cycleCount = 1; // Play once +``` + +## Best Practices + +### Performance Optimization +1. **Use GPU-optimized modules**: Prefer curve-based animation over per-frame CPU updates +2. **Limit particle count**: Balance visual quality with performance requirements +3. **Choose appropriate simulation space**: Local for attached effects, World for independent particles +4. **Optimize texture usage**: Use texture atlases for sprite-based effects +5. **Profile regularly**: Monitor particle count and frame rate impact + +### Visual Quality +1. **Layer multiple systems**: Combine different particle systems for complex effects +2. **Use proper scaling modes**: Match scaling behavior to effect requirements +3. **Animate multiple properties**: Combine size, color, velocity, and rotation for rich effects +4. **Consider render order**: Use appropriate render modes for desired visual appearance +5. **Test across platforms**: Verify performance on target devices + +### Code Organization +1. **Create reusable presets**: Build effect libraries for common particle patterns +2. **Use descriptive names**: Clear naming for particle system entities and configurations +3. **Document complex effects**: Comment unusual parameter combinations and their visual purpose +4. **Version control settings**: Track particle system configurations in project files +5. **Profile memory usage**: Monitor particle buffer sizes and cleanup procedures + +This particle system provides a comprehensive foundation for creating sophisticated visual effects with optimal performance characteristics suitable for real-time 3D applications. diff --git a/docs/scripting/PhysicsScene.md b/docs/scripting/PhysicsScene.md new file mode 100644 index 0000000000..1fa84a081e --- /dev/null +++ b/docs/scripting/PhysicsScene.md @@ -0,0 +1,834 @@ +# PhysicsScene + +## Overview +PhysicsScene is the core manager of the physics world in the Galacean Engine, responsible for handling all physics simulation, collision detection, ray queries, and shape overlap detection. It serves as the central coordinator of the physics system, managing collider lifecycles, handling physics event callbacks, and providing powerful spatial query capabilities. + +## Core Architecture + +### Physics World Structure +``` +Scene + ↓ +PhysicsScene (Physics World Manager) + ├── Colliders Management + ├── Physics Simulation + ├── Spatial Queries + └── Event System +``` + +### Main Responsibilities +- **Physics Simulation**: Managing gravity, time steps, and physics update loops +- **Collision Detection**: Handling collision enter, exit, and stay events +- **Spatial Queries**: Ray casting, shape sweeping, and overlap detection +- **Layer Management**: Controlling interactions between collision layers +- **Performance Optimization**: Collider sleeping and garbage collection mechanisms + +## API Reference + +### Physics Properties Configuration + +#### Gravity Setting +```typescript +// Gravity vector +gravity: Vector3 // @defaultValue `new Vector3(0, -9.81, 0)` + +// Gravity configuration examples +scene.physics.gravity = new Vector3(0, -9.81, 0); // Standard Earth gravity +scene.physics.gravity = new Vector3(0, -20, 0); // High gravity environment +scene.physics.gravity = new Vector3(0, 0, 0); // Zero gravity environment +``` + +#### Time Step Control +```typescript +// Fixed physics time step +fixedTimeStep: number // @defaultValue `1/60` + +// Time step configuration +scene.physics.fixedTimeStep = 1/60; // 60 FPS physics update +scene.physics.fixedTimeStep = 1/120; // 120 FPS high precision physics +``` + +### Collision Layer Management + +#### Layer Interaction Control +```typescript +// Query whether two collision layers can collide +getColliderLayerCollision(layer1: Layer, layer2: Layer): boolean + +// Set whether two collision layers can collide +setColliderLayerCollision(layer1: Layer, layer2: Layer, enabled: boolean): void + +// Usage examples +// Set player and enemy layers to collide +scene.physics.setColliderLayerCollision(Layer.Layer0, Layer.Layer1, true); + +// Set UI and game objects to not collide +scene.physics.setColliderLayerCollision(Layer.Layer2, Layer.Layer0, false); + +// Query layer interaction +const canCollide = scene.physics.getColliderLayerCollision(Layer.Layer0, Layer.Layer1); +``` + +### Raycasting + +#### Basic Raycast +```typescript +// Simple ray detection +raycast(ray: Ray): boolean + +// Ray detection with distance limit +raycast(ray: Ray, distance: number): boolean + +// Ray detection with result information +raycast(ray: Ray, outHitResult: HitResult): boolean + +// Full parameter ray detection +raycast(ray: Ray, distance: number, layerMask: Layer, outHitResult: HitResult): boolean +``` + +#### Raycast Examples +```typescript +import { Ray, Vector3, HitResult } from "@galacean/engine"; + +// Create ray from camera to mouse position +const ray = camera.screenPointToRay(mousePosition); +const hitResult = new HitResult(); + +// Basic ray detection +if (scene.physics.raycast(ray)) { + console.log("Ray hit something"); +} + +// Ray detection with distance and result +if (scene.physics.raycast(ray, 100, Layer.Everything, hitResult)) { + console.log(`Hit entity: ${hitResult.entity.name}`); + console.log(`Hit distance: ${hitResult.distance}`); + console.log(`Hit point: ${hitResult.point}`); + console.log(`Surface normal: ${hitResult.normal}`); + console.log(`Hit shape: ${hitResult.shape}`); +} + +// Ray detection for specific layers only +const playerLayerMask = Layer.Layer0; +if (scene.physics.raycast(ray, 50, playerLayerMask, hitResult)) { + // Will only hit objects on Layer0 +} +``` + +### Shape Casting + +> **Note**: Shape casting methods (`boxCast`, `sphereCast`, `capsuleCast`) are only available when using PhysX physics engine. They will throw an error with LitePhysics. + +#### Box Cast +```typescript +// Basic box cast +boxCast(center: Vector3, halfExtents: Vector3, direction: Vector3): boolean + +// Box cast with result +boxCast(center: Vector3, halfExtents: Vector3, direction: Vector3, outHitResult: HitResult): boolean + +// Box cast with distance +boxCast(center: Vector3, halfExtents: Vector3, direction: Vector3, distance: number): boolean + +// Box cast with distance and result +boxCast(center: Vector3, halfExtents: Vector3, direction: Vector3, distance: number, outHitResult: HitResult): boolean + +// Full parameter box cast +boxCast( + center: Vector3, + halfExtents: Vector3, + direction: Vector3, + orientation: Quaternion, + distance: number, + layerMask: Layer, + outHitResult?: HitResult +): boolean +``` + +#### Sphere Cast +```typescript +// Basic sphere cast +sphereCast(center: Vector3, radius: number, direction: Vector3): boolean + +// Sphere cast with result +sphereCast(center: Vector3, radius: number, direction: Vector3, outHitResult: HitResult): boolean + +// Sphere cast with distance +sphereCast(center: Vector3, radius: number, direction: Vector3, distance: number): boolean + +// Sphere cast with distance and result +sphereCast(center: Vector3, radius: number, direction: Vector3, distance: number, outHitResult: HitResult): boolean + +// Full parameter sphere cast +sphereCast( + center: Vector3, + radius: number, + direction: Vector3, + distance: number, + layerMask: Layer, + outHitResult?: HitResult +): boolean +``` + +#### Capsule Cast +```typescript +// Basic capsule cast +capsuleCast(center: Vector3, radius: number, height: number, direction: Vector3): boolean + +// Capsule cast with result +capsuleCast(center: Vector3, radius: number, height: number, direction: Vector3, outHitResult: HitResult): boolean + +// Capsule cast with distance +capsuleCast(center: Vector3, radius: number, height: number, direction: Vector3, distance: number): boolean + +// Capsule cast with distance and result +capsuleCast(center: Vector3, radius: number, height: number, direction: Vector3, distance: number, outHitResult: HitResult): boolean + +// Full parameter capsule cast +capsuleCast( + center: Vector3, + radius: number, + height: number, + direction: Vector3, + orientation: Quaternion, + distance: number, + layerMask: Layer, + outHitResult?: HitResult +): boolean +``` + +#### Shape Cast Examples +```typescript +// Character movement collision detection +const characterCenter = transform.position; +const halfSize = new Vector3(0.5, 1, 0.5); +const moveDirection = new Vector3(1, 0, 0); +const hitResult = new HitResult(); + +// Check if character movement path has obstacles +if (scene.physics.boxCast(characterCenter, halfSize, moveDirection, 2.0, Layer.Everything, hitResult)) { + console.log("Movement path blocked"); + // Get collision point for path adjustment + const obstaclePosition = hitResult.point; +} + +// Sphere area attack detection +const attackCenter = weapon.transform.position; +const attackRadius = 3.0; +const attackDirection = weapon.transform.forward; + +if (scene.physics.sphereCast(attackCenter, attackRadius, attackDirection, 5.0)) { + console.log("Attack hit target"); +} +``` + +### Overlap Detection + +#### Box Overlap Detection +```typescript +// Get all collider shapes overlapping with a box +overlapBoxAll( + center: Vector3, + halfExtents: Vector3, + orientation: Quaternion = Quaternion.IDENTITY, + layerMask: Layer = Layer.Everything, + shapes: ColliderShape[] = [] +): ColliderShape[] +``` + +#### Sphere Overlap Detection +```typescript +// Get all collider shapes overlapping with a sphere +overlapSphereAll( + center: Vector3, + radius: number, + layerMask: Layer = Layer.Everything, + shapes: ColliderShape[] = [] +): ColliderShape[] +``` + +#### Capsule Overlap Detection +```typescript +// Get all collider shapes overlapping with a capsule +overlapCapsuleAll( + center: Vector3, + radius: number, + height: number, + orientation: Quaternion = Quaternion.IDENTITY, + layerMask: Layer = Layer.Everything, + shapes: ColliderShape[] = [] +): ColliderShape[] +``` + +#### Overlap Detection Examples +```typescript +// All targets within explosion range +const explosionCenter = bomb.transform.position; +const explosionRadius = 10.0; +const affectedShapes: ColliderShape[] = []; + +// Get all colliders within explosion range +scene.physics.overlapSphereAll(explosionCenter, explosionRadius, Layer.Everything, affectedShapes); + +for (const shape of affectedShapes) { + const entity = shape.collider.entity; + console.log(`${entity.name} affected by explosion`); + + // Apply explosion force to dynamic objects + const dynamicCollider = entity.getComponent(DynamicCollider); + if (dynamicCollider) { + const direction = entity.transform.position.subtract(explosionCenter).normalize(); + const explosionForce = direction.scale(1000); + dynamicCollider.applyForce(explosionForce); + } +} + +// Detect players in trigger area +const triggerCenter = checkpoint.transform.position; +const triggerSize = new Vector3(2, 3, 2); +const playersInTrigger: ColliderShape[] = []; + +scene.physics.overlapBoxAll( + triggerCenter, + triggerSize, + checkpoint.transform.rotation, + Layer.Layer0, // Assume players are on Layer0 + playersInTrigger +); + +if (playersInTrigger.length > 0) { + console.log("Player entered checkpoint"); +} +``` + +### HitResult Information + +#### Collision Result Structure +```typescript +class HitResult { + entity: Entity; // The hit entity + distance: number; // Distance from ray origin to hit point + point: Vector3; // Hit point in world space + normal: Vector3; // Normal of the hit surface + shape: ColliderShape; // The hit collider shape +} +``` + +#### Result Information Application +```typescript +const ray = new Ray(origin, direction); +const hitResult = new HitResult(); + +if (scene.physics.raycast(ray, 100, Layer.Everything, hitResult)) { + // Get components of the hit entity + const renderer = hitResult.entity.getComponent(MeshRenderer); + if (renderer) { + // Change material color of the hit object + renderer.material.baseColor = new Color(1, 0, 0, 1); + } + + // Create effect at hit point + const effect = createEffect(); + effect.transform.position = hitResult.point.clone(); + effect.transform.rotation = Quaternion.lookRotation(hitResult.normal); + + // Calculate damage falloff based on hit distance + const damage = baseDamage * (1 - hitResult.distance / maxDistance); + + // Get material information of the hit shape + const material = hitResult.shape.material; + const surfaceType = getSurfaceType(material.staticFriction, material.bounciness); +} +``` + +## Physics Event System + +### Collision Events +PhysicsScene automatically handles collision events and calls corresponding Script methods: + +```typescript +// Handle collision events in Script component +export class CollisionHandler extends Script { + onCollisionEnter(collision: Collision) { + console.log(`${this.entity.name} started colliding`); + console.log(`Collision object: ${collision.shape.collider.entity.name}`); + } + + onCollisionExit(collision: Collision) { + console.log(`${this.entity.name} stopped colliding`); + } + + onCollisionStay(collision: Collision) { + console.log(`${this.entity.name} continuing collision`); + } +} +``` + +### Trigger Events +```typescript +export class TriggerHandler extends Script { + onTriggerEnter(shape: ColliderShape) { + console.log(`${this.entity.name} trigger activated`); + console.log(`Triggering object: ${shape.collider.entity.name}`); + } + + onTriggerExit(shape: ColliderShape) { + console.log(`${this.entity.name} trigger ended`); + } + + onTriggerStay(shape: ColliderShape) { + console.log(`${this.entity.name} trigger continuing`); + } +} +``` + +## Advanced Application Scenarios + +### Gameplay Applications + +#### First-Person Shooter Game +```typescript +export class WeaponSystem extends Script { + private camera: Camera; + private weapon: Entity; + + onAwake() { + this.camera = this.entity.getComponent(Camera); + } + + shoot() { + // Fire ray from camera center + const ray = new Ray( + this.camera.transform.position, + this.camera.transform.forward + ); + + const hitResult = new HitResult(); + const weaponRange = 100; + const enemyLayer = Layer.Layer1; + + if (this.scene.physics.raycast(ray, weaponRange, enemyLayer, hitResult)) { + // Hit enemy + const enemy = hitResult.entity.getComponent(EnemyController); + if (enemy) { + enemy.takeDamage(this.weaponDamage); + + // Create bullet hole effect at hit point + this.createBulletHole(hitResult.point, hitResult.normal); + } + } + } + + private createBulletHole(position: Vector3, normal: Vector3) { + const bulletHole = this.engine.resourceManager + .getResource("BulletHolePrefab") + .clone(); + + bulletHole.transform.position = position; + bulletHole.transform.rotation = Quaternion.lookRotation(normal); + this.scene.addRootEntity(bulletHole); + } +} +``` + +#### Platform Jump Game +```typescript +export class PlatformerController extends Script { + private collider: DynamicCollider; + private isGrounded: boolean = false; + + onAwake() { + this.collider = this.entity.getComponent(DynamicCollider); + } + + checkGrounded() { + const rayOrigin = this.entity.transform.position; + const rayDirection = new Vector3(0, -1, 0); + const groundCheckDistance = 0.6; + + // Detect if character is on the ground + this.isGrounded = this.scene.physics.raycast( + new Ray(rayOrigin, rayDirection), + groundCheckDistance, + Layer.Layer2 // Ground layer + ); + } + + jump() { + if (this.isGrounded) { + const jumpForce = new Vector3(0, 500, 0); + this.collider.applyForce(jumpForce); + } + } + + moveHorizontal(direction: number) { + // Check for obstacles in movement direction + const moveDirection = new Vector3(direction, 0, 0); + const characterSize = new Vector3(0.5, 1, 0.5); + const moveDistance = 1.0; + + if (!this.scene.physics.boxCast( + this.entity.transform.position, + characterSize, + moveDirection, + moveDistance, + Layer.Layer2 // Obstacle layer + )) { + // No obstacles, can move + const moveForce = moveDirection.scale(300); + this.collider.applyForce(moveForce); + } + } +} +``` + +#### Vehicle Physics +```typescript +export class VehiclePhysics extends Script { + private collider: DynamicCollider; + private wheelColliders: SphereColliderShape[] = []; + + onAwake() { + this.collider = this.entity.getComponent(DynamicCollider); + this.setupWheels(); + } + + private setupWheels() { + // Create ground detection for each wheel + const wheelPositions = [ + new Vector3(-1, -0.5, 1.5), // Front left wheel + new Vector3(1, -0.5, 1.5), // Front right wheel + new Vector3(-1, -0.5, -1.5), // Rear left wheel + new Vector3(1, -0.5, -1.5) // Rear right wheel + ]; + + wheelPositions.forEach(pos => { + const wheelShape = new SphereColliderShape(); + wheelShape.radius = 0.3; + wheelShape.position = pos; + this.wheelColliders.push(wheelShape); + }); + } + + simulateWheelPhysics() { + this.wheelColliders.forEach((wheel, index) => { + const wheelWorldPos = this.entity.transform.position.add(wheel.position); + const downRay = new Ray(wheelWorldPos, new Vector3(0, -1, 0)); + const hitResult = new HitResult(); + + if (this.scene.physics.raycast(downRay, 1.0, Layer.Layer2, hitResult)) { + // Wheel is touching the ground, apply suspension force + const suspensionDistance = hitResult.distance; + const suspensionForce = this.calculateSuspensionForce(suspensionDistance); + + // Apply upward force at wheel position + this.collider.applyForce(suspensionForce); + } + }); + } + + private calculateSuspensionForce(distance: number): Vector3 { + const springStrength = 1000; + const targetDistance = 0.5; + const compression = Math.max(0, targetDistance - distance); + return new Vector3(0, compression * springStrength, 0); + } +} +``` + +### Advanced Query Applications + +#### AI Vision Detection +```typescript +export class AIVision extends Script { + private viewDistance: number = 20; + private viewAngle: number = 60; + + canSeeTarget(target: Entity): boolean { + const eyePosition = this.entity.transform.position.add(new Vector3(0, 1.6, 0)); + const targetPosition = target.transform.position.add(new Vector3(0, 1, 0)); + + const direction = targetPosition.subtract(eyePosition); + const distance = direction.length(); + + // Check distance + if (distance > this.viewDistance) return false; + + // Check angle + const forward = this.entity.transform.forward; + const angle = Vector3.angle(forward, direction.normalize()); + if (angle > this.viewAngle / 2) return false; + + // Raycast for occlusion + const ray = new Ray(eyePosition, direction.normalize()); + const hitResult = new HitResult(); + + if (this.scene.physics.raycast(ray, distance, Layer.Everything, hitResult)) { + // If hit target, can see + return hitResult.entity === target; + } + + return true; // Nothing hit, can see + } + + scanForEnemies(): Entity[] { + const visibleEnemies: Entity[] = []; + const scanCenter = this.entity.transform.position; + const scanRadius = this.viewDistance; + const enemyShapes: ColliderShape[] = []; + + // Get all enemies within scan range + this.scene.physics.overlapSphereAll( + scanCenter, + scanRadius, + Layer.Layer1, // Enemy layer + enemyShapes + ); + + for (const shape of enemyShapes) { + const enemy = shape.collider.entity; + if (this.canSeeTarget(enemy)) { + visibleEnemies.push(enemy); + } + } + + return visibleEnemies; + } +} +``` + +#### Environment Interaction System +```typescript +export class InteractionSystem extends Script { + private interactionRange: number = 2.0; + + getInteractableObjects(): Entity[] { + const playerPosition = this.entity.transform.position; + const interactableShapes: ColliderShape[] = []; + + // Get all objects within interaction range + this.scene.physics.overlapSphereAll( + playerPosition, + this.interactionRange, + Layer.Layer3, // Interactable object layer + interactableShapes + ); + + return interactableShapes + .map(shape => shape.collider.entity) + .filter(entity => entity.getComponent(Interactable)); + } + + getClosestInteractable(): Entity | null { + const interactables = this.getInteractableObjects(); + if (interactables.length === 0) return null; + + const playerPos = this.entity.transform.position; + let closest: Entity | null = null; + let closestDistance = Infinity; + + for (const interactable of interactables) { + const distance = Vector3.distance(playerPos, interactable.transform.position); + if (distance < closestDistance) { + closestDistance = distance; + closest = interactable; + } + } + + return closest; + } +} +``` + +## Performance Optimization + +### Query Optimization Strategies +1. **Layer Separation**: Place different types of objects on different collision layers +2. **Distance Culling**: Limit query distance to avoid unnecessary remote detection +3. **Frequency Control**: Queries that don't need to run every frame can reduce frequency +4. **Result Caching**: Cache query results to avoid repeated calculations + +```typescript +export class OptimizedPhysicsQueries extends Script { + private lastScanTime: number = 0; + private scanInterval: number = 0.1; // Scan every 100ms + private cachedResults: Entity[] = []; + + onUpdate(deltaTime: number) { + const currentTime = this.engine.time.nowTime; + + if (currentTime - this.lastScanTime > this.scanInterval) { + this.performExpensiveScan(); + this.lastScanTime = currentTime; + } + + // Use cached results for fast operations + this.processResults(this.cachedResults); + } + + private performExpensiveScan() { + // Only scan relevant layers + const relevantLayers = Layer.Layer1 | Layer.Layer2; + const scanRange = 10; // Limit scan range + + const shapes: ColliderShape[] = []; + this.scene.physics.overlapSphereAll( + this.entity.transform.position, + scanRange, + relevantLayers, + shapes + ); + + this.cachedResults = shapes.map(s => s.collider.entity); + } +} +``` + +### Memory Management +```typescript +export class PhysicsMemoryManager { + private hitResultPool: HitResult[] = []; + private shapeArrayPool: ColliderShape[][] = []; + + getHitResult(): HitResult { + return this.hitResultPool.pop() || new HitResult(); + } + + returnHitResult(result: HitResult) { + // Clear result object + result.entity = null; + result.shape = null; + result.distance = 0; + result.point.set(0, 0, 0); + result.normal.set(0, 0, 0); + + this.hitResultPool.push(result); + } + + getShapeArray(): ColliderShape[] { + const array = this.shapeArrayPool.pop() || []; + array.length = 0; // Clear array + return array; + } + + returnShapeArray(array: ColliderShape[]) { + if (array.length < 100) { // Avoid caching very large arrays + this.shapeArrayPool.push(array); + } + } +} +``` + +## Best Practices + +### Query Design Principles +1. **Clear Purpose**: Each query should have a clear game logic purpose +2. **Minimal Scope**: Use the minimum necessary query range and layer filtering +3. **Reasonable Frequency**: Set appropriate query frequency based on game requirements +4. **Result Processing**: Process query results promptly to avoid accumulation + +### Layer Planning Recommendations +```typescript +// Recommended layer allocation +enum GameLayers { + Player = Layer.Layer0, + Enemy = Layer.Layer1, + Environment = Layer.Layer2, + Interactable = Layer.Layer3, + Projectile = Layer.Layer4, + Trigger = Layer.Layer5, + UI = Layer.Layer6, + Effect = Layer.Layer7 +} + +// Setup layer interaction matrix +function setupCollisionMatrix(physics: PhysicsScene) { + // Player collides with enemies, environment, and interactive objects + physics.setColliderLayerCollision(GameLayers.Player, GameLayers.Enemy, true); + physics.setColliderLayerCollision(GameLayers.Player, GameLayers.Environment, true); + physics.setColliderLayerCollision(GameLayers.Player, GameLayers.Interactable, false); // Trigger mode + + // Projectiles collide with players, enemies, and environment + physics.setColliderLayerCollision(GameLayers.Projectile, GameLayers.Player, true); + physics.setColliderLayerCollision(GameLayers.Projectile, GameLayers.Enemy, true); + physics.setColliderLayerCollision(GameLayers.Projectile, GameLayers.Environment, true); + + // UI layer doesn't interact with any physics layers + physics.setColliderLayerCollision(GameLayers.UI, GameLayers.Player, false); + physics.setColliderLayerCollision(GameLayers.UI, GameLayers.Enemy, false); + + // Effect layer is visual only, doesn't participate in physics interactions + physics.setColliderLayerCollision(GameLayers.Effect, GameLayers.Player, false); + physics.setColliderLayerCollision(GameLayers.Effect, GameLayers.Enemy, false); +} +``` + +### Debugging and Monitoring +```typescript +export class PhysicsDebugger extends Script { + private static debugRays: Array<{ray: Ray, color: Color, duration: number}> = []; + + static drawRay(ray: Ray, color: Color = Color.red, duration: number = 1.0) { + this.debugRays.push({ray, color, duration}); + } + + onUpdate(deltaTime: number) { + // Visualize rays (in development mode) + if (this.engine.debug) { + this.debugRays.forEach(debugRay => { + this.drawDebugLine( + debugRay.ray.origin, + debugRay.ray.origin.add(debugRay.ray.direction.scale(10)), + debugRay.color + ); + debugRay.duration -= deltaTime; + }); + + this.debugRays = this.debugRays.filter(ray => ray.duration > 0); + } + } + + private drawDebugLine(start: Vector3, end: Vector3, color: Color) { + // Implement debug line drawing + // This requires a debug rendering system + } +} + +// Use debug feature in queries +export class DebuggableRaycast extends Script { + performRaycast() { + const ray = new Ray(this.entity.transform.position, this.entity.transform.forward); + const hitResult = new HitResult(); + + // Draw debug ray + PhysicsDebugger.drawRay(ray, Color.blue, 0.5); + + if (this.scene.physics.raycast(ray, 10, Layer.Everything, hitResult)) { + // Draw hit point + PhysicsDebugger.drawRay( + new Ray(hitResult.point, hitResult.normal), + Color.red, + 1.0 + ); + } + } +} +``` + +## Important Notes + +### Performance Considerations +- Computational complexity of shape queries: Sphere < Capsule < Box +- Overlap queries are more efficient than sweep queries +- Excessive spatial queries can affect frame rate +- Use layer filtering reasonably to reduce detection range + +### Precision Issues +- Fast-moving objects may penetrate thin walls (use continuous collision detection) +- Ray origins cannot be inside colliders +- Extremely small contactOffset may cause precision issues + +### Design Limitations +- Collision layer settings must be single-layer, multi-layer combinations not supported +- HitResult objects require manual lifecycle management +- Physics queries are immediate, cannot cache internal state across frames diff --git a/docs/scripting/PostProcess.md b/docs/scripting/PostProcess.md new file mode 100644 index 0000000000..9e149c7e9c --- /dev/null +++ b/docs/scripting/PostProcess.md @@ -0,0 +1,1003 @@ +# PostProcess + +Galacean's PostProcess system provides powerful image post-processing capabilities including global and local post-processing modes. The system uses a component-based design supporting multiple built-in effects and custom effect extensions for enhanced visual rendering. + +## Overview + +The PostProcess system encompasses comprehensive image enhancement capabilities: + +- **Component Architecture**: Modular effect system with parameter management and blending +- **Processing Modes**: Global scene-wide effects and local region-based processing +- **Built-in Effects**: Bloom, tone mapping, and extensible effect library +- **Custom Effects**: Framework for developing custom post-processing shaders +- **Performance Optimization**: Effect validity checking and conditional rendering +- **Blend Distance**: Smooth transitions between local post-processing regions + +The system integrates seamlessly with the rendering pipeline to provide high-quality visual enhancements. + +## Quick Start + +### Global Post-Processing Setup + +```ts +import { BloomEffect, TonemappingEffect, TonemappingMode } from "@galacean/engine"; + +const engine = await WebGLEngine.create({ canvas: "canvas" }); +const scene = engine.sceneManager.activeScene; + +// Create entity with PostProcess component +const postProcessEntity = scene.createRootEntity("PostProcess"); +const postProcess = postProcessEntity.addComponent(PostProcess); +postProcess.isGlobal = true; + +// Add bloom effect +const bloom = postProcess.addEffect(BloomEffect); +bloom.intensity.value = 1.0; +bloom.threshold.value = 0.8; +bloom.scatter.value = 0.7; +bloom.tint.value.set(1, 1, 1, 1); + +// Add tone mapping +const tonemapping = postProcess.addEffect(TonemappingEffect); +tonemapping.mode.value = TonemappingMode.ACES; + +engine.run(); +``` + +### Local Post-Processing Regions + +```ts +import { BoxColliderShape, PostProcess, StaticCollider, Layer } from "@galacean/engine"; + +// Create local post-processing entity +const localPostEntity = scene.createRootEntity("LocalPostProcess"); +const localPost = localPostEntity.addComponent(PostProcess); + +// Configure local mode +localPost.isGlobal = false; +localPost.blendDistance = 5.0; // Blend distance +localPost.layer = Layer.Layer1; // Layer assignment + +// Add collider to define affected region +const collider = localPostEntity.addComponent(StaticCollider); +const boxShape = new BoxColliderShape(); +boxShape.size.set(10, 10, 10); +collider.addShape(boxShape); + +// Add effects to local region +const localBloom = localPost.addEffect(BloomEffect); +localBloom.intensity.value = 2.0; + +// Position the local post-processing region +localPostEntity.transform.setPosition(0, 0, 0); +``` + +### Camera Post-Processing Masks + +```ts +// Configure camera to process only specific layers +camera.postProcessMask = Layer.Layer0 | Layer.Layer1; +``` + +## PostProcess Component + +### Core Properties and Methods + +```ts +class PostProcess extends Component { + // Layer identification + layer: Layer = Layer.Layer0; + + // Global/local mode toggle + get isGlobal(): boolean; + set isGlobal(value: boolean); + + // Local post-processing blend distance + blendDistance: number = 0; + + // Priority (affects execution order) + priority: number = 0; + + // Effect management + addEffect(type: T): InstanceType; + removeEffect(type: T): InstanceType; + getEffect(type: T): InstanceType; + clearEffects(): void; +} +``` + +### Effect Management Examples + +```ts +// Dynamic effect management +const bloom = postProcess.addEffect(BloomEffect); +bloom.enabled = true; + +// Runtime parameter adjustment +bloom.intensity.value = 1.5; +bloom.threshold.value = 0.9; + +// Conditional effect removal +if (lowQualityMode) { + postProcess.removeEffect(BloomEffect); +} + +// Get existing effect for adjustment +const existingBloom = postProcess.getEffect(BloomEffect); +if (existingBloom) { + existingBloom.scatter.value *= 0.5; +} + +// Clear all effects +postProcess.clearEffects(); +``` + +## PostProcessEffect Base Class + +### Core Design and Parameters + +```ts +class PostProcessEffect { + // Enable state + get enabled(): boolean; + set enabled(value: boolean); + + // Validity check (can be overridden) + isValid(): boolean; + + // Effect blending (internal use) + _lerp(to: PostProcessEffect, factor: number): void; +} + +// Parameter system +class PostProcessEffectBoolParameter { + constructor(defaultValue: boolean); + value: boolean; +} + +class PostProcessEffectFloatParameter { + constructor(defaultValue: number, min?: number, max?: number); + value: number; + min: number; + max: number; +} + +class PostProcessEffectEnumParameter { + constructor(enumType: any, defaultValue: any); + value: any; +} + +class PostProcessEffectColorParameter { + constructor(defaultValue: Color); + value: Color; +} + +class PostProcessEffectTextureParameter { + constructor(defaultValue: Texture2D); + value: Texture2D; +} +``` + +## Built-in Effects + +### BloomEffect + +```ts +class BloomEffect extends PostProcessEffect { + // High quality filtering + highQualityFiltering = new PostProcessEffectBoolParameter(false); + + // Downscale mode + downScale = new PostProcessEffectEnumParameter(BloomDownScaleMode, BloomDownScaleMode.Half); + + // Dirt texture overlay + dirtTexture = new PostProcessEffectTextureParameter(null); + + // Brightness threshold + threshold = new PostProcessEffectFloatParameter(0.8, 0); + + // Scatter radius + scatter = new PostProcessEffectFloatParameter(0.7, 0, 1); + + // Effect intensity + intensity = new PostProcessEffectFloatParameter(0, 0); + + // Dirt intensity + dirtIntensity = new PostProcessEffectFloatParameter(0, 0); + + // Color tint + tint = new PostProcessEffectColorParameter(new Color(1, 1, 1, 1)); + + // Validity check + override isValid(): boolean { + return this.enabled && this.intensity.value > 0; + } +} + +enum BloomDownScaleMode { + Half = "Half", + Quarter = "Quarter" +} +``` + +#### Bloom Configuration Examples + +```ts +const bloom = postProcess.addEffect(BloomEffect); + +// Basic settings +bloom.threshold.value = 1.0; // Brightness threshold +bloom.intensity.value = 0.8; // Bloom intensity +bloom.scatter.value = 0.6; // Scatter range + +// Advanced settings +bloom.highQualityFiltering.value = true; // High quality filtering +bloom.downScale.value = BloomDownScaleMode.Quarter; // Quarter sampling + +// Dirt effect +bloom.dirtTexture.value = dirtTexture; +bloom.dirtIntensity.value = 0.3; + +// Color adjustment +bloom.tint.value.set(1.0, 0.9, 0.8, 1.0); // Warm tone +``` + +### TonemappingEffect + +```ts +class TonemappingEffect extends PostProcessEffect { + // Tone mapping mode + mode = new PostProcessEffectEnumParameter(TonemappingMode, TonemappingMode.Neutral); +} + +enum TonemappingMode { + Neutral = "Neutral", // Neutral tone mapping + ACES = "ACES" // ACES filmic tone mapping +} +``` + +#### Tone Mapping Usage + +```ts +const tonemapping = postProcess.addEffect(TonemappingEffect); + +// Select tone mapping algorithm +tonemapping.mode.value = TonemappingMode.ACES; // Cinematic tone mapping +// or +tonemapping.mode.value = TonemappingMode.Neutral; // Neutral tone mapping +``` + +## Detailed Effect Configuration + +### BloomEffect Advanced Configuration + +The BloomEffect provides comprehensive control over bloom rendering with multiple quality and performance options: + +```ts +const bloom = postProcess.addEffect(BloomEffect); + +// Core bloom parameters +bloom.threshold.value = 1.0; // Brightness threshold (linear space) +bloom.intensity.value = 0.8; // Overall bloom strength +bloom.scatter.value = 0.6; // Bloom spread radius (0.0-1.0) +bloom.tint.value.set(1, 0.9, 0.8, 1); // Warm color tint + +// Quality settings +bloom.highQualityFiltering.value = true; // Bicubic vs bilinear upsampling +bloom.downScale.value = BloomDownScaleMode.Half; // Half or Quarter resolution + +// Dirt lens effect +bloom.dirtTexture.value = dirtTexture; // Lens dirt texture +bloom.dirtIntensity.value = 0.3; // Dirt effect strength + +// Performance optimization examples +function configureBloomForPlatform(bloom: BloomEffect, platform: Platform) { + switch (platform) { + case Platform.Mobile: + bloom.downScale.value = BloomDownScaleMode.Quarter; + bloom.highQualityFiltering.value = false; + break; + case Platform.Desktop: + bloom.downScale.value = BloomDownScaleMode.Half; + bloom.highQualityFiltering.value = true; + break; + } +} +``` + +#### Bloom Shader Implementation Details + +The bloom effect uses a multi-pass approach with prefilter, blur, and upsample stages: + +```ts +// Bloom shader passes (internal implementation) +enum BloomPasses { + Prefilter = 0, // Brightness thresholding and knee softening + BlurH = 1, // Horizontal Gaussian blur + BlurV = 2, // Vertical Gaussian blur + Upsample = 3 // Bicubic/bilinear upsampling with additive blending +} + +// Threshold calculation with soft knee +const thresholdLinear = bloom.threshold.value; +const thresholdKnee = thresholdLinear * 0.5; // Hardcoded soft knee +const scatterLerp = MathUtil.lerp(0.05, 0.95, bloom.scatter.value); + +// Shader parameters passed to GPU +bloomParams.x = thresholdLinear; // Brightness threshold +bloomParams.y = thresholdKnee; // Soft knee value +bloomParams.z = scatterLerp; // Scatter interpolation +``` + +### FXAA Anti-Aliasing Implementation + +FXAA (Fast Approximate Anti-Aliasing) is a post-processing technique that smooths all pixel edges: + +```ts +// Enable FXAA on camera +camera.antiAliasing = AntiAliasing.FXAA; + +// FXAA shader parameters (internal constants) +const FXAA_PARAMS = { + SUBPIXEL_BLEND_AMOUNT: 0.75, // Subpixel aliasing removal + RELATIVE_CONTRAST_THRESHOLD: 0.166, // Edge detection sensitivity + ABSOLUTE_CONTRAST_THRESHOLD: 0.0833 // Minimum contrast for processing +}; +``` + +#### FXAA Quality Presets + +FXAA uses predefined quality presets that balance performance and visual quality: + +```ts +// FXAA quality presets (compile-time constants) +enum FXAAQualityPreset { + Performance = 10, // Fastest, basic edge smoothing + Default = 12, // Balanced quality and performance + Quality = 15, // Higher quality, slightly slower + HighQuality = 23, // Best quality, more expensive + Extreme = 39 // Maximum quality, very expensive +} + +// FXAA processing pipeline +class FXAAProcessor { + // 1. Luminance calculation (if not pre-computed) + computeLuminance(color: Color): number { + return 0.299 * color.r + 0.587 * color.g + 0.114 * color.b; + } + + // 2. Edge detection using local contrast + detectEdges(center: number, neighbors: number[]): boolean { + const maxLuma = Math.max(center, ...neighbors); + const minLuma = Math.min(center, ...neighbors); + const range = maxLuma - minLuma; + + return range > Math.max( + FXAA_PARAMS.ABSOLUTE_CONTRAST_THRESHOLD, + maxLuma * FXAA_PARAMS.RELATIVE_CONTRAST_THRESHOLD + ); + } + + // 3. Subpixel and edge blending + blendPixel(originalColor: Color, edgeDirection: Vector2): Color { + // Implementation details handled by FXAA shader + return originalColor; // Simplified + } +} +``` + +#### FXAA vs MSAA Comparison + +```ts +// Performance and quality comparison +class AntiAliasingComparison { + static compare() { + return { + FXAA: { + performance: "High", + coverage: "All edges (geometry + shader-generated)", + quality: "Good for most cases", + memoryUsage: "Low", + hardwareRequirement: "Any GPU" + }, + MSAA: { + performance: "Medium to Low", + coverage: "Geometry edges only", + quality: "Excellent for geometry", + memoryUsage: "High (2x-8x)", + hardwareRequirement: "Hardware MSAA support" + } + }; + } + + // Hybrid approach for best results + static configureHybridAA(camera: Camera, quality: QualityLevel) { + switch (quality) { + case QualityLevel.Low: + camera.msaaSamples = MSAASamples.None; + camera.antiAliasing = AntiAliasing.FXAA; + break; + case QualityLevel.Medium: + camera.msaaSamples = MSAASamples.TwoX; + camera.antiAliasing = AntiAliasing.FXAA; + break; + case QualityLevel.High: + camera.msaaSamples = MSAASamples.FourX; + camera.antiAliasing = AntiAliasing.None; // MSAA sufficient + break; + } + } +} +``` + +## Custom Effect Development + +### Creating Custom Effect Classes + +```ts +import { PostProcessEffect, PostProcessEffectFloatParameter } from "@galacean/engine"; + +class VignetteEffect extends PostProcessEffect { + // Define parameters with constraints + intensity = new PostProcessEffectFloatParameter(0.5, 0, 1); + smoothness = new PostProcessEffectFloatParameter(0.8, 0, 1); + center = new PostProcessEffectVector2Parameter(new Vector2(0.5, 0.5)); + + // Override validity check + override isValid(): boolean { + return this.enabled && this.intensity.value > 0; + } +} +``` + +### Advanced Parameter Types + +```ts +import { + PostProcessEffectBoolParameter, + PostProcessEffectColorParameter, + PostProcessEffectEnumParameter, + PostProcessEffectTextureParameter, + PostProcessEffectVector2Parameter, + PostProcessEffectVector3Parameter, + PostProcessEffectVector4Parameter +} from "@galacean/engine"; + +class AdvancedEffect extends PostProcessEffect { + // Boolean parameter + enabled = new PostProcessEffectBoolParameter(true); + + // Float parameter with min/max constraints + intensity = new PostProcessEffectFloatParameter(1.0, 0.0, 2.0, true); // needLerp = true + + // Color parameter with alpha + tint = new PostProcessEffectColorParameter(new Color(1, 1, 1, 1)); + + // Vector parameters + offset = new PostProcessEffectVector2Parameter(new Vector2(0, 0)); + direction = new PostProcessEffectVector3Parameter(new Vector3(0, 1, 0)); + transform = new PostProcessEffectVector4Parameter(new Vector4(1, 1, 0, 0)); + + // Texture parameter + maskTexture = new PostProcessEffectTextureParameter(null); + + // Enum parameter + blendMode = new PostProcessEffectEnumParameter(BlendMode, BlendMode.Normal); + + // Parameter interpolation control + constructor() { + super(); + // Disable interpolation for specific parameters + this.blendMode.enabled = false; // Won't interpolate in local post-processing + } +} +``` + +### Creating Custom Pass Classes + +```ts +import { + PostProcessPass, + PostProcessPassEvent, + Blitter, + Material, + Shader +} from "@galacean/engine"; + +class CustomGrayScalePass extends PostProcessPass { + private _material: Material; + + constructor(engine: Engine) { + super(engine); + this.event = PostProcessPassEvent.AfterUber; // Execute after Uber pass + this._material = this.createGrayScaleMaterial(); + } + + private createGrayScaleMaterial(): Material { + const shader = Shader.create( + "GrayScale", + // Vertex shader + ` + attribute vec4 POSITION_UV; + varying vec2 v_uv; + + void main() { + gl_Position = vec4(POSITION_UV.xy, 0.0, 1.0); + v_uv = POSITION_UV.zw; + } + `, + // Fragment shader + ` + precision mediump float; + varying vec2 v_uv; + uniform sampler2D renderer_BlitTexture; + uniform float u_intensity; + + void main() { + vec4 color = texture2D(renderer_BlitTexture, v_uv); + float gray = dot(color.rgb, vec3(0.299, 0.587, 0.114)); + gl_FragColor = vec4(mix(color.rgb, vec3(gray), u_intensity), color.a); + } + ` + ); + + const material = new Material(this.engine, shader); + + // Configure render state + const depthState = material.renderState.depthState; + depthState.enabled = false; + depthState.writeEnabled = false; + + return material; + } + + // Render implementation + override onRender(camera: Camera, srcTexture: Texture2D, destTarget: RenderTarget): void { + // Get blended effect data from post-process manager + const postProcessManager = camera.scene.postProcessManager; + const customEffect = postProcessManager.getBlendEffect(CustomGrayScaleEffect); + + if (customEffect && customEffect.isValid()) { + // Set shader parameters + this._material.shaderData.setFloat("u_intensity", customEffect.intensity.value); + + // Render using Blitter utility + Blitter.blitTexture( + this.engine, + srcTexture, + destTarget, + undefined, // source region (null = full texture) + undefined, // destination viewport (null = full target) + this._material, + 0 // pass index + ); + } else { + // Pass-through if effect is disabled + Blitter.blitTexture(this.engine, srcTexture, destTarget); + } + } + + // Cleanup resources + override destroy(): void { + this._material?.destroy(); + super.destroy(); + } +} + +// Corresponding effect class +class CustomGrayScaleEffect extends PostProcessEffect { + intensity = new PostProcessEffectFloatParameter(0.5, 0, 1); + + override isValid(): boolean { + return this.enabled && this.intensity.value > 0; + } +} +``` + +### Shader Implementation + +```glsl +// Vignette.glsl +#ifdef ENABLE_EFFECT_VIGNETTE + vec2 vignetteUV = v_uv - 0.5; + float vignette = 1.0 - dot(vignetteUV, vignetteUV) * material_VignetteIntensity; + vignette = smoothstep(0.0, material_VignetteSmoothness, vignette); + color.rgb *= vignette; +#endif +``` + +### Shader Property Registration + +```ts +class VignetteEffect extends PostProcessEffect { + static readonly SHADER_NAME = "PostProcessEffect Vignette"; + + // Shader properties + static _enableMacro: ShaderMacro = ShaderMacro.getByName("ENABLE_EFFECT_VIGNETTE"); + static _intensityProp = ShaderProperty.getByName("material_VignetteIntensity"); + static _smoothnessProp = ShaderProperty.getByName("material_VignetteSmoothness"); + + // Parameter definitions + intensity = new PostProcessEffectFloatParameter(0.5, 0, 1); + smoothness = new PostProcessEffectFloatParameter(0.8, 0, 1); +} +``` + +## PostProcessManager + +### Core Functionality + +```ts +class PostProcessManager { + // Get blended effect result + getBlendEffect(type: T): InstanceType; + + // Internal methods + _update(camera: Camera): void; // Update effect blending + _render(camera: Camera, src: RenderTarget, dest: RenderTarget): void; // Render + _isValid(): boolean; // Check for valid passes +} +``` + +### Blending Mechanism + +```ts +// Get blended result of all effects of the same type in scene +const blendedBloom = postProcessManager.getBlendEffect(BloomEffect); + +// Blended result contains averaged values from all active BloomEffects +console.log(blendedBloom.intensity.value); // Blended intensity value +``` + +## PostProcessPass Rendering Pipeline + +### Creating Custom Passes + +```ts +import { PostProcessPass, PostProcessPassEvent } from "@galacean/engine"; + +class CustomPass extends PostProcessPass { + constructor(engine: Engine) { + super(engine); + this.event = PostProcessPassEvent.AfterUber; // Set execution timing + } + + // Validity check + override isValid(postProcessManager: PostProcessManager): boolean { + return this.isActive && postProcessManager.getBlendEffect(CustomEffect)?.isValid(); + } + + // Render implementation + override onRender(camera: Camera, srcTexture: Texture2D, destTarget: RenderTarget): void { + const material = this.createMaterial(); + const effect = postProcessManager.getBlendEffect(CustomEffect); + + // Set shader parameters + material.shaderData.setFloat("intensity", effect.intensity.value); + material.shaderData.setTexture("inputTexture", srcTexture); + + // Render to target + this.renderToTarget(camera, material, destTarget); + } +} +``` + +### Pass Event Timing + +```ts +enum PostProcessPassEvent { + BeforeUber = 0, // Before Uber pass + AfterUber = 100 // After Uber pass +} + +// Custom event timing +pass.event = PostProcessPassEvent.BeforeUber + 10; // Execute 10 units after BeforeUber +``` + +## Performance Optimization + +### Effect Validity Optimization + +```ts +class OptimizedEffect extends PostProcessEffect { + override isValid(): boolean { + // Only activate when parameters reach visible threshold + return this.enabled && this.intensity.value > 0.01; + } +} +``` + +### Quality Level Adaptation + +```ts +enum QualityLevel { + Low, + Medium, + High +} + +function adjustPostProcessQuality(quality: QualityLevel, postProcess: PostProcess) { + const bloom = postProcess.getEffect(BloomEffect); + + switch (quality) { + case QualityLevel.Low: + bloom.enabled = false; + break; + case QualityLevel.Medium: + bloom.enabled = true; + bloom.highQualityFiltering.value = false; + bloom.downScale.value = BloomDownScaleMode.Quarter; + break; + case QualityLevel.High: + bloom.enabled = true; + bloom.highQualityFiltering.value = true; + bloom.downScale.value = BloomDownScaleMode.Half; + break; + } +} +``` + +### Local Post-Processing Optimization + +```ts +// Use reasonable blend distance to avoid frequent calculations +localPost.blendDistance = 3.0; // Don't set too large + +// Use simple collider shapes +const sphereShape = new SphereColliderShape(); +sphereShape.radius = 5.0; +collider.addShape(sphereShape); // Sphere calculations are faster than box +``` + +## Advanced Usage Patterns + +### Dynamic Effect Management + +```ts +class PostProcessController { + private postProcess: PostProcess; + private effectStates = new Map(); + + toggleEffect(effectType: T) { + const effect = this.postProcess.getEffect(effectType); + if (effect) { + effect.enabled = !effect.enabled; + this.effectStates.set(effectType, effect.enabled); + } + } + + saveState() { + // Save current effect states - using public API + const effects = this.postProcess.effects; + effects.forEach(effect => { + this.effectStates.set(effect.constructor as any, effect.enabled); + }); + } + + restoreState() { + // Restore effect states + this.effectStates.forEach((enabled, effectType) => { + const effect = this.postProcess.getEffect(effectType); + if (effect) { + effect.enabled = enabled; + } + }); + } +} +``` + +### Smooth Parameter Transitions + +```ts +class EffectAnimator { + private tweens = new Map(); + + animateParameter(effect: PostProcessEffect, paramName: string, targetValue: number, duration: number) { + const parameter = effect[paramName] as PostProcessEffectFloatParameter; + const startValue = parameter.value; + + // Use tween library or custom interpolation + const tween = this.createTween(startValue, targetValue, duration, (value) => { + parameter.value = value; + }); + + this.tweens.set(`${effect.constructor.name}.${paramName}`, tween); + } + + // Smooth effect enable + fadeInEffect(effect: PostProcessEffect, duration: number = 1.0) { + effect.enabled = true; + if ('intensity' in effect) { + this.animateParameter(effect, 'intensity', 1.0, duration); + } + } + + // Smooth effect disable + fadeOutEffect(effect: PostProcessEffect, duration: number = 1.0) { + if ('intensity' in effect) { + this.animateParameter(effect, 'intensity', 0.0, duration); + setTimeout(() => { + effect.enabled = false; + }, duration * 1000); + } + } +} +``` + +### Scene Transition Effects + +```ts +class SceneTransitionEffects { + static async fadeTransition(scene: Scene, duration: number = 2.0) { + const postProcess = scene.postProcessManager.getPostProcess(); + const fade = postProcess.addEffect(FadeEffect); + + // Fade in + fade.alpha.value = 1.0; + await this.animateValue(fade.alpha, 0.0, duration / 2); + + // Scene transition logic + // ... + + // Fade out + await this.animateValue(fade.alpha, 1.0, duration / 2); + postProcess.removeEffect(FadeEffect); + } +} +``` + +## Debugging and Troubleshooting + +### Effect Debugging + +```ts +function debugPostProcess(postProcess: PostProcess) { + console.log("PostProcess Debug Info:"); + console.log("- Component enabled:", postProcess.enabled); + console.log("- Effects count:", postProcess.effects.length); + + postProcess.effects.forEach((effect, index) => { + console.log(`Effect ${index}:`); + console.log(" - Type:", effect.constructor.name); + console.log(" - Enabled:", effect.enabled); + console.log(" - Valid:", effect.isValid()); + + // Check key parameters + if ('intensity' in effect) { + console.log(" - Intensity:", (effect as any).intensity.value); + } + }); + + // Check render passes + const engine = postProcess.engine; + // Note: This would be implemented through a public monitoring API + // const activePasses = engine.getPostProcessInfo(); + console.log("- Active passes:", activePasses.length); + + // Check camera masks + const cameras = postProcess.scene.findAllComponentsOfType(Camera); + cameras.forEach(camera => { + console.log(`Camera "${camera.entity.name}" mask:`, camera.postProcessMask); + }); +} +``` + +### Performance Monitoring + +```ts +class PostProcessProfiler { + private frameTime = 0; + private effectTimes = new Map(); + + startFrame() { + this.frameTime = performance.now(); + } + + endFrame() { + const totalTime = performance.now() - this.frameTime; + console.log(`PostProcess frame time: ${totalTime.toFixed(2)}ms`); + } + + profileEffect(effectName: string, fn: () => void) { + const start = performance.now(); + fn(); + const time = performance.now() - start; + + this.effectTimes.set(effectName, time); + if (time > 16.67) { // Exceeds frame time + console.warn(`Effect "${effectName}" is slow: ${time.toFixed(2)}ms`); + } + } +} +``` + +## API Reference + +```apidoc +PostProcess: + Properties: + layer: Layer + - Layer identification for the post-process component. + isGlobal: boolean + - Toggle between global and local post-processing modes. + blendDistance: number + - Blend distance for local post-processing transitions. + priority: number + - Execution priority affecting processing order. + + Methods: + addEffect(type: T): InstanceType + - Adds and returns new effect instance of specified type. + removeEffect(type: T): InstanceType + - Removes effect of specified type and returns the instance. + getEffect(type: T): InstanceType + - Gets existing effect instance of specified type. + clearEffects(): void + - Removes all effects from the post-process component. + +PostProcessEffect: + Properties: + enabled: boolean + - Controls whether the effect is active and processed. + + Methods: + isValid(): boolean + - Returns whether effect should be processed based on current state. + _lerp(to: PostProcessEffect, factor: number): void + - Internal method for blending between effect states. + +BloomEffect: + Properties: + threshold: PostProcessEffectFloatParameter + - Brightness threshold for bloom activation. @defaultValue `0.8` + intensity: PostProcessEffectFloatParameter + - Overall bloom effect intensity. @defaultValue `0` + scatter: PostProcessEffectFloatParameter + - Bloom scatter radius (0.0 to 1.0). @defaultValue `0.7` + tint: PostProcessEffectColorParameter + - Color tint applied to bloom. @defaultValue `new Color(1, 1, 1, 1)` + highQualityFiltering: PostProcessEffectBoolParameter + - Enable high quality filtering for better quality. @defaultValue `false` + downScale: PostProcessEffectEnumParameter + - Downscale mode for performance optimization. @defaultValue `BloomDownScaleMode.Half` + +TonemappingEffect: + Properties: + mode: PostProcessEffectEnumParameter + - Tone mapping algorithm selection. @defaultValue `TonemappingMode.Neutral` + +PostProcessManager: + Methods: + getBlendEffect(type: T): InstanceType + - Returns blended result of all effects of specified type in scene. + _update(camera: Camera): void + - Internal method for updating effect blending calculations. + _render(camera: Camera, src: RenderTarget, dest: RenderTarget): void + - Internal method for rendering post-processing effects. +``` + +## Best Practices + +### Effect Composition +- Apply tone mapping first, then bloom, followed by decorative effects +- Use reasonable parameter ranges to maintain visual quality +- Test effect combinations across different lighting conditions +- Consider performance impact when combining multiple effects + +### Parameter Configuration +- Set bloom threshold above 1.0 for realistic lighting +- Use moderate intensity values to avoid oversaturation +- Configure scatter values based on desired visual style +- Apply color tints sparingly to maintain color accuracy + +### Performance Optimization +- Implement effect validity checks to skip unnecessary processing +- Use quality level adaptation for different performance targets +- Monitor frame time impact of post-processing effects +- Use local post-processing sparingly due to collision detection overhead + +### Memory Management +- Clear effects when changing scenes to prevent memory leaks +- Use effect pooling for frequently toggled effects +- Monitor texture memory usage with dirt textures and custom effects +- Implement proper cleanup in custom effect destructors diff --git a/docs/scripting/Shadow.md b/docs/scripting/Shadow.md new file mode 100644 index 0000000000..ed54fd35fc --- /dev/null +++ b/docs/scripting/Shadow.md @@ -0,0 +1,694 @@ +# Shadow + +Galacean's Shadow system provides high-quality real-time shadow rendering featuring cascaded shadow maps, multiple shadow types, and adaptive quality settings. The system is optimized for directional lights and delivers stable shadow effects in large-scale scenes with comprehensive shadow management and performance optimization. + +## Overview + +The Shadow system encompasses advanced real-time shadow capabilities: + +- **Cascaded Shadow Maps**: Multi-resolution shadow mapping for optimal quality across viewing distances +- **Light Integration**: Seamless integration with directional, point, and spot lights +- **Shadow Types**: Support for hard shadows, soft shadows, and contact shadows +- **Quality Control**: Adaptive shadow resolution and bias settings for different platforms +- **Performance Optimization**: Automatic culling, distance fading, and cascade management +- **Shadow Receivers**: Comprehensive shadow casting and receiving system + +The system automatically manages shadow map generation, cascade splitting, and shadow filtering for optimal visual quality. + +## Quick Start + +### Basic Shadow Setup + +```ts +import { WebGLEngine, DirectLight, ShadowType, ShadowResolution } from "@galacean/engine"; + +const engine = await WebGLEngine.create({ canvas: "canvas" }); +const scene = engine.sceneManager.activeScene; + +// Create directional light with shadows +const lightEntity = scene.createRootEntity("DirectionalLight"); +const directLight = lightEntity.addComponent(DirectLight); + +// Configure basic shadow settings +directLight.shadowType = ShadowType.SoftLow; // Setting shadowType enables shadows +directLight.shadowStrength = 1.0; +directLight.shadowBias = 0.005; +directLight.shadowNormalBias = 0.05; + +// Configure global shadow settings +scene.castShadows = true; +scene.shadowResolution = ShadowResolution.Medium; +scene.shadowDistance = 50; +scene.shadowCascades = 4; + +// Position and orient the light +lightEntity.transform.setRotation(-45, -45, 0); + +engine.run(); +``` + +### Shadow Casting and Receiving + +```ts +import { MeshRenderer } from "@galacean/engine"; + +// Create objects that cast and receive shadows +const cubeEntity = scene.createRootEntity("Cube"); +const cubeRenderer = cubeEntity.addComponent(MeshRenderer); +cubeRenderer.mesh = PrimitiveMesh.createCuboid(engine, 2, 2, 2); +cubeRenderer.castShadows = true; +cubeRenderer.receiveShadows = true; + +// Create ground plane to receive shadows +const groundEntity = scene.createRootEntity("Ground"); +const groundRenderer = groundEntity.addComponent(MeshRenderer); +groundRenderer.mesh = PrimitiveMesh.createPlane(engine, 20, 20); +groundRenderer.castShadows = false; +groundRenderer.receiveShadows = true; + +// Position objects +cubeEntity.transform.setPosition(0, 1, 0); +groundEntity.transform.setPosition(0, -1, 0); +``` + +## Light Shadow Configuration + +### DirectLight Shadow Properties + +```ts +class DirectLight extends Light { + // Shadow type and quality (setting this enables shadows) + shadowType: ShadowType = ShadowType.Hard; + + // Shadow strength (0.0 to 1.0) + shadowStrength: number = 1.0; + + // Shadow bias to prevent shadow acne + shadowBias: number = 0.005; + + // Normal-based bias for curved surfaces + shadowNormalBias: number = 0.05; + + // Near plane for shadow cameras + shadowNearPlane: number = 0.1; +} + +enum ShadowType { + Hard = "Hard", // Hard shadows (no filtering) + SoftLow = "SoftLow", // Soft shadows (2x2 PCF) + SoftMedium = "SoftMedium", // Soft shadows (3x3 PCF) + SoftHigh = "SoftHigh" // Soft shadows (4x4 PCF) +} +``` + +### Shadow Configuration Examples + +```ts +// High quality shadow setup +directLight.shadowType = ShadowType.SoftHigh; // Enables shadows with high quality +directLight.shadowStrength = 0.8; +directLight.shadowBias = 0.001; // Reduce for high precision +directLight.shadowNormalBias = 0.02; // Fine-tune for surface quality + +// Performance-oriented setup +directLight.shadowType = ShadowType.Hard; // Enables shadows with hard edges +directLight.shadowStrength = 1.0; +directLight.shadowBias = 0.01; // Higher bias for stability +directLight.shadowNormalBias = 0.1; // Higher for performance + +// Mobile-optimized setup +directLight.shadowType = ShadowType.SoftLow; // Enables shadows with low quality +directLight.shadowStrength = 0.6; +directLight.shadowBias = 0.02; +directLight.shadowNormalBias = 0.15; +``` + +## ShadowManager Configuration + +### Global Shadow Settings + +```ts +class ShadowManager { + // Enable/disable shadow system + enabled: boolean = true; + + // Shadow map resolution + shadowMapSize: ShadowResolution = ShadowResolution.Medium; + + // Maximum shadow distance + shadowDistance: number = 50; + + // Number of shadow cascades (1-4) + cascadeCount: number = 4; + + // Cascade split distribution + cascadeSplitRatio: number[] = [0.1, 0.3, 0.6, 1.0]; + + // Shadow fade distance + shadowFadeDistance: number = 10; +} + +enum ShadowResolution { + Low = 512, + Medium = 1024, + High = 2048, + Ultra = 4096 +} +``` + +### Advanced Shadow Manager Setup + +```ts +const shadowManager = engine.shadowManager; + +// Configure for large outdoor scenes +shadowManager.enabled = true; +shadowManager.shadowMapSize = ShadowResolution.High; +shadowManager.shadowDistance = 100; +shadowManager.cascadeCount = 4; +shadowManager.cascadeSplitRatio = [0.05, 0.2, 0.5, 1.0]; // More detail near camera +shadowManager.shadowFadeDistance = 15; + +// Configure for indoor scenes +shadowManager.enabled = true; +shadowManager.shadowMapSize = ShadowResolution.Medium; +shadowManager.shadowDistance = 30; +shadowManager.cascadeCount = 2; // Fewer cascades needed +shadowManager.cascadeSplitRatio = [0.3, 1.0]; +shadowManager.shadowFadeDistance = 5; + +// Performance monitoring +shadowManager.onShadowMapUpdate = (cascadeIndex: number) => { + console.log(`Shadow cascade ${cascadeIndex} updated`); +}; +``` + +## Cascaded Shadow Maps + +### Understanding Cascade Distribution + +```ts +class CascadedShadowManager { + // Calculate cascade split distances + calculateCascadeSplits(nearPlane: number, farPlane: number, cascadeCount: number): number[] { + const splits: number[] = []; + const range = farPlane - nearPlane; + + for (let i = 0; i < cascadeCount; i++) { + const ratio = this.cascadeSplitRatio[i]; + splits[i] = nearPlane + range * ratio; + } + + return splits; + } + + // Get cascade bounds for debugging + getCascadeBounds(cascadeIndex: number): BoundingBox { + const cascade = this.shadowCascades[cascadeIndex]; + return cascade.bounds; + } + + // Optimize cascade distribution + optimizeCascades(camera: Camera): void { + const splits = this.calculateCascadeSplits( + camera.nearClipPlane, + Math.min(camera.farClipPlane, this.shadowDistance), + this.cascadeCount + ); + + this.updateCascadeMatrices(splits); + } +} +``` + +### Cascade Debugging and Visualization + +```ts +class ShadowDebugger { + private cascadeColors = [ + new Color(1, 0, 0, 0.5), // Red for cascade 0 + new Color(0, 1, 0, 0.5), // Green for cascade 1 + new Color(0, 0, 1, 0.5), // Blue for cascade 2 + new Color(1, 1, 0, 0.5) // Yellow for cascade 3 + ]; + + visualizeCascades(shadowManager: ShadowManager): void { + for (let i = 0; i < shadowManager.cascadeCount; i++) { + const cascade = shadowManager.shadowCascades[i]; + const color = this.cascadeColors[i]; + + // Render cascade frustum visualization + this.renderFrustum(cascade.viewMatrix, cascade.projectionMatrix, color); + } + } + + logCascadeInfo(shadowManager: ShadowManager): void { + for (let i = 0; i < shadowManager.cascadeCount; i++) { + const cascade = shadowManager.shadowCascades[i]; + console.log(`Cascade ${i}:`); + console.log(` Split distance: ${cascade.splitDistance}`); + console.log(` Bounds: ${cascade.bounds}`); + console.log(` Texel size: ${cascade.texelSize}`); + } + } +} +``` + +## Shadow Filtering and Quality + +### Shadow Filtering Techniques + +```ts +class ShadowFilter { + // Percentage Closer Filtering (PCF) + static setupPCF(material: Material, filterSize: number): void { + material.shaderData.setFloat("shadowPCFKernel", filterSize); + material.shaderData.setFloat("shadowMapTexelSize", 1.0 / 1024); // Adjust for shadow map size + } + + // Variance Shadow Maps (VSM) + static setupVSM(material: Material): void { + material.shaderData.enableMacro("SHADOW_VSM"); + material.shaderData.setFloat("shadowMinVariance", 0.00002); + material.shaderData.setFloat("shadowLightBleedingReduction", 0.8); + } + + // Contact shadows for fine detail + static setupContactShadows(material: Material, settings: ContactShadowSettings): void { + material.shaderData.enableMacro("CONTACT_SHADOWS"); + material.shaderData.setFloat("contactShadowLength", settings.length); + material.shaderData.setFloat("contactShadowDistanceScaleFactor", settings.distanceScale); + material.shaderData.setInt("contactShadowSampleCount", settings.sampleCount); + } +} + +interface ContactShadowSettings { + length: number; + distanceScale: number; + sampleCount: number; +} +``` + +### Adaptive Shadow Quality + +```ts +class AdaptiveShadowQuality { + private qualityLevels = { + low: { + shadowMapSize: ShadowResolution.Low, + cascadeCount: 2, + shadowType: ShadowType.Hard, + shadowDistance: 25 + }, + medium: { + shadowMapSize: ShadowResolution.Medium, + cascadeCount: 3, + shadowType: ShadowType.SoftLow, + shadowDistance: 50 + }, + high: { + shadowMapSize: ShadowResolution.High, + cascadeCount: 4, + shadowType: ShadowType.SoftMedium, + shadowDistance: 75 + }, + ultra: { + shadowMapSize: ShadowResolution.Ultra, + cascadeCount: 4, + shadowType: ShadowType.SoftHigh, + shadowDistance: 100 + } + }; + + adaptQuality(performanceMetrics: PerformanceMetrics, shadowManager: ShadowManager): void { + let targetQuality = 'medium'; + + if (performanceMetrics.averageFrameTime > 20) { + targetQuality = 'low'; + } else if (performanceMetrics.averageFrameTime < 10) { + targetQuality = 'high'; + } + + const settings = this.qualityLevels[targetQuality]; + this.applyQualitySettings(settings, shadowManager); + } + + private applyQualitySettings(settings: any, shadowManager: ShadowManager): void { + shadowManager.shadowMapSize = settings.shadowMapSize; + shadowManager.cascadeCount = settings.cascadeCount; + shadowManager.shadowDistance = settings.shadowDistance; + + // Apply to all directional lights + const lights = shadowManager.scene.findAllComponentsOfType(DirectLight); + lights.forEach(light => { + if (light.enableShadow) { + light.shadowType = settings.shadowType; + } + }); + } +} +``` + +## Performance Optimization + +### Shadow LOD System + +```ts +class ShadowLOD { + private lodLevels = [ + { distance: 10, shadowType: ShadowType.SoftHigh, bias: 0.001 }, + { distance: 25, shadowType: ShadowType.SoftMedium, bias: 0.003 }, + { distance: 50, shadowType: ShadowType.SoftLow, bias: 0.005 }, + { distance: 100, shadowType: ShadowType.Hard, bias: 0.01 } + ]; + + updateLOD(camera: Camera, lights: DirectLight[]): void { + const cameraPosition = camera.entity.transform.worldPosition; + + lights.forEach(light => { + if (!light.enableShadow) return; + + const lightDistance = Vector3.distance( + cameraPosition, + light.entity.transform.worldPosition + ); + + // Find appropriate LOD level + for (const lod of this.lodLevels) { + if (lightDistance <= lod.distance) { + light.shadowType = lod.shadowType; + light.shadowBias = lod.bias; + break; + } + } + }); + } +} +``` + +### Shadow Culling Optimization + +```ts +class ShadowCuller { + // Frustum culling for shadow casters + cullShadowCasters(light: DirectLight, renderers: MeshRenderer[]): MeshRenderer[] { + const shadowFrustum = this.calculateShadowFrustum(light); + const visibleCasters: MeshRenderer[] = []; + + for (const renderer of renderers) { + if (!renderer.castShadows) continue; + + const bounds = renderer.bounds; + if (shadowFrustum.intersectsBox(bounds)) { + visibleCasters.push(renderer); + } + } + + return visibleCasters; + } + + // Distance-based culling + cullByDistance(camera: Camera, renderers: MeshRenderer[], maxDistance: number): MeshRenderer[] { + const cameraPosition = camera.entity.transform.worldPosition; + + return renderers.filter(renderer => { + const distance = Vector3.distance( + cameraPosition, + renderer.entity.transform.worldPosition + ); + return distance <= maxDistance; + }); + } + + // Size-based culling (cull small objects) + cullBySize(renderers: MeshRenderer[], minSize: number): MeshRenderer[] { + return renderers.filter(renderer => { + const bounds = renderer.bounds; + const size = Math.max(bounds.size.x, bounds.size.y, bounds.size.z); + return size >= minSize; + }); + } +} +``` + +## Advanced Shadow Techniques + +### Soft Shadow Implementation + +```ts +class SoftShadowRenderer { + // Poisson disk sampling for soft shadows + private poissonDisk = [ + new Vector2(-0.613392, 0.617481), + new Vector2(0.170019, -0.040254), + new Vector2(-0.299417, 0.791925), + new Vector2(0.645680, 0.493210), + // ... more samples + ]; + + setupSoftShadows(material: Material, lightSize: number, sampleCount: number): void { + material.shaderData.enableMacro("SOFT_SHADOWS"); + material.shaderData.setFloat("lightSize", lightSize); + material.shaderData.setInt("shadowSampleCount", sampleCount); + material.shaderData.setVector2Array("poissonDisk", this.poissonDisk); + } + + // Percentage Closer Soft Shadows (PCSS) + setupPCSS(material: Material): void { + material.shaderData.enableMacro("PCSS_SHADOWS"); + material.shaderData.setFloat("lightWorldSize", 2.0); + material.shaderData.setFloat("nearPlane", 0.1); + material.shaderData.setInt("blockerSearchSamples", 16); + material.shaderData.setInt("pcfSamples", 16); + } +} +``` + +### Contact Shadows + +```ts +class ContactShadowRenderer { + setup(camera: Camera, settings: ContactShadowSettings): void { + const material = this.getScreenSpaceMaterial(); + + material.shaderData.enableMacro("CONTACT_SHADOWS"); + material.shaderData.setFloat("contactShadowLength", settings.length); + material.shaderData.setFloat("contactShadowThickness", settings.thickness); + material.shaderData.setInt("contactShadowSteps", settings.steps); + material.shaderData.setFloat("contactShadowFadeStart", settings.fadeStart); + material.shaderData.setFloat("contactShadowFadeEnd", settings.fadeEnd); + + // Set camera matrices for screen-space calculation + material.shaderData.setMatrix("viewMatrix", camera.viewMatrix); + material.shaderData.setMatrix("projectionMatrix", camera.projectionMatrix); + } + + render(camera: Camera, depthTexture: Texture2D, normalTexture: Texture2D): Texture2D { + const material = this.getScreenSpaceMaterial(); + material.shaderData.setTexture("depthTexture", depthTexture); + material.shaderData.setTexture("normalTexture", normalTexture); + + return this.renderFullScreenQuad(material); + } +} + +interface ContactShadowSettings { + length: number; + thickness: number; + steps: number; + fadeStart: number; + fadeEnd: number; +} +``` + +## Shadow Debugging and Profiling + +### Shadow Performance Monitor + +```ts +class ShadowProfiler { + private shadowStats = { + shadowMapUpdates: 0, + totalShadowCasters: 0, + culledCasters: 0, + shadowMapMemory: 0, + lastFrameTime: 0 + }; + + profileFrame(shadowManager: ShadowManager): void { + const startTime = performance.now(); + + // Reset frame stats + this.shadowStats.shadowMapUpdates = 0; + this.shadowStats.totalShadowCasters = 0; + this.shadowStats.culledCasters = 0; + + // Monitor shadow map updates + shadowManager.onCascadeUpdate = (cascadeIndex: number) => { + this.shadowStats.shadowMapUpdates++; + }; + + // Calculate shadow map memory usage + const resolution = shadowManager.shadowMapSize; + const cascadeCount = shadowManager.cascadeCount; + this.shadowStats.shadowMapMemory = resolution * resolution * 4 * cascadeCount; // Bytes + + this.shadowStats.lastFrameTime = performance.now() - startTime; + } + + generateReport(): string { + return `Shadow Performance Report: +- Shadow map updates: ${this.shadowStats.shadowMapUpdates} +- Total shadow casters: ${this.shadowStats.totalShadowCasters} +- Culled casters: ${this.shadowStats.culledCasters} +- Shadow map memory: ${(this.shadowStats.shadowMapMemory / 1024 / 1024).toFixed(2)} MB +- Frame time: ${this.shadowStats.lastFrameTime.toFixed(2)} ms`; + } +} +``` + +### Shadow Quality Validation + +```ts +class ShadowValidator { + validateShadowSetup(light: DirectLight): boolean { + const issues: string[] = []; + + if (!light.enableShadow) { + issues.push("Shadows not enabled on light"); + return false; + } + + if (light.shadowBias < 0.0001) { + issues.push("Shadow bias too low, may cause shadow acne"); + } + + if (light.shadowBias > 0.1) { + issues.push("Shadow bias too high, shadows may detach from objects"); + } + + if (light.shadowNormalBias < 0.01) { + issues.push("Normal bias too low for curved surfaces"); + } + + if (light.shadowStrength <= 0) { + issues.push("Shadow strength is zero or negative"); + } + + if (issues.length > 0) { + console.warn("Shadow setup issues:", issues); + return false; + } + + return true; + } + + validateShadowManager(shadowManager: ShadowManager): boolean { + if (!shadowManager.enabled) { + console.warn("Shadow manager is disabled"); + return false; + } + + if (shadowManager.shadowDistance <= 0) { + console.warn("Shadow distance is invalid"); + return false; + } + + if (shadowManager.cascadeCount < 1 || shadowManager.cascadeCount > 4) { + console.warn("Invalid cascade count"); + return false; + } + + return true; + } +} +``` + +## API Reference + +```apidoc +DirectLight: + Properties: + enableShadow: boolean + - Enable shadow casting for this light. @defaultValue `false` + shadowType: ShadowType + - Shadow filtering quality type. @defaultValue `ShadowType.Hard` + shadowStrength: number + - Shadow opacity (0.0 to 1.0). @defaultValue `1.0` + shadowBias: number + - Depth bias to prevent shadow acne. @defaultValue `0.005` + shadowNormalBias: number + - Normal-based bias for curved surfaces. @defaultValue `0.05` + shadowNearPlaneOffset: number + - Near plane offset for shadow camera. @defaultValue `0.1` + +ShadowManager: + Properties: + enabled: boolean + - Enable/disable entire shadow system. @defaultValue `true` + shadowMapSize: ShadowResolution + - Resolution of shadow maps. @defaultValue `ShadowResolution.Medium` + shadowDistance: number + - Maximum distance for shadow rendering. @defaultValue `50` + cascadeCount: number + - Number of shadow cascades (1-4). @defaultValue `4` + cascadeSplitRatio: number[] + - Distribution ratios for cascade splits. @defaultValue `[0.1, 0.3, 0.6, 1.0]` + shadowFadeDistance: number + - Distance over which shadows fade out. @defaultValue `10` + +ShadowType: + Values: + Hard: "Hard" + - Hard shadows with no filtering + SoftLow: "SoftLow" + - Soft shadows with 2x2 PCF filtering + SoftMedium: "SoftMedium" + - Soft shadows with 3x3 PCF filtering + SoftHigh: "SoftHigh" + - Soft shadows with 4x4 PCF filtering + +ShadowResolution: + Values: + Low: 512 + - 512x512 shadow map resolution + Medium: 1024 + - 1024x1024 shadow map resolution + High: 2048 + - 2048x2048 shadow map resolution + Ultra: 4096 + - 4096x4096 shadow map resolution + +MeshRenderer: + Properties: + castShadows: boolean + - Whether this renderer casts shadows. @defaultValue `true` + receiveShadows: boolean + - Whether this renderer receives shadows. @defaultValue `true` +``` + +## Best Practices + +### Shadow Quality Configuration +- Use ShadowType.SoftMedium for balanced quality and performance +- Set shadow bias between 0.001-0.01 depending on scene scale +- Configure normal bias between 0.02-0.1 for curved surfaces +- Use 2-3 cascades for most scenes, 4 for large outdoor environments + +### Performance Optimization +- Limit shadow distance to visible range for better cascade utilization +- Use shadow LOD systems to reduce quality at distance +- Implement shadow caster culling based on size and distance +- Monitor shadow map memory usage and adjust resolution accordingly + +### Visual Quality +- Position directional lights at 30-60 degree angles for optimal shadow coverage +- Use cascade visualization during development to verify distribution +- Test shadow settings across different times of day and lighting conditions +- Balance shadow strength to maintain visual contrast without overshadowing + +### Memory and Performance +- Choose shadow map resolution based on target platform capabilities +- Use contact shadows sparingly for hero objects only +- Implement adaptive quality systems for varying performance conditions +- Profile shadow rendering costs and optimize shadow caster counts diff --git a/docs/scripting/Sprite.md b/docs/scripting/Sprite.md new file mode 100644 index 0000000000..9fd4c911de --- /dev/null +++ b/docs/scripting/Sprite.md @@ -0,0 +1,459 @@ +# Sprite System - Galacean Engine LLM Documentation + +## System Overview + +The Sprite System is Galacean Engine's comprehensive 2D graphics rendering solution, providing advanced sprite management, texture atlas support, and multiple rendering modes for efficient 2D content creation. + +### Core Architecture + +```typescript +// Basic sprite creation and setup +const sprite = new Sprite(engine, texture2D); +const spriteRenderer = entity.addComponent(SpriteRenderer); +spriteRenderer.sprite = sprite; +spriteRenderer.drawMode = SpriteDrawMode.Simple; +``` + +## Core Classes + +### Sprite Class + +The `Sprite` class manages 2D sprite data including texture regions, atlas configurations, and positioning parameters. + +#### Key Properties + +```typescript +// Texture and atlas configuration +sprite.texture = texture2D; // Source texture +sprite.atlasRegion = new Rect(0, 0, 0.5, 0.5); // Normalized atlas region +sprite.atlasRotated = false; // 90-degree rotation flag +sprite.atlasRegionOffset = new Vector4(0, 0, 0, 0); // Atlas padding + +// Size and positioning +sprite.width = 100; // Custom width (optional) +sprite.height = 100; // Custom height (optional) +sprite.pivot = new Vector2(0.5, 0.5); // Center pivot point +sprite.region = new Rect(0, 0, 1, 1); // Texture sampling region + +// Nine-slice borders (for sliced mode) +sprite.border = new Vector4(10, 10, 10, 10); // Left, bottom, right, top +``` + +#### Automatic Size Calculation + +```typescript +// Sprites automatically calculate size from texture dimensions +const sprite = new Sprite(engine, texture2D); +// sprite.width automatically calculated from texture atlas settings +console.log(sprite.width); // Returns calculated width based on texture and atlas + +// Override with custom dimensions +sprite.width = 200; // Now returns custom width +sprite.height = 150; // Custom height +``` + +### SpriteRenderer Component + +The `SpriteRenderer` component handles sprite rendering with multiple draw modes and rendering optimizations. + +#### Draw Modes + +```typescript +// Simple mode - single quad rendering +spriteRenderer.drawMode = SpriteDrawMode.Simple; + +// Sliced mode - nine-slice rendering for UI elements +spriteRenderer.drawMode = SpriteDrawMode.Sliced; +sprite.border = new Vector4(20, 20, 20, 20); // Define slice borders + +// Tiled mode - repeating texture pattern +spriteRenderer.drawMode = SpriteDrawMode.Tiled; +spriteRenderer.tileMode = SpriteTileMode.Continuous; +spriteRenderer.tiledAdaptiveThreshold = 0.5; +``` + +#### Rendering Properties + +```typescript +// Color and transparency +spriteRenderer.color = new Color(1, 0.5, 0.5, 0.8); // Tinted red with alpha + +// Size override +spriteRenderer.width = 300; // Override sprite size for rendering +spriteRenderer.height = 200; + +// Flipping +spriteRenderer.flipX = true; // Horizontal flip +spriteRenderer.flipY = false; // No vertical flip + +// Masking +spriteRenderer.maskLayer = SpriteMaskLayer.UI; +spriteRenderer.maskInteraction = SpriteMaskInteraction.VisibleInsideMask; +``` + +### SpriteMask Component + +The `SpriteMask` component provides stencil-based masking for controlling sprite visibility. + +```typescript +// Create mask component +const spriteMask = entity.addComponent(SpriteMask); +spriteMask.sprite = maskSprite; +spriteMask.alphaCutoff = 0.5; // Alpha threshold +spriteMask.influenceLayers = SpriteMaskLayer.UI; // Affected layers + +// Mask positioning and sizing +spriteMask.width = 200; +spriteMask.height = 150; +spriteMask.flipX = false; +spriteMask.flipY = false; +``` + +## Advanced Features + +### Texture Atlas Integration + +```typescript +// Atlas sprite creation from SpriteAtlas +const spriteAtlas = await engine.resourceManager.load("atlas.json"); +const sprite = spriteAtlas.getSprite("character_idle"); + +// Manual atlas configuration +const sprite = new Sprite(engine, atlasTexture); +sprite.atlasRegion = new Rect(0.25, 0.25, 0.5, 0.5); // Use quarter of texture +sprite.atlasRegionOffset = new Vector4(0.1, 0.1, 0.1, 0.1); // 10% padding +sprite.atlasRotated = true; // Sprite was rotated 90° during packing +``` + +### Nine-Slice Sprites (UI Elements) + +```typescript +// Perfect for scalable UI elements +const buttonSprite = new Sprite(engine, buttonTexture); +buttonSprite.border = new Vector4(15, 15, 15, 15); // 15px borders on all sides + +const buttonRenderer = entity.addComponent(SpriteRenderer); +buttonRenderer.sprite = buttonSprite; +buttonRenderer.drawMode = SpriteDrawMode.Sliced; +buttonRenderer.width = 300; // Scales properly with borders intact +buttonRenderer.height = 80; +``` + +### Tiled Sprites (Patterns) + +```typescript +// Repeating background patterns +const patternSprite = new Sprite(engine, patternTexture); +const patternRenderer = entity.addComponent(SpriteRenderer); +patternRenderer.sprite = patternSprite; +patternRenderer.drawMode = SpriteDrawMode.Tiled; +patternRenderer.tileMode = SpriteTileMode.Continuous; +patternRenderer.width = 1000; // Large area filled with repeating pattern +patternRenderer.height = 600; + +// Adaptive tiling for crisp edges +patternRenderer.tileMode = SpriteTileMode.Adaptive; +patternRenderer.tiledAdaptiveThreshold = 0.5; // Stretch threshold +``` + +### Sprite Masking System + +```typescript +// Layer-based masking setup +enum CustomMaskLayer { + Background = 1 << 0, + Characters = 1 << 1, + UI = 1 << 2, + Effects = 1 << 3 +} + +// Create mask for UI elements +const uiMask = uiMaskEntity.addComponent(SpriteMask); +uiMask.sprite = circleMaskSprite; +uiMask.influenceLayers = CustomMaskLayer.UI; +uiMask.alphaCutoff = 0.1; // Very transparent threshold + +// Apply mask to sprite renderers +const uiElement = uiEntity.addComponent(SpriteRenderer); +uiElement.maskLayer = CustomMaskLayer.UI; +uiElement.maskInteraction = SpriteMaskInteraction.VisibleInsideMask; +``` + +## Performance Optimization + +### Sprite Batching + +```typescript +// Sprites with same material and texture automatically batch +// Use same texture atlas for multiple sprites to enable batching +const atlasTexture = await engine.resourceManager.load("characters.png"); + +const sprite1 = new Sprite(engine, atlasTexture); +sprite1.atlasRegion = new Rect(0, 0, 0.25, 0.5); // Character 1 + +const sprite2 = new Sprite(engine, atlasTexture); +sprite2.atlasRegion = new Rect(0.25, 0, 0.25, 0.5); // Character 2 + +// Both sprites will be batched together in rendering +``` + +### Memory Management + +```typescript +// Sprites are reference-counted resources +const sprite = new Sprite(engine, texture); +spriteRenderer.sprite = sprite; // Increments reference count + +// Cleanup +spriteRenderer.sprite = null; // Decrements reference count +// Sprite automatically destroyed when reference count reaches 0 +``` + +## Integration Examples + +### 2D Character System + +```typescript +class Character2D { + private spriteRenderer: SpriteRenderer; + private animations: Map = new Map(); + + constructor(entity: Entity, atlasTexture: Texture2D) { + this.spriteRenderer = entity.addComponent(SpriteRenderer); + this.spriteRenderer.drawMode = SpriteDrawMode.Simple; + + // Setup animation frames from atlas + this.setupAnimations(atlasTexture); + } + + private setupAnimations(atlasTexture: Texture2D): void { + // Idle animation frames + const idleFrames: Sprite[] = []; + for (let i = 0; i < 4; i++) { + const frame = new Sprite(this.spriteRenderer.entity.engine, atlasTexture); + frame.atlasRegion = new Rect(i * 0.25, 0, 0.25, 0.5); + idleFrames.push(frame); + } + this.animations.set("idle", idleFrames); + } + + playAnimation(name: string, frameIndex: number): void { + const frames = this.animations.get(name); + if (frames && frames[frameIndex]) { + this.spriteRenderer.sprite = frames[frameIndex]; + } + } +} +``` + +### UI System with Masking + +```typescript +class UIPanel { + private panelEntity: Entity; + private maskEntity: Entity; + private contentEntities: Entity[] = []; + + constructor(engine: Engine, panelTexture: Texture2D, maskTexture: Texture2D) { + // Create panel background + this.panelEntity = engine.sceneManager.activeScene.createRootEntity("Panel"); + const panelRenderer = this.panelEntity.addComponent(SpriteRenderer); + panelRenderer.sprite = new Sprite(engine, panelTexture); + panelRenderer.drawMode = SpriteDrawMode.Sliced; + + // Create mask for content clipping + this.maskEntity = this.panelEntity.createChild("Mask"); + const mask = this.maskEntity.addComponent(SpriteMask); + mask.sprite = new Sprite(engine, maskTexture); + mask.influenceLayers = SpriteMaskLayer.UI; + + this.setupContentArea(); + } + + private setupContentArea(): void { + // Content sprites will be clipped by the mask + const contentEntity = this.panelEntity.createChild("Content"); + const contentRenderer = contentEntity.addComponent(SpriteRenderer); + contentRenderer.maskLayer = SpriteMaskLayer.UI; + contentRenderer.maskInteraction = SpriteMaskInteraction.VisibleInsideMask; + + this.contentEntities.push(contentEntity); + } +} +``` + +### Tiled Background System + +```typescript +class ScrollingBackground { + private backgroundRenderers: SpriteRenderer[] = []; + private scrollSpeed: number = 50; + + constructor(engine: Engine, backgroundTexture: Texture2D, layers: number = 3) { + for (let i = 0; i < layers; i++) { + const entity = engine.sceneManager.activeScene.createRootEntity(`Background_${i}`); + entity.transform.setPosition(0, 0, -i); // Layer depth + + const renderer = entity.addComponent(SpriteRenderer); + renderer.sprite = new Sprite(engine, backgroundTexture); + renderer.drawMode = SpriteDrawMode.Tiled; + renderer.tileMode = SpriteTileMode.Continuous; + renderer.width = 2000; // Large repeating area + renderer.height = 1000; + + // Parallax effect - further layers move slower + const parallaxFactor = 1 - (i * 0.3); + this.backgroundRenderers.push(renderer); + } + } + + update(deltaTime: number): void { + this.backgroundRenderers.forEach((renderer, index) => { + const parallaxFactor = 1 - (index * 0.3); + const entity = renderer.entity; + const currentPos = entity.transform.position; + entity.transform.setPosition( + currentPos.x - this.scrollSpeed * parallaxFactor * deltaTime, + currentPos.y, + currentPos.z + ); + }); + } +} +``` + +## Best Practices + +### Texture Atlas Organization + +```typescript +// Use power-of-2 textures for optimal performance +// Organize related sprites in same atlas for batching +// Example atlas layout: +// - Characters: 512x512 atlas with 8x8 grid of 64x64 sprites +// - UI Elements: 256x256 atlas with variable-sized elements +// - Effects: 1024x512 atlas with animation sequences + +const characterAtlas = await engine.resourceManager.load("characters_512.png"); +const uiAtlas = await engine.resourceManager.load("ui_256.png"); +const effectsAtlas = await engine.resourceManager.load("effects_1024.png"); +``` + +### Memory Optimization + +```typescript +// Share sprites between multiple renderers when possible +const sharedSprite = new Sprite(engine, texture); + +const renderer1 = entity1.addComponent(SpriteRenderer); +const renderer2 = entity2.addComponent(SpriteRenderer); +renderer1.sprite = sharedSprite; // Reference count: 1 +renderer2.sprite = sharedSprite; // Reference count: 2 + +// Use object pooling for frequently created/destroyed sprites +class SpritePool { + private pool: Sprite[] = []; + + getSprite(texture: Texture2D): Sprite { + return this.pool.pop() || new Sprite(this.engine, texture); + } + + returnSprite(sprite: Sprite): void { + sprite.texture = null; // Clear reference + this.pool.push(sprite); + } +} +``` + +### Performance Guidelines + +```typescript +// Batch sprites by material and texture +// Minimize draw calls by using texture atlases +// Use appropriate draw modes: +// - Simple: For most 2D sprites +// - Sliced: For scalable UI elements only +// - Tiled: For repeating patterns only + +// Avoid frequent sprite swapping in renderers +// Instead, create multiple renderers and enable/disable as needed +class OptimizedSpriteManager { + private renderers: Map = new Map(); + + setupSprites(entity: Entity, sprites: Map): void { + sprites.forEach((sprite, name) => { + const renderer = entity.addComponent(SpriteRenderer); + renderer.sprite = sprite; + renderer.enabled = false; // Start disabled + this.renderers.set(name, renderer); + }); + } + + showSprite(name: string): void { + // Disable all others + this.renderers.forEach(renderer => renderer.enabled = false); + // Enable target + const target = this.renderers.get(name); + if (target) target.enabled = true; + } +} +``` + +## Common Patterns + +### Animated Sprites + +```typescript +// Frame-based animation using sprite swapping +class SpriteAnimator { + private frames: Sprite[]; + private currentFrame: number = 0; + private frameTime: number = 0; + private frameDuration: number = 0.1; // 10 FPS + + constructor(private renderer: SpriteRenderer, frames: Sprite[]) { + this.frames = frames; + this.renderer.sprite = frames[0]; + } + + update(deltaTime: number): void { + this.frameTime += deltaTime; + if (this.frameTime >= this.frameDuration) { + this.frameTime = 0; + this.currentFrame = (this.currentFrame + 1) % this.frames.length; + this.renderer.sprite = this.frames[this.currentFrame]; + } + } +} +``` + +### Dynamic UI Scaling + +```typescript +// Responsive UI using sliced sprites +class ResponsiveButton { + private renderer: SpriteRenderer; + private baseSize: Vector2; + + constructor(entity: Entity, buttonSprite: Sprite, baseWidth: number, baseHeight: number) { + this.renderer = entity.addComponent(SpriteRenderer); + this.renderer.sprite = buttonSprite; + this.renderer.drawMode = SpriteDrawMode.Sliced; + this.baseSize = new Vector2(baseWidth, baseHeight); + this.updateSize(1.0); // Default scale + } + + updateSize(scale: number): void { + this.renderer.width = this.baseSize.x * scale; + this.renderer.height = this.baseSize.y * scale; + } + + // Adapt to text content + adaptToText(textWidth: number, textHeight: number, padding: number = 20): void { + this.renderer.width = textWidth + padding; + this.renderer.height = textHeight + padding; + } +} +``` + +The Sprite System provides a comprehensive foundation for 2D graphics in Galacean Engine, supporting everything from simple 2D games to complex UI systems with advanced features like texture atlasing, nine-slice rendering, and sophisticated masking capabilities. \ No newline at end of file diff --git a/docs/scripting/Text.md b/docs/scripting/Text.md new file mode 100644 index 0000000000..550553e5c4 --- /dev/null +++ b/docs/scripting/Text.md @@ -0,0 +1,803 @@ +# Text System - Galacean Engine LLM Documentation + +## System Overview + +The Text System provides comprehensive 2D text rendering capabilities with advanced typography features, dynamic font management, and high-performance character caching. It supports multi-language text layout, complex alignment modes, and sophisticated text wrapping algorithms. + +### Core Architecture + +```typescript +// Basic text rendering setup +const textRenderer = entity.addComponent(TextRenderer); +textRenderer.text = "Hello World"; +textRenderer.font = Font.createFromOS(engine, "Arial"); +textRenderer.fontSize = 24; +textRenderer.color = new Color(1, 1, 1, 1); +``` + +## Core Classes + +### TextRenderer Component + +The `TextRenderer` component handles all aspects of text rendering including layout calculation, alignment, wrapping, and visual styling. + +#### Essential Properties + +```typescript +// Text content and basic styling +textRenderer.text = "Multi-line text content"; +textRenderer.font = Font.createFromOS(engine, "Arial"); +textRenderer.fontSize = 32; +textRenderer.fontStyle = FontStyle.Bold | FontStyle.Italic; +textRenderer.color = new Color(0.2, 0.4, 0.8, 1.0); + +// Size and layout control +textRenderer.width = 400; // Text container width +textRenderer.height = 200; // Text container height +textRenderer.lineSpacing = 5; // Additional spacing between lines + +// Text alignment +textRenderer.horizontalAlignment = TextHorizontalAlignment.Center; +textRenderer.verticalAlignment = TextVerticalAlignment.Middle; + +// Text wrapping and overflow +textRenderer.enableWrapping = true; +textRenderer.overflowMode = OverflowMode.Truncate; +``` + +#### Font Styling Options + +```typescript +// Font style combinations +textRenderer.fontStyle = FontStyle.None; // Normal text +textRenderer.fontStyle = FontStyle.Bold; // Bold text +textRenderer.fontStyle = FontStyle.Italic; // Italic text +textRenderer.fontStyle = FontStyle.Bold | FontStyle.Italic; // Bold and italic + +// Dynamic font size scaling +textRenderer.fontSize = 16; // Small text +textRenderer.fontSize = 24; // Regular text +textRenderer.fontSize = 48; // Large headers +textRenderer.fontSize = 64; // Display text +``` + +#### Text Alignment System + +```typescript +// Horizontal alignment options +textRenderer.horizontalAlignment = TextHorizontalAlignment.Left; +textRenderer.horizontalAlignment = TextHorizontalAlignment.Center; +textRenderer.horizontalAlignment = TextHorizontalAlignment.Right; + +// Vertical alignment options +textRenderer.verticalAlignment = TextVerticalAlignment.Top; +textRenderer.verticalAlignment = TextVerticalAlignment.Center; +textRenderer.verticalAlignment = TextVerticalAlignment.Bottom; + +// Perfect center alignment +textRenderer.horizontalAlignment = TextHorizontalAlignment.Center; +textRenderer.verticalAlignment = TextVerticalAlignment.Center; +``` + +#### Text Wrapping and Overflow + +```typescript +// Enable text wrapping within container bounds +textRenderer.enableWrapping = true; +textRenderer.width = 300; // Text will wrap at this width + +// Overflow handling modes +textRenderer.overflowMode = OverflowMode.Overflow; // Text extends beyond bounds +textRenderer.overflowMode = OverflowMode.Truncate; // Text clips at bounds + +// Multi-line text with proper wrapping +textRenderer.text = `This is a long paragraph that will automatically wrap to multiple lines when it exceeds the specified width of the text container.`; +``` + +### Font Class + +The `Font` class manages font resources with automatic caching and system font integration. + +#### System Font Creation + +```typescript +// Create fonts from system fonts +const arialFont = Font.createFromOS(engine, "Arial"); +const timesFont = Font.createFromOS(engine, "Times New Roman"); +const courierFont = Font.createFromOS(engine, "Courier New"); + +// Generic font families (automatically available) +const serifFont = Font.createFromOS(engine, "serif"); +const sansSerifFont = Font.createFromOS(engine, "sans-serif"); +const monospaceFont = Font.createFromOS(engine, "monospace"); + +// Font reuse through automatic caching +const font1 = Font.createFromOS(engine, "Arial"); // Creates new font +const font2 = Font.createFromOS(engine, "Arial"); // Returns cached font +console.log(font1 === font2); // true - same font instance +``` + +#### Font Resource Management + +```typescript +// Fonts are automatically reference-counted +textRenderer.font = Font.createFromOS(engine, "Arial"); // Increments reference +textRenderer.font = null; // Decrements reference, auto-cleanup when count reaches 0 + +// Multiple renderers can share the same font efficiently +const sharedFont = Font.createFromOS(engine, "Helvetica"); +textRenderer1.font = sharedFont; // Reference count: 1 +textRenderer2.font = sharedFont; // Reference count: 2 +textRenderer3.font = sharedFont; // Reference count: 3 +``` + +### TextUtils Utility Class + +The `TextUtils` class provides advanced text measurement and layout calculation functions. + +#### Text Measurement + +```typescript +// Font size measurement for layout calculations +const fontString = TextUtils.getNativeFontString("Arial", 24, FontStyle.Bold); +const fontInfo = TextUtils.measureFont(fontString); +console.log(fontInfo.ascent, fontInfo.descent, fontInfo.size); + +// Individual character measurement +const charInfo = TextUtils.measureChar("A", fontString); +console.log(charInfo.w, charInfo.h, charInfo.xAdvance); + +// Native font string generation +const nativeFont = TextUtils.getNativeFontString("Times New Roman", 18, FontStyle.Italic); +// Returns: "italic 18px Times New Roman" +``` + +#### Advanced Layout Calculation + +```typescript +// Text measurement with wrapping +const textMetrics = TextUtils.measureTextWithWrap( + textRenderer, // Text renderer instance + 400, // Container width in pixels + 200, // Container height in pixels + 5 // Line spacing in pixels +); + +console.log(textMetrics.width); // Actual text width +console.log(textMetrics.height); // Total text height +console.log(textMetrics.lines); // Array of text lines +console.log(textMetrics.lineWidths); // Width of each line +console.log(textMetrics.lineHeight); // Height of each line + +// Text measurement without wrapping +const singleLineMetrics = TextUtils.measureTextWithoutWrap( + textRenderer, + 200, // Container height + 5 // Line spacing +); +``` + +## Advanced Features + +### Multi-Language Text Support + +```typescript +// International text rendering +textRenderer.text = "Hello 世界"; + +// Chinese/Japanese text with proper word breaking +textRenderer.text = "这是一段中文文本，支持自动换行和字符断行处理。"; +textRenderer.enableWrapping = true; + +// Mixed content with different writing systems +textRenderer.text = ` +English text with proper wrapping. +中文文本支持正确的字符换行。 +العربية النص مع الدعم الصحيح. +`; +``` + +### Dynamic Text Animation + +```typescript +// Text content animation +class TypewriterEffect { + private fullText: string; + private currentText: string = ""; + private charIndex: number = 0; + + constructor(private textRenderer: TextRenderer, text: string) { + this.fullText = text; + } + + update(deltaTime: number): void { + const speed = 20; // Characters per second + this.charIndex += speed * deltaTime; + + const targetLength = Math.floor(this.charIndex); + this.currentText = this.fullText.substring(0, targetLength); + this.textRenderer.text = this.currentText; + } +} + +// Color animation +class TextColorAnimator { + private time: number = 0; + + constructor(private textRenderer: TextRenderer) {} + + update(deltaTime: number): void { + this.time += deltaTime; + const hue = (this.time * 0.5) % 1.0; + this.textRenderer.color = Color.fromHSV(hue, 0.8, 1.0); + } +} +``` + +### Text Layout Templates + +```typescript +// Responsive text layout system +class ResponsiveText { + private baseWidth: number; + private baseHeight: number; + private baseFontSize: number; + + constructor( + private textRenderer: TextRenderer, + baseWidth: number, + baseHeight: number, + baseFontSize: number + ) { + this.baseWidth = baseWidth; + this.baseHeight = baseHeight; + this.baseFontSize = baseFontSize; + } + + updateScale(scale: number): void { + this.textRenderer.width = this.baseWidth * scale; + this.textRenderer.height = this.baseHeight * scale; + this.textRenderer.fontSize = this.baseFontSize * scale; + } + + fitToContent(): void { + // Temporarily disable wrapping to measure natural size + const originalWrapping = this.textRenderer.enableWrapping; + this.textRenderer.enableWrapping = false; + + const metrics = TextUtils.measureTextWithoutWrap( + this.textRenderer, + 1000, // Large height + this.textRenderer.lineSpacing + ); + + // Restore wrapping and set optimal size + this.textRenderer.enableWrapping = originalWrapping; + this.textRenderer.width = metrics.width + 20; // Add padding + this.textRenderer.height = metrics.height + 10; + } +} +``` + +### Performance Optimization + +```typescript +// Character caching optimization +class OptimizedTextRenderer { + private static characterCache = new Map(); + + static preloadCharacters(font: Font, characters: string): void { + const fontString = TextUtils.getNativeFontString( + font.name, + 24, + FontStyle.None + ); + + for (const char of characters) { + if (!this.characterCache.has(char)) { + const charInfo = TextUtils.measureChar(char, fontString); + this.characterCache.set(char, charInfo); + } + } + } + + // Preload common characters for performance + static preloadCommonChars(font: Font): void { + const commonChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .,!?-"; + this.preloadCharacters(font, commonChars); + } +} + +// Text batching for multiple renderers +class TextBatchManager { + private textRenderers: TextRenderer[] = []; + + addRenderer(renderer: TextRenderer): void { + this.textRenderers.push(renderer); + } + + updateAllTexts(texts: string[]): void { + // Batch update all renderers to minimize dirty flag operations + this.textRenderers.forEach((renderer, index) => { + if (texts[index]) { + renderer.text = texts[index]; + } + }); + } +} +``` + +## Integration Examples + +### UI Text System + +```typescript +class UITextSystem { + private titleText: TextRenderer; + private bodyText: TextRenderer; + private buttonText: TextRenderer; + + constructor(engine: Engine, parentEntity: Entity) { + this.setupTextElements(engine, parentEntity); + } + + private setupTextElements(engine: Engine, parent: Entity): void { + // Title text + const titleEntity = parent.createChild("Title"); + this.titleText = titleEntity.addComponent(TextRenderer); + this.titleText.font = Font.createFromOS(engine, "Arial"); + this.titleText.fontSize = 32; + this.titleText.fontStyle = FontStyle.Bold; + this.titleText.color = new Color(0.1, 0.1, 0.1, 1); + this.titleText.horizontalAlignment = TextHorizontalAlignment.Center; + this.titleText.verticalAlignment = TextVerticalAlignment.Top; + + // Body text with wrapping + const bodyEntity = parent.createChild("Body"); + bodyEntity.transform.setPosition(0, -50, 0); + this.bodyText = bodyEntity.addComponent(TextRenderer); + this.bodyText.font = Font.createFromOS(engine, "serif"); + this.bodyText.fontSize = 16; + this.bodyText.color = new Color(0.2, 0.2, 0.2, 1); + this.bodyText.width = 400; + this.bodyText.height = 200; + this.bodyText.enableWrapping = true; + this.bodyText.horizontalAlignment = TextHorizontalAlignment.Left; + this.bodyText.verticalAlignment = TextVerticalAlignment.Top; + this.bodyText.lineSpacing = 3; + + // Button text + const buttonEntity = parent.createChild("Button"); + buttonEntity.transform.setPosition(0, -150, 0); + this.buttonText = buttonEntity.addComponent(TextRenderer); + this.buttonText.font = Font.createFromOS(engine, "sans-serif"); + this.buttonText.fontSize = 18; + this.buttonText.fontStyle = FontStyle.Bold; + this.buttonText.color = new Color(1, 1, 1, 1); + this.buttonText.horizontalAlignment = TextHorizontalAlignment.Center; + this.buttonText.verticalAlignment = TextVerticalAlignment.Center; + } + + setContent(title: string, body: string, buttonLabel: string): void { + this.titleText.text = title; + this.bodyText.text = body; + this.buttonText.text = buttonLabel; + } +} +``` + +### Localization System + +```typescript +interface LocalizedText { + [key: string]: { + [language: string]: string; + }; +} + +class LocalizationManager { + private currentLanguage: string = "en"; + private textDatabase: LocalizedText = { + "welcome": { + "en": "Welcome to our game!", + "zh": "欢迎来到我们的游戏！", + "ja": "ゲームへようこそ！", + "es": "¡Bienvenido a nuestro juego!" + }, + "instructions": { + "en": "Use WASD keys to move your character around the world.", + "zh": "使用WASD键移动您的角色在世界中移动。", + "ja": "WASDキーを使ってキャラクターを世界中で移動させてください。", + "es": "Usa las teclas WASD para mover tu personaje por el mundo." + } + }; + + private textRenderers: Map = new Map(); + + registerText(key: string, renderer: TextRenderer): void { + this.textRenderers.set(key, renderer); + this.updateText(key); + } + + setLanguage(language: string): void { + this.currentLanguage = language; + this.updateAllTexts(); + } + + private updateText(key: string): void { + const renderer = this.textRenderers.get(key); + const textData = this.textDatabase[key]; + + if (renderer && textData) { + const localizedText = textData[this.currentLanguage] || textData["en"]; + renderer.text = localizedText; + + // Adjust font for different languages + this.adjustFontForLanguage(renderer, this.currentLanguage); + } + } + + private adjustFontForLanguage(renderer: TextRenderer, language: string): void { + switch (language) { + case "zh": + case "ja": + // Use fonts that support CJK characters + renderer.font = Font.createFromOS(renderer.entity.engine, "Arial Unicode MS"); + break; + case "ar": + // Right-to-left languages + renderer.horizontalAlignment = TextHorizontalAlignment.Right; + break; + default: + renderer.font = Font.createFromOS(renderer.entity.engine, "Arial"); + renderer.horizontalAlignment = TextHorizontalAlignment.Left; + break; + } + } + + private updateAllTexts(): void { + for (const key of this.textRenderers.keys()) { + this.updateText(key); + } + } +} +``` + +### Text Effects System + +```typescript +class TextEffectsSystem { + private effects: Map = new Map(); + + addEffect(renderer: TextRenderer, effect: TextEffect): void { + if (!this.effects.has(renderer)) { + this.effects.set(renderer, []); + } + this.effects.get(renderer).push(effect); + } + + removeEffect(renderer: TextRenderer, effect: TextEffect): void { + const effects = this.effects.get(renderer); + if (effects) { + const index = effects.indexOf(effect); + if (index >= 0) { + effects.splice(index, 1); + } + } + } + + update(deltaTime: number): void { + for (const [renderer, effects] of this.effects) { + for (const effect of effects) { + effect.update(renderer, deltaTime); + } + } + } +} + +// Text effect implementations +class FadeInEffect implements TextEffect { + private elapsed: number = 0; + private duration: number; + private originalAlpha: number; + + constructor(duration: number = 1.0) { + this.duration = duration; + } + + update(renderer: TextRenderer, deltaTime: number): void { + if (this.originalAlpha === undefined) { + this.originalAlpha = renderer.color.a; + renderer.color.a = 0; + } + + this.elapsed += deltaTime; + const progress = Math.min(this.elapsed / this.duration, 1.0); + renderer.color.a = this.originalAlpha * progress; + } +} + +class WaveEffect implements TextEffect { + private time: number = 0; + private amplitude: number; + private frequency: number; + private originalY: number; + + constructor(amplitude: number = 10, frequency: number = 2) { + this.amplitude = amplitude; + this.frequency = frequency; + } + + update(renderer: TextRenderer, deltaTime: number): void { + if (this.originalY === undefined) { + this.originalY = renderer.entity.transform.position.y; + } + + this.time += deltaTime; + const offset = Math.sin(this.time * this.frequency) * this.amplitude; + renderer.entity.transform.setPosition( + renderer.entity.transform.position.x, + this.originalY + offset, + renderer.entity.transform.position.z + ); + } +} + +interface TextEffect { + update(renderer: TextRenderer, deltaTime: number): void; +} +``` + +## Best Practices + +### Font Management + +```typescript +// Create a centralized font manager +class FontManager { + private static fonts: Map = new Map(); + + static getFont(engine: Engine, name: string): Font { + if (!this.fonts.has(name)) { + const font = Font.createFromOS(engine, name); + this.fonts.set(name, font); + } + return this.fonts.get(name); + } + + // Preload commonly used fonts + static preloadFonts(engine: Engine): void { + const commonFonts = ["Arial", "serif", "sans-serif", "monospace"]; + for (const fontName of commonFonts) { + this.getFont(engine, fontName); + } + } +} + +// Use consistent font hierarchies +enum FontWeight { + Light = FontStyle.None, + Regular = FontStyle.None, + Bold = FontStyle.Bold, + BoldItalic = FontStyle.Bold | FontStyle.Italic +} + +enum FontSize { + Caption = 12, + Body = 16, + Subheading = 18, + Heading = 24, + Display = 32, + Banner = 48 +} +``` + +### Performance Guidelines + +```typescript +// Minimize text updates for better performance +class EfficientTextUpdater { + private lastText: string = ""; + + updateTextIfChanged(renderer: TextRenderer, newText: string): void { + if (this.lastText !== newText) { + renderer.text = newText; + this.lastText = newText; + } + } +} + +// Use object pooling for dynamic text +class TextRendererPool { + private pool: TextRenderer[] = []; + private activeRenderers: Set = new Set(); + + getRenderer(entity: Entity): TextRenderer { + let renderer = this.pool.pop(); + if (!renderer) { + renderer = entity.addComponent(TextRenderer); + } + this.activeRenderers.add(renderer); + return renderer; + } + + returnRenderer(renderer: TextRenderer): void { + if (this.activeRenderers.has(renderer)) { + renderer.text = ""; + renderer.enabled = false; + this.activeRenderers.delete(renderer); + this.pool.push(renderer); + } + } +} + +// Batch text operations +class TextBatcher { + private pendingUpdates: Array<{renderer: TextRenderer, text: string}> = []; + + queueTextUpdate(renderer: TextRenderer, text: string): void { + this.pendingUpdates.push({renderer, text}); + } + + flushUpdates(): void { + // Apply all text updates in a single frame + for (const update of this.pendingUpdates) { + update.renderer.text = update.text; + } + this.pendingUpdates.length = 0; + } +} +``` + +### Layout Best Practices + +```typescript +// Responsive text sizing +class ResponsiveTextLayout { + static setupResponsiveText( + renderer: TextRenderer, + baseSize: number, + minSize: number, + maxSize: number + ): void { + const screenWidth = renderer.entity.engine.canvas.width; + const scale = screenWidth / 1920; // Base resolution + const clampedScale = Math.max(0.5, Math.min(2.0, scale)); + + const fontSize = Math.max(minSize, Math.min(maxSize, baseSize * clampedScale)); + renderer.fontSize = fontSize; + } + + // Auto-fit text to container + static autoFitText(renderer: TextRenderer, maxWidth: number): void { + let fontSize = renderer.fontSize; + const minFontSize = 8; + + while (fontSize > minFontSize) { + renderer.fontSize = fontSize; + const metrics = TextUtils.measureTextWithoutWrap( + renderer, + 1000, // Large height + renderer.lineSpacing + ); + + if (metrics.width <= maxWidth) { + break; + } + + fontSize--; + } + } +} + +// Text container management +class TextContainer { + private padding: { top: number, right: number, bottom: number, left: number }; + + constructor( + private renderer: TextRenderer, + padding = { top: 10, right: 10, bottom: 10, left: 10 } + ) { + this.padding = padding; + } + + setContainerSize(width: number, height: number): void { + this.renderer.width = width - this.padding.left - this.padding.right; + this.renderer.height = height - this.padding.top - this.padding.bottom; + } + + updateAlignment(horizontal: TextHorizontalAlignment, vertical: TextVerticalAlignment): void { + this.renderer.horizontalAlignment = horizontal; + this.renderer.verticalAlignment = vertical; + } +} +``` + +## Common Patterns + +### Text Input Simulation + +```typescript +// Simulated text input field +class TextInputField { + private cursor: string = "|"; + private cursorVisible: boolean = true; + private cursorTime: number = 0; + private inputText: string = ""; + + constructor(private renderer: TextRenderer) { + this.updateDisplay(); + } + + addCharacter(char: string): void { + this.inputText += char; + this.updateDisplay(); + } + + removeCharacter(): void { + this.inputText = this.inputText.slice(0, -1); + this.updateDisplay(); + } + + update(deltaTime: number): void { + // Animate cursor blinking + this.cursorTime += deltaTime; + if (this.cursorTime >= 0.5) { + this.cursorVisible = !this.cursorVisible; + this.cursorTime = 0; + this.updateDisplay(); + } + } + + private updateDisplay(): void { + const displayText = this.inputText + (this.cursorVisible ? this.cursor : " "); + this.renderer.text = displayText; + } +} +``` + +### Scrolling Text Display + +```typescript +// Scrolling text for long content +class ScrollingTextDisplay { + private fullText: string; + private visibleLines: number; + private currentLine: number = 0; + + constructor( + private renderer: TextRenderer, + text: string, + visibleLines: number = 5 + ) { + this.fullText = text; + this.visibleLines = visibleLines; + this.updateDisplay(); + } + + scrollUp(): void { + if (this.currentLine > 0) { + this.currentLine--; + this.updateDisplay(); + } + } + + scrollDown(): void { + const lines = this.fullText.split('\n'); + if (this.currentLine < lines.length - this.visibleLines) { + this.currentLine++; + this.updateDisplay(); + } + } + + private updateDisplay(): void { + const lines = this.fullText.split('\n'); + const visibleText = lines + .slice(this.currentLine, this.currentLine + this.visibleLines) + .join('\n'); + this.renderer.text = visibleText; + } +} +``` + +The Text System provides a powerful foundation for all text rendering needs in Galacean Engine, from simple labels to complex multi-language interfaces with advanced typography and layout capabilities. diff --git a/docs/scripting/XRManager.md b/docs/scripting/XRManager.md new file mode 100644 index 0000000000..aef4111087 --- /dev/null +++ b/docs/scripting/XRManager.md @@ -0,0 +1,988 @@ +# XR Manager - LLM Documentation + +## System Overview + +The XR Manager provides comprehensive Extended Reality (AR/VR) capabilities for the Galacean 3D engine, enabling immersive experiences across different XR platforms. It features session management, feature-based architecture, platform abstraction, and seamless integration with the engine's component system for building cross-platform XR applications. + +## Core Architecture + +### XR Manager Structure +```typescript +// Base XR Manager (located in core package - stub implementation) +const xrManager = engine.xrManager; + +// Check XR availability +if (xrManager) { + console.log("XR capabilities available"); +} else { + console.log("XR not supported in this build"); +} + +// XR Manager Extended (full implementation in xr package) +// Automatically replaces base XR Manager through mixin pattern +``` + +### XR Session Manager (Core XR Lifecycle) +```typescript +// Access session manager +const sessionManager = xrManager.sessionManager; + +// Check session state +console.log(`Session State: ${sessionManager.state}`); +console.log(`Session Mode: ${sessionManager.mode}`); + +// Session state monitoring +sessionManager.addStateChangedListener((state: XRSessionState) => { + switch (state) { + case XRSessionState.None: + console.log("XR session not initialized"); + break; + case XRSessionState.Initializing: + console.log("XR session initializing..."); + break; + case XRSessionState.Initialized: + console.log("XR session ready"); + break; + case XRSessionState.Running: + console.log("XR session active"); + break; + case XRSessionState.Paused: + console.log("XR session paused"); + break; + } +}); +``` + +### XR Feature System (Modular XR Capabilities) +```typescript +// Check if feature is supported before adding +if (xrManager.isSupportedFeature(XRHandTracking)) { + // Add hand tracking feature + const handTracking = xrManager.addFeature(XRHandTracking, { + handedness: "both", + jointRadius: 0.01 + }); + + if (handTracking) { + console.log("Hand tracking feature added successfully"); + } +} else { + console.log("Hand tracking not supported on this platform"); +} + +// Get existing feature +const cameraManager = xrManager.getFeature(XRCameraManager); +if (cameraManager) { + // Configure camera settings + cameraManager.enabled = true; +} +``` + +## XR Session Management + +### Session Initialization and Lifecycle +```typescript +class XRController extends Script { + private xrManager: XRManager; + private originEntity: Entity; + + onAwake() { + this.xrManager = this.engine.xrManager; + + // Create XR origin entity (connection between virtual and real world) + this.originEntity = this.entity.createChild("XROrigin"); + this.xrManager.origin = this.originEntity; + } + + // Enter AR mode + async enterAR() { + try { + // Check if AR is supported + await this.xrManager.sessionManager.isSupportedMode(XRSessionMode.ImmersiveAR); + + // Enter XR with automatic session start + await this.xrManager.enterXR(XRSessionMode.ImmersiveAR, true); + console.log("AR session started successfully"); + + } catch (error) { + console.error("Failed to start AR session:", error); + } + } + + // Enter VR mode + async enterVR() { + try { + await this.xrManager.sessionManager.isSupportedMode(XRSessionMode.ImmersiveVR); + await this.xrManager.enterXR(XRSessionMode.ImmersiveVR, true); + console.log("VR session started successfully"); + + } catch (error) { + console.error("Failed to start VR session:", error); + } + } + + // Manual session control + async enterXRWithManualControl() { + try { + // Enter XR without auto-starting + await this.xrManager.enterXR(XRSessionMode.ImmersiveVR, false); + + // Manual session start when ready + this.xrManager.sessionManager.run(); + + } catch (error) { + console.error("XR initialization failed:", error); + } + } + + // Exit XR session + async exitXR() { + try { + await this.xrManager.exitXR(); + console.log("XR session ended"); + } catch (error) { + console.error("Failed to exit XR:", error); + } + } +} +``` + +### Session State Management +```typescript +class XRSessionController extends Script { + private sessionManager: XRSessionManager; + + onAwake() { + this.sessionManager = this.engine.xrManager.sessionManager; + + // Listen for session state changes + this.sessionManager.addStateChangedListener(this.onSessionStateChanged.bind(this)); + } + + onSessionStateChanged(state: XRSessionState) { + switch (state) { + case XRSessionState.Initializing: + this.showLoadingUI(); + break; + + case XRSessionState.Initialized: + this.hideLoadingUI(); + this.setupXRScene(); + break; + + case XRSessionState.Running: + this.enableXRInteractions(); + this.startXRRendering(); + break; + + case XRSessionState.Paused: + this.pauseXRInteractions(); + break; + + case XRSessionState.None: + this.cleanupXRScene(); + break; + } + } + + // Manual session control + pauseSession() { + if (this.sessionManager.state === XRSessionState.Running) { + this.sessionManager.stop(); + } + } + + resumeSession() { + if (this.sessionManager.state === XRSessionState.Paused) { + this.sessionManager.run(); + } + } + + // Session information + getSessionInfo() { + return { + mode: this.sessionManager.mode, + state: this.sessionManager.state, + frameRate: this.sessionManager.frameRate, + supportedFrameRates: this.sessionManager.supportedFrameRate + }; + } +} +``` + +## XR Feature Management + +### Feature Registration and Usage +```typescript +// Custom XR feature implementation +@registerXRFeature(XRFeatureType.CustomFeature) +class CustomXRFeature extends XRFeature { + private customData: any; + + constructor(xrManager: XRManagerExtended, config: any) { + super(xrManager); + this.customData = config; + } + + _onSessionInit(): void { + console.log("Custom XR feature initializing"); + // Initialize feature-specific resources + } + + _onSessionStart(): void { + console.log("Custom XR feature starting"); + // Start feature operation + } + + _onUpdate(): void { + // Per-frame feature updates + if (this.enabled) { + this.updateCustomLogic(); + } + } + + _onSessionStop(): void { + console.log("Custom XR feature stopping"); + // Pause feature operation + } + + _onSessionExit(): void { + console.log("Custom XR feature exiting"); + // Cleanup feature resources + } + + private updateCustomLogic() { + // Feature-specific update logic + } +} + +// Feature management in application +class XRFeatureManager extends Script { + private features: Map = new Map(); + + async initializeXRFeatures() { + const xrManager = this.engine.xrManager; + + // Add multiple features + const featureConfigs = [ + { type: XRHandTracking, name: "handTracking", config: { jointRadius: 0.01 } }, + { type: XRPlaneDetection, name: "planeDetection", config: { orientation: "horizontal" } }, + { type: CustomXRFeature, name: "customFeature", config: { customParam: "value" } } + ]; + + for (const { type, name, config } of featureConfigs) { + if (xrManager.isSupportedFeature(type)) { + const feature = xrManager.addFeature(type, config); + if (feature) { + this.features.set(name, feature); + console.log(`${name} feature added successfully`); + } + } else { + console.warn(`${name} feature not supported on this platform`); + } + } + } + + toggleFeature(featureName: string, enabled: boolean) { + const feature = this.features.get(featureName); + if (feature) { + feature.enabled = enabled; + console.log(`${featureName} feature ${enabled ? 'enabled' : 'disabled'}`); + } + } + + getFeatureStatus() { + const status = {}; + this.features.forEach((feature, name) => { + status[name] = { + enabled: feature.enabled, + supported: this.engine.xrManager.isSupportedFeature(feature.constructor) + }; + }); + return status; + } +} +``` + +## XR Input Management + +### XR Input Integration +```typescript +class XRInputController extends Script { + private inputManager: XRInputManager; + private handEntities: Map = new Map(); + + onAwake() { + this.inputManager = this.engine.xrManager.inputManager; + } + + onUpdate() { + if (this.engine.xrManager.sessionManager.state === XRSessionState.Running) { + this.updateXRInput(); + } + } + + private updateXRInput() { + // Process XR controllers + const controllers = this.inputManager.getControllers(); + controllers.forEach((controller, index) => { + if (controller.connected) { + this.updateController(controller, index); + } + }); + + // Process hand tracking + const hands = this.inputManager.getHands(); + hands.forEach((hand, handedness) => { + if (hand.tracked) { + this.updateHandTracking(hand, handedness); + } + }); + } + + private updateController(controller: XRController, index: number) { + // Get controller pose + const pose = controller.pose; + if (pose) { + // Update controller entity position/rotation + const controllerEntity = this.getControllerEntity(index); + controllerEntity.transform.position = pose.position; + controllerEntity.transform.rotationQuaternion = pose.rotation; + } + + // Handle controller input + if (controller.selectPressed) { + this.onControllerSelect(controller, index); + } + + if (controller.squeezePressed) { + this.onControllerSqueeze(controller, index); + } + } + + private updateHandTracking(hand: XRHand, handedness: string) { + // Update hand joint positions + const handEntity = this.getHandEntity(handedness); + hand.joints.forEach((joint, jointName) => { + const jointEntity = handEntity.findByName(jointName); + if (jointEntity && joint.pose) { + jointEntity.transform.position = joint.pose.position; + jointEntity.transform.rotationQuaternion = joint.pose.rotation; + } + }); + + // Gesture recognition + if (this.isGrabGesture(hand)) { + this.onGrabGesture(handedness); + } + + if (this.isPinchGesture(hand)) { + this.onPinchGesture(handedness); + } + } +} +``` + +## XR Camera Management + +### Camera Configuration for XR +```typescript +class XRCameraController extends Script { + private cameraManager: XRCameraManager; + private mainCamera: Camera; + + onAwake() { + this.cameraManager = this.engine.xrManager.cameraManager; + this.mainCamera = this.entity.getComponent(Camera); + } + + setupXRCameras() { + // XR Camera automatically handles stereo rendering + if (this.engine.xrManager.sessionManager.state === XRSessionState.Running) { + // Configure camera for XR + this.mainCamera.clearFlags = this.getXRClearFlags(); + + // Disable standard camera updates (XR handles this) + this.mainCamera.enabled = false; + + // XR camera manager takes over rendering + console.log("XR camera configuration applied"); + } + } + + private getXRClearFlags(): CameraClearFlags { + // Get appropriate clear flags for XR mode + return this.cameraManager.getIgnoreClearFlags(this.mainCamera.cameraType); + } + + restoreStandardCamera() { + // Restore normal camera operation when exiting XR + this.mainCamera.enabled = true; + this.mainCamera.clearFlags = CameraClearFlags.All; + } + + // Custom XR camera effects + applyXRPostProcessing() { + // Add XR-specific post-processing effects + const postProcess = this.mainCamera.entity.getComponent(PostProcessPass); + if (postProcess) { + // Configure for XR rendering + postProcess.enabled = true; + } + } +} +``` + +## Advanced XR Patterns + +### XR Object Interaction System +```typescript +class XRInteractionSystem extends Script { + private interactableObjects: Set = new Set(); + private raycastLayer = Layer.Layer1; + + addInteractableObject(entity: Entity) { + // Add interaction capabilities to object + entity.layer = this.raycastLayer; + + // Add visual feedback component + const interactionFeedback = entity.addComponent(XRInteractionFeedback); + interactionFeedback.highlightColor = new Color(0, 1, 0, 0.5); + + this.interactableObjects.add(entity); + } + + onUpdate() { + if (this.engine.xrManager.sessionManager.state === XRSessionState.Running) { + this.processXRInteractions(); + } + } + + private processXRInteractions() { + const controllers = this.engine.xrManager.inputManager.getControllers(); + + controllers.forEach((controller, index) => { + if (controller.connected && controller.pose) { + // Perform raycast from controller + const ray = this.createControllerRay(controller); + const hitResult = this.engine.physicsManager.raycast( + ray.origin, + ray.direction, + Number.MAX_VALUE, + this.raycastLayer + ); + + if (hitResult.entity) { + this.highlightObject(hitResult.entity); + + // Handle interaction input + if (controller.selectPressed) { + this.interactWithObject(hitResult.entity, controller); + } + } + } + }); + } + + private createControllerRay(controller: XRController): { origin: Vector3, direction: Vector3 } { + const transform = controller.pose; + return { + origin: transform.position, + direction: Vector3.transformByQuat(Vector3.forward, transform.rotation) + }; + } + + private interactWithObject(entity: Entity, controller: XRController) { + // Trigger interaction events + const interactable = entity.getComponent(XRInteractable); + if (interactable) { + interactable.onInteract(controller); + } + } +} + +// Custom XR interaction component +class XRInteractable extends Component { + @property() + interactionType: "grab" | "touch" | "activate" = "activate"; + + @property() + hapticFeedback = true; + + onInteract(controller: XRController) { + switch (this.interactionType) { + case "grab": + this.startGrab(controller); + break; + case "touch": + this.onTouch(controller); + break; + case "activate": + this.onActivate(controller); + break; + } + + if (this.hapticFeedback) { + this.triggerHaptics(controller); + } + } + + private startGrab(controller: XRController) { + // Implement grab interaction + const grabComponent = this.entity.getComponent(XRGrabbable); + if (grabComponent) { + grabComponent.attachToController(controller); + } + } + + private triggerHaptics(controller: XRController) { + // Trigger haptic feedback + controller.triggerHaptic(0.5, 100); // intensity, duration + } +} +``` + +### XR Scene Anchoring and Persistence +```typescript +class XRSceneAnchorManager extends Script { + private anchors: Map = new Map(); + private persistentObjects: Map = new Map(); + + async createAnchor(position: Vector3, rotation: Quaternion, id: string): Promise { + try { + const anchor = await this.engine.xrManager.sessionManager.createAnchor(position, rotation); + if (anchor) { + this.anchors.set(id, anchor); + console.log(`Anchor ${id} created successfully`); + return anchor; + } + } catch (error) { + console.error(`Failed to create anchor ${id}:`, error); + } + return null; + } + + async placePersistentObject(entity: Entity, anchorId: string) { + const anchor = this.anchors.get(anchorId); + if (anchor) { + // Attach entity to anchor + entity.transform.position = anchor.pose.position; + entity.transform.rotationQuaternion = anchor.pose.rotation; + + // Mark as persistent + this.persistentObjects.set(anchorId, entity); + + // Save to persistent storage + await this.savePersistentScene(); + } + } + + async loadPersistentScene() { + try { + // Load saved anchor data + const savedAnchors = await this.loadAnchorData(); + + for (const [id, anchorData] of Object.entries(savedAnchors)) { + // Recreate anchors + const anchor = await this.createAnchor( + anchorData.position, + anchorData.rotation, + id + ); + + if (anchor) { + // Recreate associated objects + await this.recreateObject(id, anchorData.objectData); + } + } + } catch (error) { + console.error("Failed to load persistent scene:", error); + } + } + + private async savePersistentScene() { + const sceneData = {}; + + this.persistentObjects.forEach((entity, anchorId) => { + const anchor = this.anchors.get(anchorId); + if (anchor) { + sceneData[anchorId] = { + position: anchor.pose.position, + rotation: anchor.pose.rotation, + objectData: this.serializeEntity(entity) + }; + } + }); + + // Save to browser storage or cloud + localStorage.setItem('xrPersistentScene', JSON.stringify(sceneData)); + } + + onUpdate() { + // Update anchor poses if they change + this.anchors.forEach((anchor, id) => { + const entity = this.persistentObjects.get(id); + if (entity && anchor.isTracked) { + entity.transform.position = anchor.pose.position; + entity.transform.rotationQuaternion = anchor.pose.rotation; + } + }); + } +} +``` + +### XR Performance Optimization +```typescript +class XRPerformanceManager extends Script { + private frameRateTarget = 90; // Target frame rate for VR + private performanceMetrics = { + frameTime: 0, + cpuTime: 0, + gpuTime: 0, + droppedFrames: 0 + }; + + onUpdate() { + if (this.engine.xrManager.sessionManager.state === XRSessionState.Running) { + this.monitorPerformance(); + this.optimizeRendering(); + } + } + + private monitorPerformance() { + const currentFrameRate = this.engine.xrManager.sessionManager.frameRate; + + // Detect performance issues + if (currentFrameRate < this.frameRateTarget * 0.85) { + console.warn(`Frame rate below target: ${currentFrameRate}/${this.frameRateTarget}`); + this.applyPerformanceOptimizations(); + } + + // Update metrics + this.performanceMetrics.frameTime = 1000 / currentFrameRate; + } + + private applyPerformanceOptimizations() { + // Reduce rendering quality + this.adjustRenderingQuality(); + + // Optimize object LOD + this.updateLevelOfDetail(); + + // Disable non-essential features + this.disableNonEssentialFeatures(); + } + + private adjustRenderingQuality() { + // Reduce shadow resolution + const shadowManager = this.engine.shadowManager; + if (shadowManager.shadowMapSize > 512) { + shadowManager.shadowMapSize = 512; + } + + // Reduce anti-aliasing + const camera = this.engine.sceneManager.activeScene.findEntityByName("XRCamera"); + if (camera) { + const cameraComponent = camera.getComponent(Camera); + cameraComponent.msaaSamples = Math.max(1, cameraComponent.msaaSamples / 2); + } + } + + private updateLevelOfDetail() { + // Implement distance-based LOD for XR + const xrOrigin = this.engine.xrManager.origin; + if (!xrOrigin) return; + + const entities = this.engine.sceneManager.activeScene.rootEntities; + this.processEntitiesForLOD(entities, xrOrigin.transform.worldPosition); + } + + private processEntitiesForLOD(entities: readonly Entity[], viewerPosition: Vector3) { + entities.forEach(entity => { + const distance = Vector3.distance(entity.transform.worldPosition, viewerPosition); + + // Adjust mesh detail based on distance + const meshRenderer = entity.getComponent(MeshRenderer); + if (meshRenderer) { + if (distance > 50) { + // Use low-poly mesh for distant objects + meshRenderer.enabled = false; + } else if (distance > 20) { + // Use medium-poly mesh + this.setMeshLOD(meshRenderer, 1); + } else { + // Use high-poly mesh for close objects + this.setMeshLOD(meshRenderer, 0); + } + } + + // Process children recursively + if (entity.children.length > 0) { + this.processEntitiesForLOD(entity.children, viewerPosition); + } + }); + } + + getPerformanceReport() { + return { + ...this.performanceMetrics, + targetFrameRate: this.frameRateTarget, + currentFrameRate: this.engine.xrManager.sessionManager.frameRate, + sessionMode: this.engine.xrManager.sessionManager.mode, + activeFeatures: this.engine.xrManager.features.filter(f => f.enabled).length + }; + } +} +``` + +## XR Event System and Lifecycle Integration + +### Complete XR Application Framework +```typescript +class XRApplication extends Script { + private xrManager: XRManager; + private xrUI: XRUIManager; + private xrAudio: XRAudioManager; + + onAwake() { + this.xrManager = this.engine.xrManager; + this.setupXRApplication(); + } + + private async setupXRApplication() { + // Initialize XR origin + const origin = this.entity.createChild("XROrigin"); + this.xrManager.origin = origin; + + // Setup XR features + await this.initializeXRFeatures(); + + // Setup XR UI + this.xrUI = new XRUIManager(this.engine); + + // Setup spatial audio for XR + this.xrAudio = new XRAudioManager(this.engine); + + // Register event listeners + this.registerXREventListeners(); + } + + private async initializeXRFeatures() { + const features = [ + { type: XRHandTracking, config: { jointRadius: 0.01 } }, + { type: XRPlaneDetection, config: { orientation: "both" } }, + { type: XRHitTest, config: { entityTypes: ["plane", "point"] } } + ]; + + for (const { type, config } of features) { + if (this.xrManager.isSupportedFeature(type)) { + this.xrManager.addFeature(type, config); + } + } + } + + private registerXREventListeners() { + const sessionManager = this.xrManager.sessionManager; + + sessionManager.addStateChangedListener((state) => { + switch (state) { + case XRSessionState.Initializing: + this.onXRInitializing(); + break; + case XRSessionState.Running: + this.onXRStarted(); + break; + case XRSessionState.Paused: + this.onXRPaused(); + break; + case XRSessionState.None: + this.onXREnded(); + break; + } + }); + } + + private onXRInitializing() { + console.log("XR initializing - show loading screen"); + this.xrUI.showLoadingScreen(); + } + + private onXRStarted() { + console.log("XR session started - enable XR interactions"); + this.xrUI.hideLoadingScreen(); + this.xrUI.showXRInterface(); + this.xrAudio.enableSpatialAudio(); + } + + private onXRPaused() { + console.log("XR session paused"); + this.xrUI.showPauseScreen(); + } + + private onXREnded() { + console.log("XR session ended - cleanup"); + this.xrUI.hideXRInterface(); + this.xrAudio.disableSpatialAudio(); + } + + // Public API for starting XR experiences + async startARExperience() { + try { + await this.xrManager.enterXR(XRSessionMode.ImmersiveAR); + } catch (error) { + this.xrUI.showErrorMessage("AR not supported or failed to start"); + } + } + + async startVRExperience() { + try { + await this.xrManager.enterXR(XRSessionMode.ImmersiveVR); + } catch (error) { + this.xrUI.showErrorMessage("VR not supported or failed to start"); + } + } + + async exitXRExperience() { + await this.xrManager.exitXR(); + } +} +``` + +## Integration with Engine Systems + +### XR Component Integration +```typescript +// XR-aware Transform component extension +class XRTransform extends Script { + private originalTransform: Transform; + private xrOffset = new Vector3(); + + onAwake() { + this.originalTransform = this.entity.transform; + } + + onUpdate() { + if (this.engine.xrManager.sessionManager.state === XRSessionState.Running) { + // Apply XR space transformation + const xrOrigin = this.engine.xrManager.origin; + if (xrOrigin) { + const worldPosition = Vector3.add(this.originalTransform.position, this.xrOffset); + this.entity.transform.position = Vector3.transformByMatrix( + worldPosition, + xrOrigin.transform.worldMatrix + ); + } + } + } + + setXROffset(offset: Vector3) { + this.xrOffset = offset; + } +} + +// XR Physics integration +class XRPhysicsManager extends Script { + onUpdate() { + if (this.engine.xrManager.sessionManager.state === XRSessionState.Running) { + this.updateXRPhysics(); + } + } + + private updateXRPhysics() { + // Sync XR hand/controller colliders with physics + const hands = this.engine.xrManager.inputManager.getHands(); + hands.forEach((hand, handedness) => { + if (hand.tracked) { + this.updateHandColliders(hand, handedness); + } + }); + + // Sync controller colliders + const controllers = this.engine.xrManager.inputManager.getControllers(); + controllers.forEach((controller, index) => { + if (controller.connected) { + this.updateControllerCollider(controller, index); + } + }); + } + + private updateHandColliders(hand: XRHand, handedness: string) { + // Update physics colliders for hand joints + hand.joints.forEach((joint, jointName) => { + const collider = this.getJointCollider(handedness, jointName); + if (collider && joint.pose) { + collider.entity.transform.position = joint.pose.position; + collider.entity.transform.rotationQuaternion = joint.pose.rotation; + } + }); + } +} +``` + +## Best Practices + +1. **Session Management**: Always check session state before performing XR operations +2. **Feature Detection**: Test feature support before adding features to avoid runtime errors +3. **Performance Monitoring**: Continuously monitor frame rate and adjust quality for smooth XR experience +4. **Origin Management**: Properly set and manage XR origin for correct spatial alignment +5. **Graceful Degradation**: Provide fallbacks for unsupported XR features +6. **Resource Cleanup**: Properly cleanup XR resources when sessions end +7. **Error Handling**: Handle XR initialization failures gracefully with user feedback +8. **Cross-Platform Compatibility**: Test XR functionality across different devices and browsers + +## Common Patterns and Solutions + +### XR State Management +```typescript +class XRStateManager { + private xrState = { + isXRSupported: false, + activeSession: null, + currentMode: XRSessionMode.None, + availableFeatures: [], + userPreferences: { + preferredMode: XRSessionMode.ImmersiveVR, + enableHandTracking: true, + hapticFeedback: true + } + }; + + async initialize(engine: Engine) { + const xrManager = engine.xrManager; + + // Check XR support + this.xrState.isXRSupported = !!xrManager; + + if (this.xrState.isXRSupported) { + // Detect available features + this.detectAvailableFeatures(xrManager); + + // Load user preferences + this.loadUserPreferences(); + } + } + + private detectAvailableFeatures(xrManager: XRManager) { + const featuresToTest = [ + XRHandTracking, + XRPlaneDetection, + XRHitTest, + XRAnchorSupport + ]; + + this.xrState.availableFeatures = featuresToTest.filter(feature => + xrManager.isSupportedFeature(feature) + ); + } + + getRecommendedConfiguration(): XRConfiguration { + return { + mode: this.selectOptimalMode(), + features: this.selectOptimalFeatures(), + performance: this.getPerformanceSettings() + }; + } +} +``` + +This comprehensive XR Manager system provides robust Extended Reality capabilities with session management, feature-based architecture, input handling, and seamless integration with the Galacean 3D engine for building immersive AR/VR experiences across multiple platforms. \ No newline at end of file diff --git a/docs/scripting/advanced-rendering.md b/docs/scripting/advanced-rendering.md new file mode 100644 index 0000000000..58cb0df266 --- /dev/null +++ b/docs/scripting/advanced-rendering.md @@ -0,0 +1,492 @@ +# Advanced Rendering Features + +Galacean Engine provides sophisticated rendering capabilities including HDR rendering, multi-sample anti-aliasing (MSAA), advanced texture formats, and high-performance render targets. These features enable high-quality visual effects and optimized rendering pipelines. + +## HDR Rendering + +High Dynamic Range (HDR) rendering allows for a wider range of colors and brightness values, enabling more realistic lighting and post-processing effects. + +### HDR Configuration + +```ts +import { Camera, TextureFormat, AntiAliasing } from "@galacean/engine"; + +const camera = cameraEntity.getComponent(Camera); + +// Enable HDR rendering +camera.enableHDR = true; + +// HDR requires compatible hardware +const rhi = engine._hardwareRenderer; +const supportsHDR = rhi.isWebGL2 || rhi.canIUse(GLCapabilityType.textureHalfFloat); + +if (!supportsHDR) { + console.warn("HDR not supported on this device"); + camera.enableHDR = false; +} +``` + +### HDR Texture Formats + +HDR rendering uses high-precision texture formats: + +```ts +// HDR texture formats (automatically selected by engine) +enum HDRFormats { + // WebGL2 + no alpha required: 11-bit RGB + 10-bit shared exponent + R11G11B10_UFloat = "R11G11B10_UFloat", + + // WebGL1 or alpha required: 16-bit per channel + R16G16B16A16 = "R16G16B16A16", + + // Full precision: 32-bit per channel + R32G32B32A32 = "R32G32B32A32" +} + +// Engine automatically selects optimal format +function getHDRFormat(camera: Camera): TextureFormat { + const { engine, isAlphaOutputRequired } = camera; + const rhi = engine._hardwareRenderer; + + if (rhi.isWebGL2 && !isAlphaOutputRequired) { + return TextureFormat.R11G11B10_UFloat; // Most efficient + } else { + return TextureFormat.R16G16B16A16; // Compatible fallback + } +} +``` + +### HDR Render Targets + +```ts +// Create HDR render target +const hdrRenderTarget = new RenderTarget( + engine, + 1920, 1080, + new Texture2D(engine, 1920, 1080, TextureFormat.R16G16B16A16, false), + TextureFormat.Depth24Stencil8, + 1 // No MSAA for HDR by default +); + +// Apply to camera +camera.renderTarget = hdrRenderTarget; +camera.enableHDR = true; +``` + +### HDR Post-Processing Integration + +```ts +// HDR works seamlessly with post-processing +camera.enablePostProcess = true; + +const postProcess = scene.postProcessManager; + +// Bloom effect benefits greatly from HDR +const bloom = postProcess.addEffect(BloomEffect); +bloom.threshold.value = 1.0; // HDR allows values > 1.0 +bloom.intensity.value = 2.5; // Higher intensity possible with HDR + +// Tone mapping converts HDR to display range +const tonemap = postProcess.addEffect(TonemappingEffect); +tonemap.mode.value = TonemappingMode.ACES; // Filmic tone mapping +``` + +## Multi-Sample Anti-Aliasing (MSAA) + +MSAA provides hardware-accelerated edge smoothing by sampling multiple points per pixel during rasterization. + +### MSAA Configuration + +```ts +import { MSAASamples } from "@galacean/engine"; + +// Configure MSAA samples +camera.msaaSamples = MSAASamples.FourX; // 4x MSAA (default) + +// Available MSAA levels +enum MSAASamples { + None = 1, // No anti-aliasing + TwoX = 2, // 2x MSAA + FourX = 4, // 4x MSAA (recommended) + EightX = 8 // 8x MSAA (high-end devices) +} + +// Hardware capability detection +const maxMSAA = engine._hardwareRenderer.capability.maxAntiAliasing; +console.log(`Max MSAA samples supported: ${maxMSAA}`); + +// Adaptive MSAA based on device capability +if (maxMSAA >= 8) { + camera.msaaSamples = MSAASamples.EightX; +} else if (maxMSAA >= 4) { + camera.msaaSamples = MSAASamples.FourX; +} else { + camera.msaaSamples = MSAASamples.TwoX; +} +``` + +### MSAA with Render Targets + +```ts +// Create MSAA render target +const msaaRenderTarget = new RenderTarget( + engine, + 1920, 1080, + new Texture2D(engine, 1920, 1080, TextureFormat.R8G8B8A8, false), + TextureFormat.Depth24Stencil8, + 4 // 4x MSAA +); + +camera.renderTarget = msaaRenderTarget; + +// MSAA is automatically resolved to final texture +// No additional code needed for resolve operation +``` + +### MSAA Performance Considerations + +```ts +// Performance-aware MSAA configuration +class MSAAManager { + private camera: Camera; + private targetFrameRate = 60; + private frameTimeHistory: number[] = []; + + constructor(camera: Camera) { + this.camera = camera; + } + + adaptiveMSAA(): void { + const avgFrameTime = this.getAverageFrameTime(); + const currentFPS = 1000 / avgFrameTime; + + if (currentFPS < this.targetFrameRate * 0.8) { + // Performance too low, reduce MSAA + this.reduceMSAA(); + } else if (currentFPS > this.targetFrameRate * 1.1) { + // Performance headroom, increase MSAA + this.increaseMSAA(); + } + } + + private reduceMSAA(): void { + const current = this.camera.msaaSamples; + if (current > MSAASamples.None) { + this.camera.msaaSamples = Math.max(MSAASamples.None, current / 2); + console.log(`Reduced MSAA to ${this.camera.msaaSamples}x`); + } + } + + private increaseMSAA(): void { + const current = this.camera.msaaSamples; + const maxSupported = engine._hardwareRenderer.capability.maxAntiAliasing; + if (current < maxSupported) { + this.camera.msaaSamples = Math.min(maxSupported, current * 2); + console.log(`Increased MSAA to ${this.camera.msaaSamples}x`); + } + } +} +``` + +## Fast Approximate Anti-Aliasing (FXAA) + +FXAA is a post-processing anti-aliasing technique that smooths all pixels, including shader-generated edges. + +### FXAA Configuration + +```ts +import { AntiAliasing } from "@galacean/engine"; + +// Enable FXAA (post-processing anti-aliasing) +camera.antiAliasing = AntiAliasing.FXAA; + +// FXAA vs MSAA comparison +// MSAA: Hardware-based, only smooths geometry edges, higher performance cost +// FXAA: Shader-based, smooths all edges including alpha-cutoff, lower cost + +// Combining FXAA with MSAA (not recommended due to redundancy) +camera.msaaSamples = MSAASamples.TwoX; // Light MSAA +camera.antiAliasing = AntiAliasing.FXAA; // + FXAA for complete coverage +``` + +### FXAA Implementation Details + +```ts +// FXAA requires specific render target format +// Engine automatically creates intermediate R8G8B8A8 target for FXAA +// Then applies FXAA shader and outputs to final target + +// FXAA shader parameters (internal, not user-configurable) +const FXAA_PARAMS = { + SUBPIXEL_BLEND_AMOUNT: 0.75, // Subpixel blending + RELATIVE_CONTRAST_THRESHOLD: 0.166, // Edge detection sensitivity + ABSOLUTE_CONTRAST_THRESHOLD: 0.0833 // Minimum contrast for processing +}; +``` + +## Texture2DArray + +Texture arrays allow efficient storage and sampling of multiple textures with the same dimensions. + +### Creating Texture Arrays + +```ts +import { Texture2DArray, TextureFormat } from "@galacean/engine"; + +// Create texture array (WebGL2 only) +const textureArray = new Texture2DArray( + engine, + 512, 512, // width, height + 16, // array length (number of textures) + TextureFormat.R8G8B8A8, + true, // generate mipmaps + true // sRGB color space +); + +// WebGL1 compatibility check +if (!engine._hardwareRenderer.isWebGL2) { + throw new Error("Texture2DArray requires WebGL2"); +} +``` + +### Loading Data into Texture Arrays + +```ts +// Load images into texture array +const images = [ + "texture0.jpg", "texture1.jpg", "texture2.jpg", // ... up to 16 images +]; + +images.forEach(async (imagePath, index) => { + const image = new Image(); + image.onload = () => { + textureArray.setImageSource( + index, // array element index + image, // image source + 0, // mip level + false, // flip Y + false, // premultiply alpha + 0, 0 // x, y offset + ); + }; + image.src = imagePath; +}); + +// Set pixel data directly +const pixelData = new Uint8Array(512 * 512 * 4); // RGBA data +// ... fill pixel data ... +textureArray.setPixelBuffer( + 0, // array element index + pixelData, // pixel buffer + 0, // mip level + 0, 0, // x, y offset + 512, 512, // width, height + 1 // length (number of array elements to update) +); +``` + +### Using Texture Arrays in Shaders + +```glsl +// Vertex shader +attribute vec3 a_position; +attribute vec2 a_texCoord; +attribute float a_textureIndex; // Which texture in array to use + +varying vec2 v_texCoord; +varying float v_textureIndex; + +void main() { + gl_Position = renderer_MVPMat * vec4(a_position, 1.0); + v_texCoord = a_texCoord; + v_textureIndex = a_textureIndex; +} + +// Fragment shader +precision mediump float; +uniform sampler2DArray u_textureArray; + +varying vec2 v_texCoord; +varying float v_textureIndex; + +void main() { + // Sample from texture array + vec4 color = texture(u_textureArray, vec3(v_texCoord, v_textureIndex)); + gl_FragColor = color; +} +``` + +## Advanced Render Target Usage + +### Multiple Render Targets (MRT) + +```ts +// Create multiple color textures +const colorTexture1 = new Texture2D(engine, 1024, 1024, TextureFormat.R8G8B8A8); +const colorTexture2 = new Texture2D(engine, 1024, 1024, TextureFormat.R16G16B16A16); +const colorTexture3 = new Texture2D(engine, 1024, 1024, TextureFormat.R11G11B10_UFloat); + +// Create MRT render target +const mrtRenderTarget = new RenderTarget( + engine, + 1024, 1024, + [colorTexture1, colorTexture2, colorTexture3], // Multiple color attachments + TextureFormat.Depth24Stencil8, + 1 // No MSAA for MRT +); + +// Use in deferred rendering pipeline +camera.renderTarget = mrtRenderTarget; +``` + +### Depth-Only Rendering + +```ts +// Create depth-only render target +const depthTexture = new Texture2D(engine, 1024, 1024, TextureFormat.Depth32); +const depthOnlyTarget = new RenderTarget( + engine, + 1024, 1024, + null, // No color attachment + depthTexture // Depth texture +); + +// Use for shadow mapping +const shadowCamera = shadowCasterEntity.getComponent(Camera); +shadowCamera.renderTarget = depthOnlyTarget; +shadowCamera.clearFlags = CameraClearFlags.Depth; +``` + +### Render Target Chains + +```ts +// Create render target chain for post-processing +class RenderTargetChain { + private targets: RenderTarget[] = []; + + constructor(engine: Engine, width: number, height: number, count: number) { + for (let i = 0; i < count; i++) { + const colorTexture = new Texture2D( + engine, width, height, + TextureFormat.R16G16B16A16, // HDR format + false // No mipmaps for intermediate targets + ); + + this.targets.push(new RenderTarget( + engine, width, height, colorTexture, null, 1 + )); + } + } + + getTarget(index: number): RenderTarget { + return this.targets[index % this.targets.length]; + } + + // Ping-pong between targets + pingPong(currentIndex: number): number { + return (currentIndex + 1) % this.targets.length; + } +} + +// Usage in post-processing pipeline +const rtChain = new RenderTargetChain(engine, 1920, 1080, 2); +let currentTarget = 0; + +// Pass 1: Bloom prefilter +camera.renderTarget = rtChain.getTarget(currentTarget); +// ... render bloom prefilter ... + +// Pass 2: Bloom blur +currentTarget = rtChain.pingPong(currentTarget); +camera.renderTarget = rtChain.getTarget(currentTarget); +// ... render bloom blur ... +``` + +## Performance Optimization + +### Adaptive Quality Settings + +```ts +class AdaptiveRenderingManager { + private camera: Camera; + private qualityLevel = 1.0; + + constructor(camera: Camera) { + this.camera = camera; + } + + updateQuality(frameTime: number): void { + const targetFrameTime = 16.67; // 60 FPS + const ratio = frameTime / targetFrameTime; + + if (ratio > 1.2) { + // Performance too low + this.qualityLevel = Math.max(0.5, this.qualityLevel - 0.1); + } else if (ratio < 0.8) { + // Performance headroom + this.qualityLevel = Math.min(1.0, this.qualityLevel + 0.05); + } + + this.applyQualitySettings(); + } + + private applyQualitySettings(): void { + if (this.qualityLevel < 0.7) { + // Low quality + this.camera.msaaSamples = MSAASamples.None; + this.camera.enableHDR = false; + this.camera.antiAliasing = AntiAliasing.None; + } else if (this.qualityLevel < 0.9) { + // Medium quality + this.camera.msaaSamples = MSAASamples.TwoX; + this.camera.enableHDR = false; + this.camera.antiAliasing = AntiAliasing.FXAA; + } else { + // High quality + this.camera.msaaSamples = MSAASamples.FourX; + this.camera.enableHDR = true; + this.camera.antiAliasing = AntiAliasing.FXAA; + } + } +} +``` + +### Memory Management + +```ts +// Efficient render target management +class RenderTargetPool { + private pool = new Map(); + + getRenderTarget( + engine: Engine, + width: number, + height: number, + format: TextureFormat, + msaa: number = 1 + ): RenderTarget { + const key = `${width}x${height}_${format}_${msaa}`; + const targets = this.pool.get(key) || []; + + if (targets.length > 0) { + return targets.pop()!; + } + + // Create new render target + const colorTexture = new Texture2D(engine, width, height, format, false); + return new RenderTarget(engine, width, height, colorTexture, null, msaa); + } + + returnRenderTarget(target: RenderTarget): void { + const { width, height, antiAliasing } = target; + const format = target.getColorTexture(0)?.format; + const key = `${width}x${height}_${format}_${antiAliasing}`; + + const targets = this.pool.get(key) || []; + targets.push(target); + this.pool.set(key, targets); + } +} +``` + +These advanced rendering features provide the foundation for high-quality visual effects while maintaining optimal performance across different hardware configurations. diff --git a/docs/scripting/background-system.md b/docs/scripting/background-system.md new file mode 100644 index 0000000000..c9ed66001c --- /dev/null +++ b/docs/scripting/background-system.md @@ -0,0 +1,268 @@ +# Background System + +The Background system in Galacean Engine provides comprehensive scene background rendering capabilities. It supports multiple background modes including solid colors, skyboxes, and textures, allowing developers to create immersive visual environments. + +## Overview + +The Background system is integrated into the Scene class and provides three main rendering modes: +- **SolidColor**: Simple solid color backgrounds +- **Sky**: Advanced skybox rendering with materials +- **Texture**: 2D texture backgrounds with flexible fill modes + +## Background Modes + +### SolidColor Mode + +The simplest background mode that renders a solid color across the entire viewport: + +```typescript +import { BackgroundMode, Color } from "@galacean/engine"; + +// Access scene background +const background = scene.background; + +// Set solid color mode +background.mode = BackgroundMode.SolidColor; + +// Configure color (RGBA values from 0-1) +background.solidColor.set(0.2, 0.4, 0.8, 1.0); // Blue background +background.solidColor = new Color(0.1, 0.1, 0.1, 1.0); // Dark gray + +// Common color presets +background.solidColor = Color.BLACK; +background.solidColor = Color.WHITE; +background.solidColor = new Color(0.05, 0.05, 0.05, 1.0); // Default engine color +``` + +### Sky Mode + +Advanced skybox rendering using Sky materials for realistic environmental backgrounds: + +```typescript +import { BackgroundMode, SkyBoxMaterial, TextureCube } from "@galacean/engine"; + +// Set sky mode +background.mode = BackgroundMode.Sky; + +// Method 1: Using SkyBoxMaterial with cube texture +const skyMaterial = new SkyBoxMaterial(engine); +const cubeTexture = await engine.resourceManager.load({ + urls: [ + "px.jpg", "nx.jpg", // Positive/Negative X + "py.jpg", "ny.jpg", // Positive/Negative Y + "pz.jpg", "nz.jpg" // Positive/Negative Z + ], + type: AssetType.TextureCube +}); +skyMaterial.texture = cubeTexture; +background.sky.material = skyMaterial; + +// Method 2: Using procedural sky +const proceduralSky = new SkyProceduralMaterial(engine); +proceduralSky.sunSize = 0.04; +proceduralSky.sunSizeConvergence = 5; +proceduralSky.atmosphereThickness = 1.0; +proceduralSky.skyTint = new Color(0.5, 0.5, 0.5, 1.0); +proceduralSky.groundColor = new Color(0.369, 0.349, 0.341, 1.0); +background.sky.material = proceduralSky; + +// Custom sky mesh (optional) +const skyMesh = await engine.resourceManager.load("custom-sky.mesh"); +background.sky.mesh = skyMesh; +``` + +### Texture Mode + +2D texture backgrounds with flexible scaling and positioning options: + +```typescript +import { BackgroundMode, BackgroundTextureFillMode, Texture2D } from "@galacean/engine"; + +// Set texture mode +background.mode = BackgroundMode.Texture; + +// Load and set background texture +const backgroundTexture = await engine.resourceManager.load({ + url: "background.jpg", + type: AssetType.Texture2D +}); +background.texture = backgroundTexture; + +// Configure fill mode +background.textureFillMode = BackgroundTextureFillMode.AspectFitHeight; // Default +``` + +## Texture Fill Modes + +The texture fill mode determines how the background texture is scaled and positioned: + +### AspectFitWidth +Maintains aspect ratio and scales texture width to match canvas width, centering vertically: + +```typescript +background.textureFillMode = BackgroundTextureFillMode.AspectFitWidth; +// Best for: Wide textures, landscape orientations +// Result: Texture width = Canvas width, height scaled proportionally +``` + +### AspectFitHeight +Maintains aspect ratio and scales texture height to match canvas height, centering horizontally: + +```typescript +background.textureFillMode = BackgroundTextureFillMode.AspectFitHeight; +// Best for: Tall textures, portrait orientations +// Result: Texture height = Canvas height, width scaled proportionally +``` + +### Fill +Stretches texture to fill entire canvas, potentially distorting aspect ratio: + +```typescript +background.textureFillMode = BackgroundTextureFillMode.Fill; +// Best for: When exact canvas coverage is required +// Result: Texture fills entire canvas, aspect ratio may change +``` + +## Advanced Configuration + +### Dynamic Background Switching + +```typescript +class BackgroundController extends Script { + private backgrounds = { + day: { mode: BackgroundMode.Sky, material: dayMaterial }, + night: { mode: BackgroundMode.Sky, material: nightMaterial }, + indoor: { mode: BackgroundMode.SolidColor, color: new Color(0.1, 0.1, 0.1, 1.0) } + }; + + switchToDay(): void { + const bg = this.entity.scene.background; + bg.mode = BackgroundMode.Sky; + bg.sky.material = this.backgrounds.day.material; + } + + switchToNight(): void { + const bg = this.entity.scene.background; + bg.mode = BackgroundMode.Sky; + bg.sky.material = this.backgrounds.night.material; + } + + switchToIndoor(): void { + const bg = this.entity.scene.background; + bg.mode = BackgroundMode.SolidColor; + bg.solidColor = this.backgrounds.indoor.color; + } +} +``` + +### Responsive Background Textures + +```typescript +class ResponsiveBackground extends Script { + private mobileTexture: Texture2D; + private desktopTexture: Texture2D; + + onAwake(): void { + this.updateBackgroundForDevice(); + + // Listen for canvas size changes + this.engine.canvas._sizeUpdateFlagManager.addListener(() => { + this.updateBackgroundForDevice(); + }); + } + + private updateBackgroundForDevice(): void { + const bg = this.entity.scene.background; + const canvas = this.engine.canvas; + const aspectRatio = canvas.width / canvas.height; + + bg.mode = BackgroundMode.Texture; + + if (aspectRatio > 1.5) { + // Wide screen - use desktop texture + bg.texture = this.desktopTexture; + bg.textureFillMode = BackgroundTextureFillMode.AspectFitHeight; + } else { + // Mobile/square screen - use mobile texture + bg.texture = this.mobileTexture; + bg.textureFillMode = BackgroundTextureFillMode.AspectFitWidth; + } + } +} +``` + +### Performance Optimization + +```typescript +class OptimizedBackground extends Script { + private lowQualityTexture: Texture2D; + private highQualityTexture: Texture2D; + + onAwake(): void { + this.setBackgroundQuality(); + } + + private setBackgroundQuality(): void { + const bg = this.entity.scene.background; + const canvas = this.engine.canvas; + const pixelCount = canvas.width * canvas.height; + + bg.mode = BackgroundMode.Texture; + + // Use lower quality texture for high-resolution displays + if (pixelCount > 1920 * 1080) { + bg.texture = this.lowQualityTexture; + } else { + bg.texture = this.highQualityTexture; + } + } +} +``` + +## Best Practices + +### Performance Considerations +- **Texture Size**: Use appropriately sized textures to avoid memory waste +- **Compression**: Enable texture compression for background textures when possible +- **LOD**: Consider using different quality textures based on device capabilities +- **Sky Materials**: Procedural sky materials are more memory-efficient than cube textures + +### Visual Quality +- **Aspect Ratios**: Choose fill modes that preserve important visual elements +- **Color Spaces**: Ensure background colors match your scene's lighting setup +- **Seamless Transitions**: Use smooth interpolation when switching backgrounds dynamically + +### Memory Management +```typescript +// Properly dispose of background resources +const oldTexture = background.texture; +background.texture = newTexture; +oldTexture?.destroy(); // Free memory + +// Clear background when switching modes +background.mode = BackgroundMode.SolidColor; +background.texture = null; // Release texture reference +``` + +## Integration with Other Systems + +### Lighting Integration +```typescript +// Coordinate background with ambient lighting +if (background.mode === BackgroundMode.Sky) { + scene.ambientLight.diffuseMode = DiffuseMode.SphericalHarmonics; + scene.ambientLight.diffuseSolidColor = background.sky.material.tint; +} +``` + +### Post-Processing Integration +```typescript +// Adjust background for post-processing effects +const postProcess = camera.getComponent(PostProcessVolume); +if (postProcess.bloom.enabled) { + // Use darker background to enhance bloom effect + background.solidColor.set(0.02, 0.02, 0.02, 1.0); +} +``` + +The Background system provides a flexible foundation for creating visually appealing scenes while maintaining optimal performance across different platforms and devices. diff --git a/docs/scripting/base-systems.md b/docs/scripting/base-systems.md new file mode 100644 index 0000000000..7c3db72721 --- /dev/null +++ b/docs/scripting/base-systems.md @@ -0,0 +1,131 @@ +# Base Systems + +Galacean exposes several foundational utilities that almost every runtime feature builds upon. Understanding these base systems—event dispatching, logging, time management, and engine-bound object lifecycles—helps you integrate custom gameplay logic with the engine’s core services. + +## EventDispatcher +`EventDispatcher` implements a lightweight publish/subscribe mechanism. The class is designed for inheritance (`Engine`, `Entity`, `Scene`, and many subsystems extend it) but can also be composed when needed. + +```ts +import { EventDispatcher, WebGLEngine } from "@galacean/engine"; + +class GameManager extends EventDispatcher { + private _score = 0; + + get score(): number { + return this._score; + } + + addScore(points: number): void { + this._score += points; + this.dispatch("scoreChanged", { score: this._score, delta: points }); + } +} + +const manager = new GameManager(); +const handler = ({ score, delta }) => console.log(`+${delta}, total: ${score}`); + +manager.on("scoreChanged", handler); +manager.addScore(10); +manager.off("scoreChanged", handler); +``` + +Key API surface: +- `on(event, fn)` / `once(event, fn)` register persistent or one-shot listeners. +- `off(event, fn?)` removes listeners; omit `fn` to clear every listener for that event. +- `removeAllEventListeners(event?)` drops listeners for one event or all events. +- `hasEvent`, `eventNames`, and `listenerCount` expose diagnostics. +- `dispatch(event, data?)` synchronously invokes listeners. One-shot listeners automatically unregister themselves after invocation. + +The engine itself extends `EventDispatcher` and currently emits: +- `"run"` once the main loop starts. +- `"shutdown"` during shutdown. +- `"devicelost"` and `"devicerestored"` around WebGL context loss. + +```ts +const engine = await WebGLEngine.create({ canvas: "canvas" }); +engine.on("devicerestored", () => rebuildUserTextures()); +``` + +## Logger +`Logger` is a simple wrapper around the browser console. Logging is disabled by default to avoid noisy output—call `Logger.enable()` when you need diagnostics. + +```ts +import { Logger } from "@galacean/engine"; + +Logger.enable(); +Logger.debug("Frame begin"); +Logger.info("Loaded scene", sceneName); +Logger.warn("Missing lightmap for", entity.name); +Logger.error("Failed to load", err); + +if (Logger.isEnabled) { + Logger.info("Verbose logging is active"); +} + +Logger.disable(); +``` + +`Logger.enable()` binds `debug/info/warn/error` to their respective console methods; `Logger.disable()` replaces them with no-ops. The `isEnabled` flag mirrors the current state. + +## Time +`engine.time` centralizes frame timing. Values are updated once per engine tick and feed both scripting logic and shaders. + +```ts +const { time } = engine; + +function update() { + const dt = time.deltaTime; // scaled delta (respects timeScale & maximumDeltaTime) + const actual = time.actualDeltaTime; // unscaled delta + totalDistance += speed * dt; +} +``` + +Important properties: +- `frameCount`: total frames since the engine started. +- `deltaTime` / `elapsedTime`: scaled timing controlled by `timeScale`. +- `actualDeltaTime` / `actualElapsedTime`: real clock time (ignores `timeScale` and `maximumDeltaTime`). +- `maximumDeltaTime`: clamps large frame steps before scaling (default `0.333333` seconds). +- `timeScale`: global multiplier for the simulation (set to `0` to pause gameplay while UI continues to receive actual time). + +## EngineObject +Anything that belongs to an engine instance derives from `EngineObject`. This base class provides: +- `instanceId`: a unique identifier within the process. +- `engine`: back-reference to the owning `Engine`. +- `destroyed`: indicates whether `destroy()` has been called. + +```ts +import { EngineObject, Engine } from "@galacean/engine"; + +class ManagedHandle extends EngineObject { + constructor(engine: Engine) { + super(engine); + } + + protected override _onDestroy(): void { + // Always release custom resources first. + releaseNativeHandle(); + // Call base last so ResourceManager bookkeeping stays intact. + super._onDestroy(); + } +} + +const handle = new ManagedHandle(engine); +handle.destroy(); +``` + +`EngineObject.destroy()` calls `_onDestroy()` once and marks the instance as destroyed. The default implementation unregisters the object from the `ResourceManager`, so overrides should end with `super._onDestroy()` unless you intentionally bypass that behavior. + +## Quick reference +| System | Key types | Highlights | +| --- | --- | --- | +| Events | `EventDispatcher` (and any subclass) | `on`, `once`, `off`, `removeAllEventListeners`, `dispatch`, diagnostics helpers. | +| Logging | `Logger` | `enable/disable`, `debug/info/warn/error`, `isEnabled`. | +| Timing | `Time` via `engine.time` | `frameCount`, `deltaTime`, `actualDeltaTime`, `timeScale`, `maximumDeltaTime`, shader uniforms (`scene_ElapsedTime`, `scene_DeltaTime`). | +| Lifetimes | `EngineObject` | `instanceId`, `engine`, `destroyed`, override `_onDestroy()` for cleanup. | + +## Best practices +- Favor `EventDispatcher` for decoupled communication between components or gameplay systems. Remember to remove listeners (`off` or `removeAllEventListeners`) when entities or scripts are destroyed. +- Enable `Logger` only during development or when diagnosing issues; disable it in production builds to avoid unnecessary console work. +- Use `engine.time.deltaTime` for simulation updates and `actualDeltaTime` for UI or analytics that must ignore pauses. +- When extending `EngineObject`, encapsulate external resources and release them in `_onDestroy()`; avoid reusing instances after `destroyed` becomes `true`. +- Listen to engine-wide events such as `"devicelost"`/`"devicerestored"` if you maintain custom GPU resources. diff --git a/docs/scripting/batching-system.md b/docs/scripting/batching-system.md new file mode 100644 index 0000000000..f69cec1848 --- /dev/null +++ b/docs/scripting/batching-system.md @@ -0,0 +1,453 @@ +# Batching System + +The Galacean Engine's batching system is a sophisticated rendering optimization mechanism that reduces draw calls by combining compatible render elements. This system is crucial for achieving high performance, especially when rendering many similar objects like sprites, UI elements, and particles. + +## Core Components + +### BatcherManager + +The `BatcherManager` is the central coordinator of the batching system, managing different types of primitive chunk managers for various rendering contexts. + +```ts +// BatcherManager is internal to the engine - not directly accessible +// It manages three specialized chunk managers: + +// 2D rendering (sprites, images) +const manager2D = engine._batcherManager.primitiveChunkManager2D; + +// Mask rendering (sprite masks) +const managerMask = engine._batcherManager.primitiveChunkManagerMask; + +// UI rendering (UI components) +const managerUI = engine._batcherManager.primitiveChunkManagerUI; +``` + +#### Key Features +- **Automatic Management**: Creates chunk managers on-demand +- **Type Specialization**: Different managers for 2D, UI, and mask rendering +- **Memory Optimization**: Mask manager uses smaller chunks (128 vertices vs 4096) +- **Buffer Coordination**: Synchronizes vertex buffer uploads across all managers + +### RenderQueue + +The `RenderQueue` organizes render elements by type and handles sorting and batching operations. + +```ts +// RenderQueue types (internal enum) +enum RenderQueueType { + Opaque = 1000, // Solid objects, front-to-back sorting + AlphaTest = 2000, // Alpha-tested objects + Transparent = 3000 // Transparent objects, back-to-front sorting +} + +// Sorting strategies +class RenderQueue { + // Opaque objects: priority first, then distance (front-to-back) + static compareForOpaque(a: RenderElement, b: RenderElement): number { + return a.priority - b.priority || a.distanceForSort - b.distanceForSort; + } + + // Transparent objects: priority first, then distance (back-to-front) + static compareForTransparent(a: RenderElement, b: RenderElement): number { + return a.priority - b.priority || b.distanceForSort - a.distanceForSort; + } +} +``` + +### RenderElement and SubRenderElement + +These classes represent individual render operations and their sub-components. + +```ts +// RenderElement - represents a complete renderable object +class RenderElement { + priority: number; // Render priority (lower = earlier) + distanceForSort: number; // Distance from camera for sorting + subRenderElements: SubRenderElement[]; // Sub-elements for multi-material objects + renderQueueFlags: RenderQueueFlags; // Which queues this element belongs to +} + +// SubRenderElement - represents a single draw call +class SubRenderElement { + component: Renderer; // The renderer component + primitive: Primitive; // Geometry data + material: Material; // Material and shader + subPrimitive: SubMesh; // Specific mesh section + batched: boolean; // Whether this element is batched + texture?: Texture2D; // Primary texture (for 2D elements) + subChunk?: SubPrimitiveChunk; // Memory chunk (for batched elements) +} +``` + +## Memory Management + +### PrimitiveChunk + +The `PrimitiveChunk` manages large vertex and index buffers that can be subdivided for batching. + +```ts +// PrimitiveChunk configuration +class PrimitiveChunk { + // Default vertex layout for 2D/UI elements + // POSITION (Vector3) + TEXCOORD_0 (Vector2) + COLOR_0 (Vector4) + // Total: 36 bytes per vertex (9 floats) + + maxVertexCount: number = 4096; // Default max vertices per chunk + vertexStride: number = 36; // Bytes per vertex + + // Memory buffers + vertices: Float32Array; // Vertex data + indices: Uint16Array; // Index data + + // Free space management + vertexFreeAreas: VertexArea[]; // Available memory regions +} +``` + +#### Vertex Layout +```ts +// Standard 2D/UI vertex format +const vertexElements = [ + new VertexElement("POSITION", 0, VertexElementFormat.Vector3, 0), // 12 bytes + new VertexElement("TEXCOORD_0", 12, VertexElementFormat.Vector2, 0), // 8 bytes + new VertexElement("COLOR_0", 20, VertexElementFormat.Vector4, 0) // 16 bytes +]; +// Total: 36 bytes per vertex +``` + +### SubPrimitiveChunk + +Represents a allocated portion of a `PrimitiveChunk` for a specific render element. + +```ts +class SubPrimitiveChunk { + chunk: PrimitiveChunk; // Parent chunk + vertexArea: VertexArea; // Allocated vertex region + subMesh: SubMesh; // Drawing information + indices: number[]; // Local indices for this sub-chunk +} +``` + +### Memory Allocation Strategy + +```ts +// Allocation process +class PrimitiveChunkManager { + allocateSubChunk(vertexCount: number): SubPrimitiveChunk { + // 1. Try existing chunks first + for (const chunk of this.primitiveChunks) { + const subChunk = chunk.allocateSubChunk(vertexCount); + if (subChunk) return subChunk; + } + + // 2. Create new chunk if needed + const newChunk = new PrimitiveChunk(this.engine, this.maxVertexCount); + this.primitiveChunks.push(newChunk); + return newChunk.allocateSubChunk(vertexCount); + } +} +``` + +## Batching Process + +### 1. Element Collection + +Renderers create render elements during the culling phase: + +```ts +// Example: SpriteRenderer creating render elements +class SpriteRenderer extends Renderer { + _render(context: RenderContext): void { + const engine = context.camera.engine; + + // Get render element from pool + const renderElement = engine._renderElementPool.get(); + renderElement.set(this.priority, this._distanceForSort); + + // Create sub-render element + const subRenderElement = engine._subRenderElementPool.get(); + const subChunk = this._subChunk; // Pre-allocated chunk + + subRenderElement.set( + this, // renderer + material, // material + subChunk.chunk.primitive, // primitive + subChunk.subMesh, // sub-mesh + this.sprite.texture, // texture + subChunk // chunk reference + ); + + renderElement.addSubRenderElement(subRenderElement); + context.camera._renderPipeline.pushRenderElement(context, renderElement); + } +} +``` + +### 2. Sorting and Batching + +The render queue sorts elements and performs batching: + +```ts +// RenderQueue batching process +class RenderQueue { + sortBatch(compareFunc: Function, batcherManager: BatcherManager): void { + // 1. Sort elements by priority and distance + Utils._quickSort(this.elements, 0, this.elements.length, compareFunc); + + // 2. Perform batching + this.batch(batcherManager); + } + + batch(batcherManager: BatcherManager): void { + batcherManager.batch(this); + } +} +``` + +### 3. Compatibility Checking + +Elements can only be batched if they are compatible: + +```ts +// BatchUtils compatibility checking +class BatchUtils { + static canBatchSprite(elementA: SubRenderElement, elementB: SubRenderElement): boolean { + // Check if batching is disabled + if (elementB.shaderPasses[0].getTagValue("DisableBatch") === true) { + return false; + } + + // Must use same chunk + if (elementA.subChunk.chunk !== elementB.subChunk.chunk) { + return false; + } + + const rendererA = elementA.component as SpriteRenderer; + const rendererB = elementB.component as SpriteRenderer; + + // Check compatibility + return ( + rendererA.maskInteraction === rendererB.maskInteraction && + rendererA.maskLayer === rendererB.maskLayer && + elementA.texture === elementB.texture && + elementA.material === elementB.material + ); + } +} +``` + +### 4. Batch Execution + +The `BatcherManager` processes compatible elements: + +```ts +// BatcherManager.batch() algorithm +batch(renderQueue: RenderQueue): void { + const { elements, batchedSubElements } = renderQueue; + let previousElement: SubRenderElement; + let previousRenderer: Renderer; + let previousConstructor: Function; + + for (const element of elements) { + for (const subElement of element.subRenderElements) { + const renderer = subElement.component; + const constructor = renderer.constructor; + + if (previousElement) { + // Check if can batch with previous element + if (previousConstructor === constructor && + previousRenderer._canBatch(previousElement, subElement)) { + + // Batch elements together + previousRenderer._batch(previousElement, subElement); + previousElement.batched = true; + } else { + // Cannot batch - add previous element to render list + batchedSubElements.push(previousElement); + + // Start new batch + previousElement = subElement; + previousRenderer = renderer; + previousConstructor = constructor; + renderer._batch(subElement); + subElement.batched = false; + } + } else { + // First element + previousElement = subElement; + previousRenderer = renderer; + previousConstructor = constructor; + renderer._batch(subElement); + subElement.batched = false; + } + } + } + + // Add final element + if (previousElement) { + batchedSubElements.push(previousElement); + } +} +``` + +## Optimization Strategies + +### 1. Material Sharing + +```ts +// Good: Shared material enables batching +const sharedMaterial = new UnlitMaterial(engine); +sharedMaterial.baseTexture = atlas; // Use texture atlas + +// Apply to multiple sprites +sprite1.setMaterial(sharedMaterial); +sprite2.setMaterial(sharedMaterial); +sprite3.setMaterial(sharedMaterial); +// These can be batched together +``` + +### 2. Texture Atlasing + +```ts +// Good: Single atlas texture +const atlas = await engine.resourceManager.load("atlas.png"); + +// Bad: Multiple individual textures +const tex1 = await engine.resourceManager.load("sprite1.png"); +const tex2 = await engine.resourceManager.load("sprite2.png"); +// Cannot batch due to different textures +``` + +### 3. Render Priority Management + +```ts +// Group objects by priority for better batching +class GameObjectManager { + setupBatchingPriorities(): void { + // Background elements + this.backgroundSprites.forEach(sprite => sprite.priority = 0); + + // Game objects + this.gameObjects.forEach(obj => obj.priority = 100); + + // UI elements + this.uiElements.forEach(ui => ui.priority = 200); + + // Effects + this.effects.forEach(effect => effect.priority = 300); + } +} +``` + +### 4. Chunk Size Optimization + +```ts +// Different chunk sizes for different use cases +const batcherManager = engine._batcherManager; + +// 2D sprites: Large chunks (4096 vertices) +const manager2D = batcherManager.primitiveChunkManager2D; + +// UI elements: Large chunks (4096 vertices) +const managerUI = batcherManager.primitiveChunkManagerUI; + +// Masks: Small chunks (128 vertices) - masks are typically few +const managerMask = batcherManager.primitiveChunkManagerMask; +``` + +## Performance Considerations + +### Buffer Upload Optimization + +```ts +// PrimitiveChunk uses optimized buffer updates +uploadBuffer(): void { + const { primitive, updateVertexStart, updateVertexEnd } = this; + + // Only upload changed regions + if (updateVertexStart !== Number.MAX_SAFE_INTEGER) { + primitive.vertexBufferBindings[0].buffer.setData( + this.vertices, + updateVertexStart * 4, // byte offset + updateVertexStart, // element offset + updateVertexEnd - updateVertexStart, // element count + SetDataOptions.Discard // Discard for performance + ); + } + + // Upload index data + primitive.indexBufferBinding.buffer.setData( + this.indices, 0, 0, this.updateIndexLength, SetDataOptions.Discard + ); +} +``` + +### Memory Pool Usage + +```ts +// Object pools reduce garbage collection +class PrimitiveChunk { + static areaPool = new ReturnableObjectPool(VertexArea, 10); + static subChunkPool = new ReturnableObjectPool(SubPrimitiveChunk, 10); + static subMeshPool = new ReturnableObjectPool(SubMesh, 10); +} + +// Engine-level pools +class Engine { + _renderElementPool = new ClearableObjectPool(RenderElement); + _subRenderElementPool = new ClearableObjectPool(SubRenderElement); +} +``` + +## Best Practices + +### 1. Design for Batching + +```ts +// Good: Design systems with batching in mind +class ParticleSystem { + constructor() { + // Use shared material for all particles + this.material = new UnlitMaterial(engine); + this.material.baseTexture = this.particleAtlas; + } + + createParticle(): Particle { + const particle = new Particle(); + particle.setMaterial(this.material); // Shared material + return particle; + } +} +``` + +### 2. Minimize State Changes + +```ts +// Good: Group by material properties +const sprites = [sprite1, sprite2, sprite3]; +sprites.sort((a, b) => { + // Sort by material first, then by texture + if (a.material !== b.material) { + return a.material.instanceId - b.material.instanceId; + } + return a.texture.instanceId - b.texture.instanceId; +}); +``` + +### 3. Monitor Batching Effectiveness + +```ts +// Check batching statistics (development only) +class BatchingProfiler { + measureBatchingEfficiency(): void { + const renderQueue = camera._renderPipeline._cullingResults.opaqueQueue; + const totalElements = renderQueue.elements.length; + const batchedElements = renderQueue.batchedSubElements.length; + + console.log(`Batching efficiency: ${batchedElements}/${totalElements} elements`); + console.log(`Draw call reduction: ${((totalElements - batchedElements) / totalElements * 100).toFixed(1)}%`); + } +} +``` + +The batching system is a critical performance optimization that works automatically but can be significantly improved through careful design decisions around materials, textures, and object organization. diff --git a/docs/scripting/build-development-tools.md b/docs/scripting/build-development-tools.md new file mode 100644 index 0000000000..a22d10369a --- /dev/null +++ b/docs/scripting/build-development-tools.md @@ -0,0 +1,73 @@ +# Build and Development Tools + +Galacean Engine is developed in a pnpm workspace and relies on a Rollup-based toolchain for bundling, SWC for transpilation, Vitest/Playwright for automated testing, and ESLint/Prettier for linting and formatting. This document summarizes the scripts and configurations that drive those workflows. + +## Project scripts +All commands are defined in the workspace `package.json` and should be invoked with pnpm. + +| Command | Purpose | +| --- | --- | +| `pnpm install` | Install workspace dependencies (enforced by `npx only-allow pnpm`). | +| `pnpm dev` | Start Rollup in development mode (`BUILD_TYPE=MODULE`, `NODE_ENV=development`) with live rebuilds and a static server on port `9999`. | +| `pnpm watch` | Continuous module build (`BUILD_TYPE=MODULE`, `NODE_ENV=release`). Source maps are kept inline for debugging optimized output. | +| `pnpm watch:umd` | Continuous UMD build (`BUILD_TYPE=UMD`). | +| `pnpm build` | Perform a release build by running `b:module` and `b:types` across all packages. | +| `pnpm b:module` | Single-pass Rollup build that emits both ES module and CommonJS bundles. | +| `pnpm b:umd` | Generate minified and verbose UMD bundles. | +| `pnpm b:types` | Run each package’s TypeScript declaration build. | +| `pnpm b:all` | Produce module and UMD bundles plus type declarations in one pass. | +| `pnpm clean` | Remove `dist/` and `types/` folders from every package. | +| `pnpm lint` | ESLint over `packages/*/src` (TypeScript only). | +| `pnpm test` | Execute unit tests with Vitest (pretest installs Vitest and Playwright Chromium). | +| `pnpm coverage` | Run Vitest with V8 coverage reporting (`HEADLESS=true`). | +| `pnpm e2e` | Run Playwright end-to-end tests (Chromium is installed via `pree2e`). | +| `pnpm e2e:debug` | Launch Playwright in UI/debug mode. | +| `pnpm examples` | Start the examples workspace (`pnpm --filter @galacean/engine-examples dev`). | +| `pnpm release` | Use `bumpp` to bump package versions. | + +Husky is installed via `pnpm prepare`, and `lint-staged` runs `eslint --fix` on staged `.ts` files before commits. + +## Rollup configuration +The root `rollup.config.js` orchestrates builds for every package under `packages/` (excluding `design`). Key aspects: + +- **Inputs**: Each package’s `src/index.ts` is used as the root entry. +- **Environment variables**: `BUILD_TYPE` determines whether module, UMD, or both outputs are produced. `NODE_ENV` toggles debug/development behavior (e.g., the dev server plugin runs when `NODE_ENV=development`). +- **Plugins**: + - `@rollup/plugin-node-resolve` and `@rollup/plugin-commonjs` allow bundling npm dependencies. + - A custom GLSL plugin in `rollup-plugin-glsl` inlines shader files; compression is enabled for minified UMD builds. + - `rollup-plugin-swc3` transpiles TypeScript/ESNext to ES5 (loose mode with external helpers) and produces source maps. + - `rollup-plugin-jscc` injects compile-time macros (for example `_VERBOSE` for verbose shaderlab builds). + - `@rollup/plugin-replace` wires build metadata, including the package version (`__buildVersion`). + - `rollup-plugin-serve` serves package directories on port `9999` during `pnpm dev`. + - `rollup-plugin-swc3/minify` compresses UMD bundles when `compress=true`. +- **Outputs**: + - Module builds emit both ES module and CommonJS files as specified in each package’s `package.json` (`module` and `main` fields). + - UMD builds honor the `umd` field in each package’s `package.json` (global name and externals) and emit `.js`, `.min.js`, and optional verbose builds. + +## Type declarations +Each package exposes a `b:types` script (see the individual `package.json` files). `pnpm b:types` runs all of them in parallel, emitting `.d.ts` files to the `types/` directory inside each package. + +## Development workflow +1. **Install** dependencies: `pnpm install`. +2. **Start** the dev server: `pnpm dev` (Rollup serves builds at `http://localhost:9999` and rebuilds on change). Use `pnpm examples` to launch the example playground in parallel. +3. **Run tests** whenever you touch core logic: `pnpm test` for unit tests, `pnpm e2e` for Playwright scenarios. +4. **Lint** before committing: `pnpm lint` (automatically enforced by `lint-staged` on staged files). +5. **Build** release artifacts: `pnpm build` or the more granular `b:*` scripts (module, UMD, types). +6. **Clean** generated folders with `pnpm clean` when switching branches or resolving build issues. + +## Testing stack +- **Vitest** (configured by the root scripts) runs fast unit tests. Coverage uses the V8 provider via `@vitest/coverage-v8`. +- **Playwright** drives Chromium-based e2e tests. The repository installs the browser binary on-demand in the `pretest`/`pree2e` scripts. +- `pnpm coverage` sets `HEADLESS=true` to keep the tests deterministic under CI. + +## Optimization notes +- The Rollup build respects package-level `dependencies` and `peerDependencies` when marking externals. Keep these fields current to avoid bundling unintended code. +- For UMD builds, declare globals in the package `umd.globals` map (`package.json`) so Rollup can wire the correct runtime module names. +- When authoring shader files, remember that release UMD builds will minify GLSL; keep debug builds (`NODE_ENV=development`) handy for readable output. +- Prefer named imports (e.g., `import { WebGLEngine } from "@galacean/engine"`) to enable downstream tree shaking when consumers bundle their apps. + +## Best practices +- Use the provided scripts rather than invoking Rollup directly—environment variables and plugin options are coordinated for every package. +- Run `pnpm lint` and `pnpm test` before pushing to catch style or regression issues early. +- Keep the `rollup.config.js` plugin list in sync with any new package requirements (e.g., if you add a new asset suffix that needs to be inlined). +- Monitor bundle artifacts in `packages/*/dist` to ensure verbose/minified/module builds are generated as expected before publishing. diff --git a/docs/scripting/camera.md b/docs/scripting/camera.md new file mode 100644 index 0000000000..51c3291f7c --- /dev/null +++ b/docs/scripting/camera.md @@ -0,0 +1,252 @@ +# Camera + +The `Camera` component is the lens into the 3D world, determining what is rendered and how. It controls the projection (perspective or orthographic), culling, and rendering order. A scene can have multiple cameras, each rendering to a different part of the screen or to a texture. + +## Creating a Camera + +A Camera is a component that must be attached to an `Entity`. You can create a camera entity and add it to the scene. + +```ts +import { WebGLEngine, Entity, Camera } from "@galacean/engine"; + +const engine = await WebGLEngine.create({ canvas: "canvas" }); +const scene = engine.sceneManager.activeScene; +const rootEntity = scene.createRootEntity(); + +// Create an entity for the camera +const cameraEntity = rootEntity.createChild("camera_entity"); +cameraEntity.transform.setPosition(0, 0, 10); + +// Add a Camera component to the entity +const camera = cameraEntity.addComponent(Camera); +``` + +## Projection Types + +The camera supports two main projection types: Perspective and Orthographic. + +### Perspective Camera +This is the most common projection for 3D games. It creates a sense of depth, where objects appear smaller as they move further away. The viewing volume is a frustum. + +```ts +// By default, a camera is a perspective camera +camera.isOrthographic = false; + +// Adjust the field of view (in degrees) +camera.fieldOfView = 60; + +// Set the near and far clipping planes +camera.nearClipPlane = 0.1; +camera.farClipPlane = 100; +``` + +### Orthographic Camera +This projection removes perspective, making it ideal for 2D games, UI, or isometric views. All objects appear at the same scale regardless of their distance from the camera. The viewing volume is a rectangular box. + +```ts +camera.isOrthographic = true; + +// Set the size of the viewing volume. +// For a 2D game, this can correspond to half the screen height in world units. +camera.orthographicSize = 5; + +// Set the near and far clipping planes +camera.nearClipPlane = 0.1; +camera.farClipPlane = 100; +``` + +## Viewport and Multiple Cameras + +The `viewport` property allows the camera to render to a specific rectangular area of the canvas, defined in normalized coordinates (0,0 to 1,1). This is useful for split-screen multiplayer or minimaps. + +The `priority` property determines the rendering order. Cameras with a higher priority are rendered on top of those with a lower priority. + +```ts +// Main camera renders to the full screen +const mainCamera = mainCameraEntity.addComponent(Camera); +mainCamera.viewport = new Vector4(0, 0, 1, 1); +mainCamera.priority = 0; + +// Minimap camera renders to the top-right corner +const minimapCameraEntity = rootEntity.createChild("minimap_camera"); +const minimapCamera = minimapCameraEntity.addComponent(Camera); +minimapCamera.viewport = new Vector4(0.7, 0.7, 0.3, 0.3); // x, y, width, height +minimapCamera.priority = 1; // Rendered after the main camera +``` + +## Coordinate Conversion + +The Camera provides essential utility methods to convert coordinates between different spaces. + +- **World Space**: The 3D coordinate system of the scene. +- **Viewport Space**: A 2D normalized coordinate system where (0,0) is the bottom-left and (1,1) is the top-right of the camera's viewport. +- **Screen Space**: A 2D pixel-based coordinate system where (0,0) is the top-left of the canvas. + +```ts +import { Vector2, Vector3, Ray } from "@galacean/engine-math"; + +// Create a ray from the mouse position (in screen space) +const mousePosition = new Vector2(inputManager.pointers[0].position.x, inputManager.pointers[0].position.y); +const ray = camera.screenPointToRay(mousePosition); + +// Now you can use this ray for physics picking +const hitResult = new HitResult(); +if (scene.physics.raycast(ray, Number.MAX_VALUE, Layer.Default, hitResult)) { + console.log("Hit entity:", hitResult.entity.name); +} + +// Convert a 3D world point to 2D screen coordinates +const worldPosition = new Vector3(5, 2, 0); +const screenPosition = camera.worldToScreenPoint(worldPosition); +console.log(`Screen position: ${screenPosition.x}, ${screenPosition.y}`); + +// Convert viewport point to world space +const viewportPoint = new Vector3(0.5, 0.5, 10); // center of screen, 10 units from camera +const worldPoint = camera.viewportToWorldPoint(viewportPoint); +console.log(`World position: ${worldPoint.x}, ${worldPoint.y}, ${worldPoint.z}`); +``` + +## Rendering to a Texture + +Set the `renderTarget` property to make a camera render its view into a texture instead of the screen. This is the foundation for effects like security camera monitors, reflections, or post-processing. + +```ts +import { RenderTarget, Texture2D, TextureFormat } from "@galacean/engine"; + +// Create a render target +const renderColorTexture = new Texture2D(engine, 512, 512, TextureFormat.R8G8B8A8, true); +const renderDepthTexture = new Texture2D(engine, 512, 512, TextureFormat.Depth16, true); +const renderTarget = new RenderTarget(engine, 512, 512, renderColorTexture, renderDepthTexture); + +// Assign it to the camera +camera.renderTarget = renderTarget; + +// The `renderColorTexture` can now be used in a material +material.shaderData.setTexture("u_baseTexture", renderColorTexture); +``` + +## Advanced Camera Features + +### Depth and Opaque Textures + +Cameras can generate depth and opaque textures for advanced rendering effects: + +```ts +// Enable depth texture (accessible as camera_DepthTexture in shaders) +camera.depthTextureMode = true; + +// Enable opaque texture for transparent queue effects +camera.opaqueTextureEnabled = true; +camera.opaqueTextureDownsampling = 2; // Half resolution for performance + +// Note: Only non-transparent objects write to depth texture +``` + +### Anti-Aliasing and Quality Settings + +```ts +import { AntiAliasing } from "@galacean/engine"; + +// Enable FXAA anti-aliasing +camera.antiAliasing = AntiAliasing.FXAA; + +// Configure MSAA samples +camera.msaaSamples = 4; // 2, 4, or 8 samples + +// Enable HDR for wider color range +camera.enableHDR = true; + +// Enable post-processing pipeline +camera.enablePostProcess = true; + +// Preserve alpha channel in output +camera.isAlphaOutputRequired = true; +``` + +### Culling and Layer Management + +```ts +import { Layer } from "@galacean/engine"; + +// Control which layers this camera renders +camera.cullingMask = Layer.Layer0 | Layer.Layer1; // Only render these layers + +// Enable/disable frustum culling for performance +camera.enableFrustumCulling = true; // Default: true + +// Set clear flags +camera.clearFlags = CameraClearFlags.All; // Clear color, depth, and stencil +``` + +## API Reference + +```apidoc +Camera: + Properties: + isOrthographic: boolean + - Toggles between perspective (false) and orthographic (true) projection. + fieldOfView: number + - The vertical field of view in degrees (for perspective cameras). + orthographicSize: number + - Half the vertical size of the viewing volume (for orthographic cameras). + nearClipPlane: number + - The closest point the camera will render. + farClipPlane: number + - The furthest point the camera will render. + aspectRatio: number + - The aspect ratio (width/height). Automatically calculated from viewport by default. + viewport: Vector4 + - The normalized screen-space rectangle to render into `(x, y, width, height)`. + clearFlags: CameraClearFlags + - What to clear before rendering (e.g., color, depth). `CameraClearFlags.All` is default. + cullingMask: Layer + - A bitmask that determines which layers this camera renders. + priority: number + - Render order for cameras. Higher values render later (on top). + renderTarget: RenderTarget | null + - The target to render to. If null, renders to the canvas. + viewMatrix: Readonly + - The matrix that transforms from world to camera space. + projectionMatrix: Readonly + - The matrix that transforms from camera to clip space. + enableFrustumCulling: boolean + - If true, objects outside the camera's view frustum are not rendered. Default is true. + enablePostProcess: boolean + - If true, post-processing effects will be applied to this camera's output. + enableHDR: boolean + - If true, enables High Dynamic Range rendering, requiring a compatible device. + depthTextureMode: boolean + - Enables depth texture generation. Accessible as camera_DepthTexture in shaders. + opaqueTextureEnabled: boolean + - Enables opaque texture for transparent queue rendering effects. + opaqueTextureDownsampling: number + - Downsampling level for opaque texture (1 = no downsampling, 2 = half resolution). + antiAliasing: AntiAliasing + - Anti-aliasing method (None, FXAA). + msaaSamples: number + - Multi-sample anti-aliasing sample count (2, 4, 8). + isAlphaOutputRequired: boolean + - Whether to preserve alpha channel in output. + pixelViewport: Vector4 + - Camera viewport in screen pixels (read-only). + + Methods: + worldToScreenPoint(point: Vector3): Vector2 + - Transforms a point from world space to screen space (pixels). + screenToWorldPoint(point: Vector2, zDistance: number): Vector3 + - Transforms a point from screen space to world space at specified distance. + worldToViewportPoint(point: Vector3): Vector3 + - Transforms a point from world space to viewport space (normalized 0-1). + viewportToWorldPoint(point: Vector3): Vector3 + - Transforms a point from viewport space to world space. + screenPointToRay(point: Vector2): Ray + - Creates a Ray in world space from a 2D screen-space point. + viewportPointToRay(point: Vector2): Ray + - Creates a Ray in world space from a 2D viewport-space point. + screenToViewportPoint(point: Vector2): Vector2 + - Converts screen coordinates to viewport coordinates. + viewportToScreenPoint(point: Vector2): Vector2 + - Converts viewport coordinates to screen coordinates. + render(): void + - Manually triggers the camera to render a frame. +``` diff --git a/docs/scripting/clone-system.md b/docs/scripting/clone-system.md new file mode 100644 index 0000000000..b385299be4 --- /dev/null +++ b/docs/scripting/clone-system.md @@ -0,0 +1,105 @@ +# Clone System + +Galacean supports cloning entities (and their components) at runtime. The cloning pipeline is driven by `Entity.clone()` together with a set of property decorators that determine how component fields are copied. This document explains the default behavior and how to customize it for your own components. + +## Entity cloning +Calling `entity.clone()` creates a new `Entity` that: +- Copies the original entity’s name, layer, active state, and transform hierarchy. +- Instantiates the same component types in the same order. +- Recursively clones all child entities. + +```ts +const entity = scene.createRootEntity("Hero"); +const meshRenderer = entity.addComponent(MeshRenderer); +meshRenderer.mesh = heroMesh; + +const clone = entity.clone(); +clone.name = "Hero_Clone"; +scene.addRootEntity(clone); + +// Mesh references are shared; transforms are independent +console.log(clone.getComponent(MeshRenderer).mesh === meshRenderer.mesh); // true +console.log(clone.transform.worldPosition.equals(entity.transform.worldPosition)); // true +``` + +The transform on the clone is independent (`Transform` implements a deep clone), but resources such as materials or meshes remain shared unless you explicitly deep-clone them. + +## Clone decorators +Component fields default to *assignment* cloning (values are copied for primitives; object references are shared). Apply clone decorators to override that behavior on a per-property basis: + +| Decorator | Description | +| --- | --- | +| `@ignoreClone` | Skip this property entirely. Commonly used for cached handles or transient state. | +| `@assignmentClone` | Explicitly keep the default assignment behavior. Useful for documentation. | +| `@shallowClone` | Clone the container (object/array/typed array) but keep references to its elements. | +| `@deepClone` | Recursively deep clone the value. Array elements and object fields are copied according to their own decorators. | + +```ts +import { Script, ignoreClone, shallowClone, deepClone } from "@galacean/engine"; + +class Inventory extends Script { + @ignoreClone + tempHUDState: object | null = null; + + @shallowClone + items: string[] = ["potion", "sword"]; + + @deepClone + loadouts: { name: string; stats: number[] }[] = []; +} + +const entity = scene.createRootEntity("Player"); +const inv = entity.addComponent(Inventory); +inv.items[0] = "elixir"; +inv.loadouts.push({ name: "mage", stats: [10, 20] }); + +const clone = entity.clone().getComponent(Inventory); +clone.items[0] = "bomb"; // Mutates both arrays (shallow clone) +clone.loadouts[0].stats[0] = 99; // Only affects the clone (deep clone) +``` + +## Custom cloning hooks +For advanced scenarios you can implement the optional methods defined in `ICustomClone` / `IComponentCustomClone`: + +- Implement `copyFrom(source)` on your class to copy data into an existing instance when the clone system encounters it. +- Implement `_cloneTo(target)` on your component or class to run additional logic after the default property cloning finishes. Component implementations receive the source and target root entities so you can remap entity references. + +```ts +class Weapon implements ICustomClone { + constructor(public id: number, public owner?: Entity) {} + + copyFrom(source: Weapon): void { + this.id = source.id; + this.owner = source.owner; // intentionally shared + } +} + +class WeaponHolder extends Script implements IComponentCustomClone { + weapon: Weapon | null = null; + + _cloneTo(target: WeaponHolder, srcRoot: Entity, targetRoot: Entity): void { + if (this.weapon?.owner) { + // Map the owner entity from the source hierarchy to the clone hierarchy. + const path: number[] = []; + if (Entity._getEntityHierarchyPath(srcRoot, this.weapon.owner, path)) { + target.weapon = new Weapon(this.weapon.id); + target.weapon.owner = Entity._getEntityByHierarchyPath(targetRoot, path); + } + } + } +} +``` + +Most built-in components already supply appropriate decorators and `_cloneTo` hooks. For example, renderers mark GPU handles with `@ignoreClone` so live WebGL resources are not duplicated. + +## CloneManager (internal) +`CloneManager` drives decorator registration (`ignoreClone`, `assignmentClone`, `shallowClone`, `deepClone`) and performs property copies via `cloneProperty`. You typically do not call it directly—decorators automatically register the metadata, and `ComponentCloner.cloneComponent` uses that metadata whenever an entity is cloned. + +## Best practices +- **Decorate fields**: Mark transient or runtime-only fields with `@ignoreClone` to avoid copying state that should be reinitialized. +- **Watch nested data**: Use `@deepClone` on arrays/objects that should not share references with the original; otherwise they will point to the same data. +- **Reuse heavy resources**: Leave meshes, materials, and textures as assignment clones so clones share GPU assets. +- **Remap entity references**: Implement `_cloneTo` on components that store references to other entities to ensure those references point to the cloned hierarchy. +- **Test clones**: After introducing new clone rules, clone the entity in a unit test or editor script to confirm fields behave as expected. + +For more examples, see `docs/en/core/clone.mdx` in the repository. diff --git a/docs/scripting/component-dependencies.md b/docs/scripting/component-dependencies.md new file mode 100644 index 0000000000..0b3363f086 --- /dev/null +++ b/docs/scripting/component-dependencies.md @@ -0,0 +1,319 @@ +# Component Dependencies System + +The Component Dependencies system in Galacean Engine ensures proper component relationships and prevents runtime errors by automatically managing component dependencies. This system uses decorators to declare dependencies and provides automatic validation and resolution. + +## Overview + +The dependencies system provides: +- **Automatic dependency checking** when adding components +- **Dependency resolution** with configurable modes +- **Removal validation** to prevent breaking dependencies +- **Inheritance support** for component hierarchies + +## Core Concepts + +### @dependentComponents Decorator + +The `@dependentComponents` decorator declares that a component requires other components to function properly: + +```typescript +import { dependentComponents, DependentMode, Transform, Camera } from "@galacean/engine"; + +// Single dependency +@dependentComponents(Transform, DependentMode.CheckOnly) +export class Renderer extends Component { + // This component requires Transform to exist +} + +// Multiple dependencies +@dependentComponents([Transform, Camera], DependentMode.AutoAdd) +export class CameraController extends Component { + // This component requires both Transform and Camera +} +``` + +### Dependency Modes + +The system supports two dependency resolution modes: + +#### DependentMode.CheckOnly +Validates dependencies exist but doesn't automatically add them: + +```typescript +@dependentComponents(Transform, DependentMode.CheckOnly) +export class MeshRenderer extends Component {} + +// Usage +const entity = scene.createRootEntity(); +entity.addComponent(MeshRenderer); // ❌ Throws error: "Should add Transform before adding MeshRenderer" + +// Correct usage +entity.addComponent(Transform); // ✅ Add dependency first +entity.addComponent(MeshRenderer); // ✅ Now succeeds +``` + +#### DependentMode.AutoAdd +Automatically adds missing dependencies: + +```typescript +@dependentComponents(Transform, DependentMode.AutoAdd) +export class AutoComponent extends Component {} + +// Usage +const entity = scene.createRootEntity(); +entity.addComponent(AutoComponent); // ✅ Automatically adds Transform first +console.log(entity.getComponent(Transform)); // ✅ Transform was auto-added +``` + +## Built-in Component Dependencies + +Many core components have predefined dependencies: + +### Rendering Components +```typescript +// All renderers require Transform +@dependentComponents(Transform, DependentMode.CheckOnly) +export class Renderer extends Component {} + +@dependentComponents(Transform, DependentMode.CheckOnly) +export class MeshRenderer extends Renderer {} + +@dependentComponents(Transform, DependentMode.CheckOnly) +export class SpriteRenderer extends Renderer {} +``` + +### Camera System +```typescript +@dependentComponents(Transform, DependentMode.CheckOnly) +export class Camera extends Component {} +``` + +### Physics Components +```typescript +@dependentComponents(Transform, DependentMode.CheckOnly) +export class Collider extends Component {} + +@dependentComponents(DynamicCollider, DependentMode.AutoAdd) +export class Joint extends Component {} +``` + +### UI Components +```typescript +@dependentComponents(UITransform, DependentMode.AutoAdd) +export class UICanvas extends Component {} +``` + +## Custom Component Dependencies + +### Creating Dependent Components + +```typescript +import { Component, dependentComponents, DependentMode } from "@galacean/engine"; + +// Example: Audio component that requires Transform for 3D positioning +@dependentComponents(Transform, DependentMode.CheckOnly) +export class AudioSource extends Component { + private audioContext: AudioContext; + private gainNode: GainNode; + + onAwake(): void { + // Can safely access transform because dependency is guaranteed + const position = this.entity.transform.worldPosition; + this.updateAudioPosition(position); + } +} + +// Example: AI component with multiple dependencies +@dependentComponents([Transform, MeshRenderer], DependentMode.AutoAdd) +export class AIAgent extends Component { + onAwake(): void { + // Both Transform and MeshRenderer are guaranteed to exist + const renderer = this.entity.getComponent(MeshRenderer); + const transform = this.entity.transform; + } +} +``` + +### Complex Dependency Chains + +```typescript +// Base component with its own dependencies +@dependentComponents(Transform, DependentMode.CheckOnly) +export class MovementComponent extends Component {} + +// Derived component inherits parent dependencies +@dependentComponents(Camera, DependentMode.AutoAdd) +export class CameraMovement extends MovementComponent { + // Inherits Transform dependency from MovementComponent + // Adds Camera dependency +} + +// Usage +const entity = scene.createRootEntity(); +entity.addComponent(Transform); // Required by MovementComponent +entity.addComponent(CameraMovement); // Auto-adds Camera, inherits Transform requirement +``` + +## Dependency Validation + +### Add-time Validation + +The system validates dependencies when components are added: + +```typescript +@dependentComponents(MeshRenderer, DependentMode.CheckOnly) +export class MaterialController extends Component {} + +const entity = scene.createRootEntity(); + +try { + entity.addComponent(MaterialController); +} catch (error) { + console.error(error); // "Should add MeshRenderer before adding MaterialController" +} + +// Correct approach +entity.addComponent(Transform); // MeshRenderer dependency +entity.addComponent(MeshRenderer); // MaterialController dependency +entity.addComponent(MaterialController); // ✅ All dependencies satisfied +``` + +### Remove-time Validation + +The system prevents removing components that other components depend on: + +```typescript +const entity = scene.createRootEntity(); +const transform = entity.addComponent(Transform); +const renderer = entity.addComponent(MeshRenderer); + +try { + transform.destroy(); // ❌ Throws error: "Should remove MeshRenderer before remove Transform" +} catch (error) { + console.error(error); +} + +// Correct removal order +renderer.destroy(); // Remove dependent component first +transform.destroy(); // ✅ Now safe to remove +``` + +## Advanced Usage Patterns + +### Conditional Dependencies + +```typescript +@dependentComponents(Transform, DependentMode.CheckOnly) +export class ConditionalComponent extends Component { + private requiresRenderer: boolean = false; + + onAwake(): void { + if (this.requiresRenderer) { + // Manually check for additional dependencies + const renderer = this.entity.getComponent(MeshRenderer); + if (!renderer) { + throw new Error("MeshRenderer required when requiresRenderer is true"); + } + } + } +} +``` + +### Dynamic Dependency Management + +```typescript +export class DynamicSystem extends Component { + private currentMode: string = "basic"; + + switchMode(mode: string): void { + switch (mode) { + case "physics": + this.ensureComponent(Collider); + break; + case "audio": + this.ensureComponent(AudioSource); + break; + case "rendering": + this.ensureComponent(MeshRenderer); + break; + } + this.currentMode = mode; + } + + private ensureComponent( + componentType: new (entity: Entity) => T + ): T { + let component = this.entity.getComponent(componentType); + if (!component) { + component = this.entity.addComponent(componentType); + } + return component; + } +} +``` + +### Dependency Groups + +```typescript +// Create logical dependency groups +const RenderingDependencies = [Transform, MeshRenderer, Material]; +const PhysicsDependencies = [Transform, Collider]; +const UIDependencies = [UITransform, UIRenderer]; + +@dependentComponents(RenderingDependencies, DependentMode.AutoAdd) +export class AdvancedRenderer extends Component {} + +@dependentComponents(PhysicsDependencies, DependentMode.CheckOnly) +export class PhysicsController extends Component {} +``` + +## Best Practices + +### Choosing Dependency Modes +- **Use CheckOnly** for core architectural dependencies (e.g., Transform for renderers) +- **Use AutoAdd** for convenience components or when dependencies are always needed +- **Consider user experience** - AutoAdd reduces boilerplate but may hide component relationships + +### Dependency Design +- **Keep dependencies minimal** - only declare truly required components +- **Use inheritance wisely** - derived components inherit parent dependencies +- **Document dependencies** - make component relationships clear to users + +### Error Handling +```typescript +export class SafeComponent extends Component { + onAwake(): void { + try { + // Attempt to use dependent component + const renderer = this.entity.getComponent(MeshRenderer); + if (renderer) { + this.setupRendering(renderer); + } + } catch (error) { + console.warn("Optional dependency not available:", error); + } + } +} +``` + +### Testing Dependencies +```typescript +describe("Component Dependencies", () => { + it("should enforce Transform dependency", () => { + const entity = scene.createRootEntity(); + + expect(() => { + entity.addComponent(MeshRenderer); + }).toThrow("Should add Transform before adding MeshRenderer"); + }); + + it("should auto-add dependencies when configured", () => { + const entity = scene.createRootEntity(); + entity.addComponent(AutoDependentComponent); + + expect(entity.getComponent(Transform)).toBeTruthy(); + }); +}); +``` + +The Component Dependencies system ensures robust component relationships while providing flexibility in how dependencies are resolved, leading to more reliable and maintainable component architectures. diff --git a/docs/scripting/component.md b/docs/scripting/component.md new file mode 100644 index 0000000000..7a800e1445 --- /dev/null +++ b/docs/scripting/component.md @@ -0,0 +1,138 @@ +# Component + +Galacean follows an entity–component pattern. Entities are simple containers, while components (renderers, colliders, audio sources, scripts…) provide concrete behaviour. Most gameplay logic is implemented by extending `Script`—see `docs/llm/script-system.md` for the full scripting guide. This page focuses on managing components in general. + +## Managing Components on an Entity + +Components are attached to entities and retrieved by type. + +```ts +import { Entity, Script, MeshRenderer, DirectLight, Color } from "@galacean/engine"; + +// Assume `rotatorEntity` is an existing Entity + +// Add a component by its class +const rotator = rotatorEntity.addComponent(RotatorScript); + +// Add built-in components with configuration +const light = rotatorEntity.addComponent(DirectLight); +light.color = new Color(1, 1, 1); +light.intensity = 1.0; + +// Get a component that is already on the entity +const existingRotator = rotatorEntity.getComponent(RotatorScript); + +if (rotator === existingRotator) { + console.log("They are the same instance."); +} + +// Get single component (returns null if not found) +const renderer = rotatorEntity.getComponent(MeshRenderer); +if (renderer) { + renderer.enabled = false; +} + +// Get all components of a certain type +const allScripts: Script[] = []; +rotatorEntity.getComponents(Script, allScripts); + +// Get components from entity and its children +const allRenderers: MeshRenderer[] = []; +rotatorEntity.getComponentsIncludeChildren(MeshRenderer, allRenderers); + +// Access entity from component +const entityFromComponent = light.entity; // Returns rotatorEntity + +// To remove a component, you must destroy it +rotator.destroy(); +``` + +> **Note:** Adding a component that already exists on an entity will not add a second one; the existing instance is returned. + +## Enabling and Disabling + +A component is active only when both `component.enabled` and `entity.isActiveInHierarchy` are `true`. Disabling a component pauses its behaviour without removing it: + +```ts +const rotator = rotatorEntity.getComponent(RotatorScript); + +rotator.enabled = false; // pauses updates +rotator.enabled = true; // resumes updates +``` + +## Built-in Component Overview + +- **Rendering**: `MeshRenderer`, `SkinnedMeshRenderer`, `SpriteRenderer`, `TrailRenderer`, `ParticleRenderer` +- **Lighting**: `DirectLight`, `PointLight`, `SpotLight`, `AmbientLight` +- **Physics (optional packages)**: `RigidBody`, `Collider` variants, `Joint` components +- **Audio**: `AudioListener`, `AudioSource` +- **UI & 2D**: `SpriteMask`, `TextRenderer`, `UI*` components +- **Logic**: `Script` for custom behaviour (lifecycle详情见脚本文档) + +Consult each component’s dedicated documentation for configuration details. + +## Removal and Cleanup + +Destroying a component detaches it permanently: + +```ts +const renderer = entity.getComponent(MeshRenderer); +renderer?.destroy(); +``` + +`destroy()` triggers the component’s `onDestroy()` hook so you can release resources in custom scripts. + +## Further Reading + +- `docs/llm/script-system.md` – 编写脚本、生命周期、事件回调 +- `docs/llm/renderer.md` – 渲染组件详解 +- `docs/llm/physics-scene.md` / `docs/llm/collider.md` – 物理组件 + + + +## API Reference + +```apidoc +Entity Component Management: + Methods: + addComponent(type: new() => T): T + - Creates and attaches a component of the specified type to the entity. + - Returns the component instance. + getComponent(type: new() => T): T | null + - Returns the first component of the specified type or null if not found. + getComponents(type: new() => T, results: T[]): void + - Populates the provided array with all components of the specified type. + getComponentsIncludeChildren(type: new() => T, results: T[]): void + - Recursively finds components in entity and all its children. + +Component: + Properties: + enabled: boolean + - Controls component activation. When false, lifecycle methods are not called. + entity: Entity + - Reference to the entity this component is attached to. (Read-only) + scene: Scene + - Reference to the scene containing this component's entity. (Read-only) + + Methods: + destroy(): void + - Destroys the component and removes it from its entity. + - Triggers onDestroy() lifecycle method before removal. + +Script (extends Component): + Lifecycle Methods: + onAwake(): void + - Called once when script is first initialized. Use for setup. + onEnable(): void + - Called when script becomes active (enabled=true and entity active). + onStart(): void + - Called before first onUpdate, after all onAwake calls complete. + onUpdate(deltaTime: number): void + - Called every frame. Use for game logic updates. + onLateUpdate(deltaTime: number): void + - Called after all onUpdate calls. Use for camera following, etc. + onDisable(): void + - Called when script becomes inactive (enabled=false or entity inactive). + onDestroy(): void + - Called once before component destruction. Use for cleanup. +``` diff --git a/docs/scripting/engine-configuration.md b/docs/scripting/engine-configuration.md new file mode 100644 index 0000000000..7ba89f6a56 --- /dev/null +++ b/docs/scripting/engine-configuration.md @@ -0,0 +1,441 @@ +# Engine Configuration + +Galacean Engine provides comprehensive configuration options for initialization, performance tuning, and runtime behavior. This guide covers all available configuration parameters and best practices for different deployment scenarios. + +## Engine Initialization + +### Basic Configuration + +```ts +import { WebGLEngine } from "@galacean/engine"; + +// Minimal configuration +const engine = await WebGLEngine.create({ + canvas: "canvas-id" // Canvas element ID or HTMLCanvasElement +}); + +// Canvas element reference +const canvasElement = document.getElementById("canvas") as HTMLCanvasElement; +const engine = await WebGLEngine.create({ + canvas: canvasElement +}); + +// OffscreenCanvas support (for Web Workers) +const offscreenCanvas = new OffscreenCanvas(800, 600); +const engine = await WebGLEngine.create({ + canvas: offscreenCanvas +}); +``` + +### Complete Configuration Interface + +```ts +interface WebGLEngineConfiguration { + // Required: Canvas target + canvas: HTMLCanvasElement | OffscreenCanvas | string; + + // Optional: Graphics device configuration + graphicDeviceOptions?: WebGLGraphicDeviceOptions; + + // Optional: Color space configuration + colorSpace?: ColorSpace; + + // Optional: Physics engine + physics?: IPhysics; + + // Optional: XR device for VR/AR + xrDevice?: IXRDevice; + + // Optional: Custom shader compilation system + shaderLab?: IShaderLab; + + // Optional: Input system configuration + input?: IInputOptions; + + // Optional: GLTF loader configuration + gltf?: GLTFConfiguration; + + // Optional: KTX2 texture loader configuration + ktx2Loader?: KTX2Configuration; +} +``` + +## Graphics Device Options + +### WebGL Context Configuration + +```ts +interface WebGLGraphicDeviceOptions { + // WebGL version control + webGLMode?: WebGLMode; // Auto, WebGL1, or WebGL2 + + // Context creation parameters + alpha?: boolean; // Alpha channel support + depth?: boolean; // Depth buffer + stencil?: boolean; // Stencil buffer + antialias?: boolean; // MSAA anti-aliasing + premultipliedAlpha?: boolean; // Premultiplied alpha + preserveDrawingBuffer?: boolean; // Preserve buffer contents + powerPreference?: WebGLPowerPreference; // GPU selection preference + failIfMajorPerformanceCaveat?: boolean; // Fail on software rendering + desynchronized?: boolean; // Low-latency mode + xrCompatible?: boolean; // WebXR compatibility + + // Internal options (advanced) + _forceFlush?: boolean; // Force command buffer flush + _maxAllowSkinUniformVectorCount?: number; // Skinning uniform limit +} + +// WebGL version modes +enum WebGLMode { + Auto = 0, // Prefer WebGL2, fallback to WebGL1 + WebGL2 = 1, // Force WebGL2 (fail if not supported) + WebGL1 = 2 // Force WebGL1 +} + +// GPU power preferences +type WebGLPowerPreference = "default" | "high-performance" | "low-power"; +``` + +### Platform-Specific Configurations + +```ts +import { SystemInfo, Platform } from "@galacean/engine"; + +// Adaptive configuration based on platform +const engine = await WebGLEngine.create({ + canvas: "canvas", + graphicDeviceOptions: { + // WebGL version selection + webGLMode: SystemInfo.platform === Platform.iOS ? WebGLMode.WebGL2 : WebGLMode.Auto, + + // Performance optimizations + powerPreference: SystemInfo.platform === Platform.Mac ? "high-performance" : "default", + + // Platform-specific features + antialias: SystemInfo.platform !== Platform.Android, // Disable on Android for performance + alpha: SystemInfo.platform === Platform.Web, // Enable for web transparency + + // Memory optimizations + preserveDrawingBuffer: false, // Disable for better performance + + // XR support + xrCompatible: SystemInfo.platform === Platform.Web && 'xr' in navigator + } +}); +``` + +### Performance-Oriented Configuration + +```ts +// High-performance configuration +const highPerformanceEngine = await WebGLEngine.create({ + canvas: "canvas", + graphicDeviceOptions: { + webGLMode: WebGLMode.WebGL2, // Use latest WebGL + powerPreference: "high-performance", // Request discrete GPU + alpha: false, // Disable alpha for performance + antialias: false, // Disable MSAA (use FXAA instead) + preserveDrawingBuffer: false, // Don't preserve buffer + desynchronized: true, // Enable low-latency mode + failIfMajorPerformanceCaveat: true // Fail on software rendering + } +}); + +// Quality-oriented configuration +const qualityEngine = await WebGLEngine.create({ + canvas: "canvas", + graphicDeviceOptions: { + webGLMode: WebGLMode.Auto, + powerPreference: "high-performance", + alpha: true, // Enable alpha blending + antialias: true, // Enable MSAA + depth: true, // Enable depth buffer + stencil: true, // Enable stencil operations + premultipliedAlpha: true // Correct alpha handling + } +}); +``` + +## Runtime Configuration + +### Frame Rate and Timing Control + +```ts +// V-sync based timing (recommended) +engine.vSyncCount = 1; // 60 FPS on 60Hz display +engine.vSyncCount = 2; // 30 FPS on 60Hz display +engine.vSyncCount = 0; // Disable V-sync + +// Custom frame rate (when V-sync disabled) +engine.targetFrameRate = 60; // Target 60 FPS +engine.targetFrameRate = 120; // Target 120 FPS +engine.targetFrameRate = Number.POSITIVE_INFINITY; // Unlimited + +// Adaptive frame rate based on performance +class AdaptiveFrameRate { + private engine: Engine; + private targetFrameTime = 16.67; // 60 FPS + private frameTimeHistory: number[] = []; + + constructor(engine: Engine) { + this.engine = engine; + } + + update(): void { + const frameTime = this.engine.time.deltaTime * 1000; + this.frameTimeHistory.push(frameTime); + + if (this.frameTimeHistory.length > 60) { + this.frameTimeHistory.shift(); + } + + const avgFrameTime = this.frameTimeHistory.reduce((a, b) => a + b, 0) / this.frameTimeHistory.length; + + if (avgFrameTime > this.targetFrameTime * 1.2) { + // Performance too low, reduce target + this.engine.targetFrameRate = Math.max(30, this.engine.targetFrameRate - 5); + } else if (avgFrameTime < this.targetFrameTime * 0.8) { + // Performance headroom, increase target + this.engine.targetFrameRate = Math.min(120, this.engine.targetFrameRate + 5); + } + } +} +``` + +### Engine Lifecycle Management + +```ts +// Engine state management +class EngineManager { + private engine: Engine; + private isInitialized = false; + + async initialize(config: WebGLEngineConfiguration): Promise { + if (this.isInitialized) { + throw new Error("Engine already initialized"); + } + + this.engine = await WebGLEngine.create(config); + this.isInitialized = true; + + // Setup event listeners + this.setupEventListeners(); + + // Start render loop + this.engine.run(); + } + + pause(): void { + if (this.engine && !this.engine.isPaused) { + this.engine.pause(); + } + } + + resume(): void { + if (this.engine && this.engine.isPaused) { + this.engine.resume(); + } + } + + destroy(): void { + if (this.engine && !this.engine.destroyed) { + this.engine.destroy(); + this.isInitialized = false; + } + } + + private setupEventListeners(): void { + // Handle visibility changes + document.addEventListener('visibilitychange', () => { + if (document.hidden) { + this.pause(); + } else { + this.resume(); + } + }); + + // Handle device context loss + this.engine.on('devicelost', () => { + console.warn('WebGL context lost'); + }); + + this.engine.on('devicerestored', () => { + console.log('WebGL context restored'); + }); + } +} +``` + +## Subsystem Configuration + +### Physics Engine Configuration + +```ts +import { PhysXPhysics, LitePhysics } from "@galacean/engine"; + +// PhysX physics (full-featured) +const physxEngine = await WebGLEngine.create({ + canvas: "canvas", + physics: new PhysXPhysics() +}); + +// Lite physics (lightweight) +const liteEngine = await WebGLEngine.create({ + canvas: "canvas", + physics: new LitePhysics() +}); + +// Conditional physics based on requirements +const physics = needsAdvancedPhysics ? new PhysXPhysics() : new LitePhysics(); +const engine = await WebGLEngine.create({ + canvas: "canvas", + physics: physics +}); +``` + +### Input System Configuration + +```ts +// Input configuration +const engine = await WebGLEngine.create({ + canvas: "canvas", + input: { + pointerTarget: document, // Event listener target + enablePointerLock: true, // Enable pointer lock API + enableGamepad: true, // Enable gamepad support + touchSensitivity: 1.0, // Touch input sensitivity + mouseSensitivity: 1.0 // Mouse input sensitivity + } +}); +``` + +### Loader Configuration + +```ts +// GLTF loader configuration +const engine = await WebGLEngine.create({ + canvas: "canvas", + gltf: { + meshOpt: { + workerCount: navigator.hardwareConcurrency || 4, // Use available CPU cores + wasmUrl: "/path/to/meshopt_decoder.wasm" // Custom WASM path + } + }, + ktx2Loader: { + workerCount: 2, // KTX2 worker threads + wasmUrl: "/path/to/basis_transcoder.wasm" // Custom transcoder path + } +}); +``` + +## Environment-Specific Configurations + +### Development Configuration + +```ts +// Development environment +const developmentConfig: WebGLEngineConfiguration = { + canvas: "canvas", + graphicDeviceOptions: { + webGLMode: WebGLMode.Auto, + alpha: true, // Enable for debugging + preserveDrawingBuffer: true, // Enable for screenshots + failIfMajorPerformanceCaveat: false // Allow software rendering + } +}; +``` + +### Production Configuration + +```ts +// Production environment +const productionConfig: WebGLEngineConfiguration = { + canvas: "canvas", + graphicDeviceOptions: { + webGLMode: WebGLMode.Auto, + alpha: false, // Optimize performance + preserveDrawingBuffer: false, // Optimize memory + powerPreference: "high-performance", + failIfMajorPerformanceCaveat: true // Ensure hardware acceleration + } +}; +``` + +### Mobile-Optimized Configuration + +```ts +// Mobile optimization +const mobileConfig: WebGLEngineConfiguration = { + canvas: "canvas", + graphicDeviceOptions: { + webGLMode: WebGLMode.Auto, + powerPreference: "low-power", // Preserve battery + antialias: false, // Reduce GPU load + alpha: false, // Optimize performance + preserveDrawingBuffer: false // Reduce memory usage + } +}; + +// Apply mobile-specific settings +const engine = await WebGLEngine.create(mobileConfig); +engine.targetFrameRate = 30; // Reduce frame rate for battery life +``` + +## Configuration Validation and Error Handling + +```ts +class EngineConfigValidator { + static validate(config: WebGLEngineConfiguration): ValidationResult { + const errors: string[] = []; + const warnings: string[] = []; + + // Validate canvas + if (!config.canvas) { + errors.push("Canvas is required"); + } + + // Validate WebGL support + if (config.graphicDeviceOptions?.webGLMode === WebGLMode.WebGL2) { + if (!this.isWebGL2Supported()) { + errors.push("WebGL2 not supported on this device"); + } + } + + // Performance warnings + if (config.graphicDeviceOptions?.preserveDrawingBuffer) { + warnings.push("preserveDrawingBuffer may impact performance"); + } + + return { errors, warnings, isValid: errors.length === 0 }; + } + + private static isWebGL2Supported(): boolean { + const canvas = document.createElement('canvas'); + return !!canvas.getContext('webgl2'); + } +} + +// Usage +const config: WebGLEngineConfiguration = { + canvas: "canvas", + graphicDeviceOptions: { + webGLMode: WebGLMode.WebGL2, + preserveDrawingBuffer: true + } +}; + +const validation = EngineConfigValidator.validate(config); +if (!validation.isValid) { + console.error("Configuration errors:", validation.errors); + return; +} + +if (validation.warnings.length > 0) { + console.warn("Configuration warnings:", validation.warnings); +} + +const engine = await WebGLEngine.create(config); +``` + +This configuration system provides fine-grained control over engine behavior while maintaining reasonable defaults for most use cases. Choose configurations based on your specific performance requirements, target platforms, and feature needs. diff --git a/docs/scripting/engine-scene.md b/docs/scripting/engine-scene.md new file mode 100644 index 0000000000..cf7bcd5600 --- /dev/null +++ b/docs/scripting/engine-scene.md @@ -0,0 +1,417 @@ +# Engine & Scene + +Galacean's `Engine` is the central orchestrator that manages the rendering loop, resource lifecycle, and subsystem coordination. The `Scene` represents a complete 3D environment containing entities, lighting, and post-processing configuration, while `SceneManager` handles multiple scene creation, switching, and merging operations. + +## Quick Start + +```ts +import { WebGLEngine, Scene, Entity, MeshRenderer, PrimitiveMesh } from "@galacean/engine"; + +// Create and initialize engine +const engine = await WebGLEngine.create({ canvas: "canvas" }); +const scene = engine.sceneManager.activeScene; + +// Create a basic entity with mesh +const cubeEntity = scene.createRootEntity("Cube"); +const renderer = cubeEntity.addComponent(MeshRenderer); +renderer.mesh = PrimitiveMesh.createCuboid(engine, 1, 1, 1); + +// Start the render loop +engine.run(); +``` + +- Engine automatically creates a default scene accessible via `sceneManager.activeScene`. +- Call `engine.run()` to start the main loop; use `pause()` and `resume()` for runtime control. +- Scene entities require explicit creation via `createRootEntity()` or constructor + `addRootEntity()`. + +## Engine Lifecycle Management + +```ts +// Engine configuration and creation +const engine = await WebGLEngine.create({ + canvas: "canvas", + physics: new PhysXPhysics(), + graphicDeviceOptions: { + powerPreference: "high-performance", + alpha: false, + depth: true, + stencil: true + } +}); + +// Runtime control +engine.targetFrameRate = 60; // Limit to 60 FPS +engine.vSyncCount = 1; // V-sync configuration + +// Pause and resume functionality +engine.pause(); // Stops render loop +engine.resume(); // Resumes from pause state + +// Cleanup when done +engine.destroy(); // Disposes all resources +``` + +- `WebGLEngine.create()` initializes WebGL context, creates default scene, and prepares all subsystems. +- `targetFrameRate` controls frame limiting; set to 0 for unlimited framerate. +- `vSyncCount` adjusts vertical synchronization (0 = disabled, 1 = 60Hz, 2 = 30Hz). +- Always call `destroy()` for proper resource cleanup when the application ends. + +> **Performance Tip**: Use `pause()` during inactive periods to reduce CPU/GPU usage and battery consumption. + +## Scene Management Operations + +```ts +const sceneManager = engine.sceneManager; + +// Create additional scenes +const gameScene = new Scene(engine, "GameLevel"); +const menuScene = new Scene(engine, "MainMenu"); + +sceneManager.addScene(gameScene); +sceneManager.addScene(menuScene); + +// Switch between scenes +sceneManager.activeScene = gameScene; + +// Merge scene content +sceneManager.mergeScenes(gameScene, menuScene); // Merge menuScene into gameScene + +// Load scene from external source +const loadedScene = await sceneManager.loadScene(sceneUrl); +``` + +- `SceneManager` maintains a collection of scenes but only renders the `activeScene`. +- Scene switching immediately updates rendering; previous scene remains in memory. +- `mergeScenes()` moves all entities from source to target scene and destroys the source. +- Scene names are optional but useful for debugging and identification. + +## Scene Hierarchy Operations + +```ts +const scene = engine.sceneManager.activeScene; + +// Create entities in scene +const rootEntity = scene.createRootEntity("GameRoot"); +const childEntity = new Entity(engine, "Child"); +rootEntity.addChild(childEntity); + +// Scene-level entity management +scene.addRootEntity(rootEntity); +scene.removeRootEntity(rootEntity); + +// Search operations +const foundEntity = scene.findEntityByName("Player"); +const deepEntity = scene.findEntityByPath("/GameRoot/Player/Weapon"); + +// Access root entities +const rootCount = scene.rootEntitiesCount; +const allRoots = scene.rootEntities; // ReadonlyArray +``` + +- Scene maintains a flat array of root entities, each managing their own hierarchy. +- `findEntityByPath()` supports slash-delimited navigation similar to file systems. +- `findEntityByName()` performs depth-first search across all entities in the scene. +- Root entity operations automatically handle activation/deactivation propagation. + +## Environment Configuration + +```ts +const scene = engine.sceneManager.activeScene; + +// Background and ambient lighting +scene.background.mode = BackgroundMode.Sky; +scene.ambientLight.diffuseIntensity = 0.3; +scene.ambientLight.specularIntensity = 0.2; + +// Fog configuration +scene.fogMode = FogMode.Linear; +scene.fogColor.set(0.5, 0.5, 0.6, 1.0); +scene.fogStart = 10; +scene.fogEnd = 100; + +// Shadow settings +scene.castShadows = true; +scene.shadowResolution = ShadowResolution.Medium; +scene.shadowDistance = 50; +scene.shadowCascades = ShadowCascadesMode.TwoCascades; +``` + +- Scene background handles skybox, solid color, or texture rendering. +- Ambient light provides global illumination without directional bias. +- Fog affects all rendered objects; exponential modes use `fogDensity` instead of start/end. +- Shadow configuration impacts performance; choose resolution based on quality requirements. + +## Post-Processing Integration + +```ts +// Engine-level post-processing +const bloomEffect = new BloomEffect(); +engine.addPostProcessPass(bloomEffect); + +// Scene-specific effects +const scene = engine.sceneManager.activeScene; +scene.postProcessManager.addEffect(new TonemappingEffect()); + +// Ambient occlusion (requires depth texture) +scene.ambientOcclusion.enabled = true; +scene.ambientOcclusion.quality = AmbientOcclusionQuality.Medium; +``` + +- Engine post-processing applies to all scenes; scene effects apply only to that scene. +- Post-process order matters; add effects in desired execution sequence. +- Some effects require specific render targets; engine handles allocation automatically. + +## Resource Management + +```ts +// Engine resource access +const resourceManager = engine.resourceManager; +const texture = await resourceManager.load("path/to/texture.jpg"); + +// Scene data management +const shaderData = scene.shaderData; +shaderData.setFloat("u_CustomValue", 1.5); +shaderData.setTexture("u_CustomTexture", texture); + +// Physics integration +const physicsScene = scene.physics; +physicsScene.gravity = new Vector3(0, -9.81, 0); +``` + +- Engine coordinates resource loading across all scenes and systems. +- Scene shader data provides global uniform values accessible in all materials. +- Physics scene manages simulation parameters and collider registration. + +## Performance Monitoring + +```ts +// Frame timing information +const time = engine.time; +console.log(`FPS: ${1 / time.deltaTime}`); +console.log(`Frame: ${time.frameCount}`); + +// Rendering statistics +console.log(`Render calls: ${engine.renderCount}`); + +// Device capabilities +if (engine.destroyed) { + console.log("Engine has been destroyed"); +} + +// Force device context testing +engine.forceLoseDevice(); // Simulate device loss +engine.forceRestoreDevice(); // Simulate device restoration +``` + +- `Time` provides frame timing, delta time, and total elapsed time since engine start. +- Engine tracks render call count and other performance metrics internally. +- Device loss testing helps validate application recovery behavior. + +## API Reference + +```apidoc +Engine: + Properties: + canvas: HTMLCanvasElement | OffscreenCanvas + - Target canvas element for rendering output. + resourceManager: ResourceManager + - Central resource loading and caching system. + sceneManager: SceneManager + - Scene collection and active scene management. + settings: EngineSettings + - Engine configuration and capability information. + time: Time + - Frame timing and temporal information. + targetFrameRate: number + - Maximum frames per second; 0 = unlimited. + vSyncCount: number + - Vertical sync interval (0, 1, 2). + isPaused: boolean + - Whether the render loop is currently paused. + destroyed: boolean + - Whether the engine has been destroyed. + + Methods: + static create(config: WebGLEngineConfiguration): Promise + - Creates and initializes a new WebGL engine instance. + run(): void + - Starts the main render loop. + pause(): void + - Pauses the render loop without destroying resources. + resume(): void + - Resumes the render loop from pause state. + update(): void + - Single frame update; called automatically by run(). + destroy(): void + - Disposes all resources and stops the engine. + createEntity(name?: string): Entity + - Creates a detached entity owned by this engine. + addPostProcessPass(pass: PostProcessPass): void + - Adds engine-level post-processing effect. + forceLoseDevice(): void + - Simulates WebGL context loss for testing. + forceRestoreDevice(): void + - Simulates WebGL context restoration for testing. + +SceneManager: + Properties: + engine: Engine + - Owning engine instance. + activeScene: Scene + - Currently rendering scene. + scenes: ReadonlyArray + - All scenes managed by this instance. + + Methods: + addScene(scene: Scene, index?: number): void + - Adds scene to management collection. + removeScene(scene: Scene): void + - Removes scene from collection and deactivates it. + loadScene(url: string, options?: LoaderOptions): Promise + - Loads scene from external resource. + mergeScenes(target: Scene, source: Scene): void + - Moves all entities from source to target scene. + +Scene: + Properties: + name: string + - Scene identifier for debugging and management. + isActive: boolean + - Whether this scene participates in rendering. + physics: PhysicsScene + - Physics simulation instance for this scene. + background: Background + - Background rendering configuration. + ambientLight: AmbientLight + - Global ambient lighting settings. + postProcessManager: PostProcessManager + - Scene-specific post-processing effects. + shaderData: ShaderData + - Global shader uniform data for this scene. + + // Shadow Configuration + castShadows: boolean + - Enable shadow casting globally. + shadowResolution: ShadowResolution + - Shadow map texture resolution. + shadowDistance: number + - Maximum shadow rendering distance. + shadowCascades: ShadowCascadesMode + - Number of cascade splits for directional lights. + + // Fog Configuration + fogMode: FogMode + - Fog calculation method (Linear, Exponential, ExponentialSquared). + fogColor: Color + - Fog color applied to distant objects. + fogStart: number + - Linear fog start distance. + fogEnd: number + - Linear fog end distance. + fogDensity: number + - Exponential fog density parameter. + + Methods: + createRootEntity(name?: string): Entity + - Creates and adds new root entity to scene. + addRootEntity(entity: Entity, index?: number): void + - Adds existing entity as scene root. + removeRootEntity(entity: Entity): void + - Removes root entity from scene hierarchy. + getRootEntity(index: number): Entity | undefined + - Gets root entity by index. + findEntityByName(name: string): Entity | null + - Depth-first search for entity by name. + findEntityByPath(path: string): Entity | null + - Finds entity using slash-delimited path. +``` + +## Configuration Reference + +### WebGLEngineConfiguration + +```apidoc +WebGLEngineConfiguration: + Properties: + canvas: HTMLCanvasElement | OffscreenCanvas | string + - Canvas element, OffscreenCanvas, or canvas element ID for rendering. + physics?: IPhysics + - Physics system implementation (e.g., PhysXPhysics, LitePhysics). + xrDevice?: IXRDevice + - XR device implementation for WebXR support. + shaderLab?: IShaderLab + - Custom shader compilation system. + input?: IInputOptions + - Input handling configuration options. + graphicDeviceOptions?: WebGLGraphicDeviceOptions + - WebGL context and rendering configuration. +``` + +### WebGLGraphicDeviceOptions + +```apidoc +WebGLGraphicDeviceOptions: + Properties: + webGLMode?: WebGLMode + - WebGL version mode (WebGL 1.0 or 2.0). + alpha?: boolean + - Whether the canvas has an alpha channel. @defaultValue `true` + depth?: boolean + - Whether to enable depth buffer. @defaultValue `true` + desynchronized?: boolean + - Whether to enable low-latency rendering mode. + failIfMajorPerformanceCaveat?: boolean + - Whether to fail creation if performance is significantly reduced. + powerPreference?: WebGLPowerPreference + - GPU power preference: "default", "high-performance", or "low-power". + premultipliedAlpha?: boolean + - Whether alpha values are premultiplied. @defaultValue `true` + preserveDrawingBuffer?: boolean + - Whether to preserve drawing buffer contents. @defaultValue `false` + stencil?: boolean + - Whether to enable stencil buffer. @defaultValue `false` + xrCompatible?: boolean + - Whether the context is compatible with WebXR. @defaultValue `false` +``` + +### Configuration Examples + +```typescript +// Basic configuration +const engine = await WebGLEngine.create({ + canvas: "canvas-id" +}); + +// Full configuration with physics and graphics options +const engine = await WebGLEngine.create({ + canvas: document.getElementById("canvas"), + physics: new PhysXPhysics(), + graphicDeviceOptions: { + webGLMode: WebGLMode.WebGL2, + powerPreference: "high-performance", + alpha: false, + depth: true, + stencil: true, + preserveDrawingBuffer: false, + xrCompatible: true + } +}); + +// XR-enabled configuration +const engine = await WebGLEngine.create({ + canvas: "canvas", + xrDevice: new WebXRDevice(), + graphicDeviceOptions: { + xrCompatible: true + } +}); +``` + +## Best Practices + +- **Engine Lifecycle**: Create engine once, reuse across scenes. Destroy only when application terminates. +- **Scene Organization**: Use meaningful entity hierarchies; group related objects under parent entities. +- **Resource Sharing**: Load resources at engine level to share across multiple scenes efficiently. +- **Performance Monitoring**: Track `time.deltaTime` for frame-rate independent animations and logic. +- **Device Recovery**: Implement device loss handlers for robust web deployment. +- **Memory Management**: Use `pause()` instead of `destroy()` for temporary inactivity to avoid re-initialization costs. diff --git a/docs/scripting/engine.md b/docs/scripting/engine.md new file mode 100644 index 0000000000..0792320fba --- /dev/null +++ b/docs/scripting/engine.md @@ -0,0 +1,758 @@ +# Engine + +Galacean's `Engine` class is the central orchestrator that manages all core systems including rendering, physics, input, resources, and scene management. The Engine coordinates the frame loop, handles device context, provides cross-system communication, and serves as the primary entry point for creating 3D applications. + +## Overview + +The Engine manages multiple interconnected systems: +- **Rendering Pipeline**: Frame-based rendering with post-processing support +- **Resource Management**: Asset loading, caching, and memory management +- **Scene Management**: Multiple scene support with activation control +- **Input System**: Mouse, keyboard, and touch event handling +- **Physics Integration**: Optional physics engine coordination +- **XR Support**: Virtual and augmented reality capabilities +- **Timing Control**: Frame rate management and synchronization + +Each Engine instance is bound to a specific rendering context (Canvas) and provides isolated execution environments for different applications or views. + +## Quick Start + +```ts +import { WebGLEngine } from "@galacean/engine"; + +// Create engine with canvas +const engine = await WebGLEngine.create({ + canvas: "canvas-id" // or HTMLCanvasElement +}); + +// Access core systems +const scene = engine.sceneManager.activeScene; +const resourceManager = engine.resourceManager; +const inputManager = engine.inputManager; + +// Create and start the main loop +const rootEntity = scene.createRootEntity("Root"); +engine.run(); // Start the frame loop + +// Pause/resume as needed +engine.pause(); +engine.resume(); + +// Clean shutdown +engine.destroy(); +``` + +## Engine Creation and Configuration + +```ts +import { WebGLEngine, PhysXPhysics, ColorSpace } from "@galacean/engine"; + +// Basic configuration +const engine = await WebGLEngine.create({ + canvas: "canvas-id", // string ID or HTMLCanvasElement + graphicDeviceOptions: { + antialias: true, + alpha: false, + stencil: true + } +}); + +// Advanced configuration with all options +const engine = await WebGLEngine.create({ + canvas: document.getElementById("canvas"), + colorSpace: ColorSpace.Gamma, // or ColorSpace.Linear + graphicDeviceOptions: { + antialias: true, + alpha: false, + stencil: true, + webGLMode: "auto" // "webgl1", "webgl2", or "auto" + }, + physics: new PhysXPhysics(), // Optional physics engine + input: { + pointerTarget: document // Set touch event listener source + }, + gltf: { + meshOpt: { workerCount: 4 } // GLTF loader configuration + }, + ktx2Loader: { + workerCount: 2 // KTX2 loader configuration + } +}); + +// Initialize physics after creation (alternative approach) +engine.physicsManager.initialize(PhysXPhysics); +``` + +## Core Systems Access + +### Resource Management + +```ts +import { AssetType } from "@galacean/engine"; + +const resourceManager = engine.resourceManager; + +// Load single asset +const texture = await resourceManager.load({ + url: "path/to/texture.jpg", + type: AssetType.Texture2D +}); + +// Load multiple assets +const [texture2D, cubeTexture] = await resourceManager.load([ + { url: "path/to/texture.jpg", type: AssetType.Texture2D }, + { + url: ["px.jpg", "nx.jpg", "py.jpg", "ny.jpg", "pz.jpg", "nz.jpg"], + type: AssetType.TextureCube + } +]); + +// Load GLTF/GLB models +const scene = await resourceManager.load({ + url: "path/to/model.glb", + type: AssetType.GLTF +}); + +// Manage resource lifecycle - force garbage collection +resourceManager.gc(); // Releases unused cached resources +``` + +### Scene Management + +```ts +import { Scene } from "@galacean/engine"; + +const sceneManager = engine.sceneManager; + +// Access default active scene +const activeScene = sceneManager.activeScene; + +// Create and manage multiple scenes +const gameScene = new Scene(engine, "GameScene"); +const menuScene = new Scene(engine, "MenuScene"); + +// Add scenes to engine +sceneManager.addScene(gameScene); +sceneManager.addScene(menuScene); + +// Remove scenes +sceneManager.removeScene(menuScene); + +// Load scene from asset +const loadedScene = await sceneManager.loadScene("path/to/scene.json"); + +// Merge scenes +sceneManager.mergeScenes(menuScene, gameScene); // Merge menuScene into gameScene + +// Access all scenes +const allScenes = sceneManager.scenes; // ReadonlyArray +console.log(`Total scenes: ${allScenes.length}`); + +// Destroy scene (also removes it from engine) +gameScene.destroy(); +``` + +### Input Handling + +```ts +import { Keys } from "@galacean/engine"; + +const inputManager = engine.inputManager; + +// Check keyboard input states +if (inputManager.isKeyHeldDown(Keys.Space)) { + // Space key is currently being held down +} + +if (inputManager.isKeyDown(Keys.Space)) { + // Space key was pressed this frame +} + +if (inputManager.isKeyUp(Keys.Space)) { + // Space key was released this frame +} + +// Access pointer data (supports multiple pointers) +const pointer0 = inputManager.pointers[0]; +if (pointer0) { + console.log("Pointer position:", pointer0.position); + console.log("Pointer delta:", pointer0.deltaPosition); +} + +// Check pointer states +if (inputManager.isPointerDown()) { + // Mouse/touch is currently pressed +} +``` + +## Frame Loop and Timing + +### Frame Rate Control + +```ts +// V-sync based timing (default) +engine.vSyncCount = 1; // 60fps on 60Hz display +engine.vSyncCount = 2; // 30fps on 60Hz display +engine.vSyncCount = 0; // Disable v-sync, use target frame rate + +// Custom frame rate (when v-sync disabled) +engine.targetFrameRate = 30; // Target 30 FPS +engine.targetFrameRate = 120; // Target 120 FPS +engine.targetFrameRate = Number.POSITIVE_INFINITY; // Unlimited + +// Access timing information +const time = engine.time; +console.log("Delta time:", time.deltaTime); +console.log("Total time:", time.totalTime); +console.log("Frame count:", time.frameCount); +``` + +### Manual Frame Control + +```ts +// Manual update (when not using engine.run()) +class CustomGameLoop { + constructor(private engine: Engine) {} + + start(): void { + this.update(); + } + + private update = (): void => { + // Manual frame update + engine.update(); + + // Continue loop + requestAnimationFrame(this.update); + } +} + +// Usage +const gameLoop = new CustomGameLoop(engine); +gameLoop.start(); +``` + +## Engine Lifecycle + +### Running and Control + +```ts +// Start the engine main loop +engine.run(); // Begins automatic frame updates + +// Check engine state +console.log(engine.isPaused); // false when running + +// Pause execution +engine.pause(); // Stops frame updates and input processing + +// Resume execution +engine.resume(); // Resumes from where it was paused + +// Clean shutdown +engine.destroy(); // Releases all resources and stops execution + +// Check if destroyed +console.log(engine.destroyed); // true after destroy() is called +``` + +### Event Handling + +```ts +// Engine lifecycle methods (not events) +engine.run(); // Start the engine +engine.pause(); // Pause execution +engine.resume(); // Resume from pause +engine.destroy(); // Clean shutdown + +// Device context events +engine.on("devicelost", (engine) => { + console.log("Graphics device lost - rendering suspended"); +}); + +engine.on("devicerestored", (engine) => { + console.log("Graphics device restored - rendering resumed"); +}); + +// Check engine state +console.log("Is paused:", engine.isPaused); +console.log("Is destroyed:", engine.destroyed); +``` + +## Post-Processing Pipeline + +```ts +import { PostProcessPass, BloomEffect } from "@galacean/engine"; + +// Add custom post-process pass +class CustomPostProcess extends PostProcessPass { + constructor(engine: Engine) { + super(engine); + // Configure pass + } + + render(context: RenderContext): void { + // Custom post-processing logic + } +} + +// Add to engine +const customPass = new CustomPostProcess(engine); +engine.addPostProcessPass(customPass); + +// Access all post-process passes +const allPasses = engine.postProcessPasses; +console.log(`${allPasses.length} post-process passes registered`); +``` + +## Device Management + +### Graphics Device Control + +```ts +// Simulate device loss (for testing) +engine.forceLoseDevice(); + +// Simulate device restoration (for testing) +engine.forceRestoreDevice(); + +// Handle device events automatically +engine.on("devicelost", () => { + // Engine automatically handles resource cleanup + console.log("Device lost - pausing rendering"); +}); + +engine.on("devicerestored", () => { + // Engine automatically restores resources + console.log("Device restored - resuming rendering"); +}); +``` + +### Memory Management + +```ts +// Clean up unused resources explicitly +const resourceManager = engine.resourceManager; +resourceManager.gc(); + +// Maintain your own bookkeeping for loaded assets +const loadedResources = new Set(); +loadedResources.add("texture.png"); +console.log(`Tracked resources: ${loadedResources.size}`); + +// Check engine state +console.log("Engine running:", !engine.isPaused); +console.log("Total entities:", engine.sceneManager.activeScene.rootEntitiesCount); +``` + +## XR Integration + +```ts +import { WebXRDevice } from "@galacean/engine-xr-webxr"; + +// Create engine with XR support +const engine = await WebGLEngine.create({ + canvas: "canvas", + xrDevice: new WebXRDevice() +}); + +// Access XR manager +const xrManager = engine.xrManager; + +if (xrManager) { + // Check XR availability + const isSupported = await xrManager.isSessionSupported("immersive-vr"); + + if (isSupported) { + // Start XR session + await xrManager.startSession("immersive-vr"); + } +} +``` + +## Performance Optimization + +### Efficient Frame Loop + +```ts +class OptimizedEngine { + private engine: Engine; + private lastUpdateTime = 0; + private updateInterval = 1000 / 30; // 30 FPS for game logic + + constructor(engine: Engine) { + this.engine = engine; + this.setupOptimizedLoop(); + } + + private setupOptimizedLoop(): void { + // Separate render and update frequencies + this.engine.targetFrameRate = 60; // High framerate for smooth rendering + + this.engine.on("update", this.onUpdate); + } + + private onUpdate = (): void => { + const now = performance.now(); + + // Run game logic at lower frequency + if (now - this.lastUpdateTime >= this.updateInterval) { + this.updateGameLogic(); + this.lastUpdateTime = now; + } + } + + private updateGameLogic(): void { + // Heavy game logic runs at 30fps + // while rendering continues at 60fps + } +} +``` + +### Resource Management Strategy + +```ts +class ResourceOptimizer { + constructor(private engine: Engine) { + this.setupResourceStrategy(); + } + + private setupResourceStrategy(): void { + const resourceManager = this.engine.resourceManager; + + // Preload critical resources + this.preloadCriticalAssets(); + + // Set up periodic cleanup + setInterval(() => { + resourceManager.gc(); + }, 30000); // Clean up every 30 seconds + } + + private async preloadCriticalAssets(): Promise { + const resourceManager = this.engine.resourceManager; + + // Load essential assets at startup + const criticalAssets = [ + { url: "textures/ui-atlas.png", type: AssetType.Texture2D }, + { url: "models/player.glb", type: AssetType.GLTF }, + { url: "audio/ambient.mp3", type: AssetType.AudioClip } + ]; + + await Promise.all( + criticalAssets.map(asset => resourceManager.load(asset)) + ); + } +} +``` + +## Advanced Patterns + +### Multi-Engine Applications + +```ts +class MultiEngineManager { + private engines: Map = new Map(); + + async createEngine(id: string, canvasId: string): Promise { + const engine = await WebGLEngine.create({ canvas: canvasId }); + this.engines.set(id, engine); + return engine; + } + + getEngine(id: string): Engine | undefined { + return this.engines.get(id); + } + + pauseAll(): void { + this.engines.forEach(engine => engine.pause()); + } + + resumeAll(): void { + this.engines.forEach(engine => engine.resume()); + } + + destroyAll(): void { + this.engines.forEach(engine => engine.destroy()); + this.engines.clear(); + } +} + +// Usage for multi-viewport applications +const manager = new MultiEngineManager(); +const mainEngine = await manager.createEngine("main", "main-canvas"); +const miniMapEngine = await manager.createEngine("minimap", "minimap-canvas"); +``` + +### Engine Extension Pattern + +```ts +class ExtendedEngine { + private analyticsEnabled = false; + private frameStats = { + averageFPS: 0, + frameCount: 0, + totalTime: 0 + }; + + constructor(private engine: Engine) { + this.setupExtensions(); + } + + private setupExtensions(): void { + // Add analytics + this.setupPerformanceAnalytics(); + + // Add custom managers + this.setupCustomSystems(); + } + + private setupPerformanceAnalytics(): void { + this.engine.on("update", () => { + if (this.analyticsEnabled) { + this.updateFrameStats(); + } + }); + } + + private updateFrameStats(): void { + const time = this.engine.time; + this.frameStats.frameCount++; + this.frameStats.totalTime += time.deltaTime; + this.frameStats.averageFPS = this.frameStats.frameCount / this.frameStats.totalTime; + + // Log performance warnings + if (time.deltaTime > 0.033) { // > 30 FPS + console.warn(`Frame time spike: ${time.deltaTime * 1000}ms`); + } + } + + enableAnalytics(): void { + this.analyticsEnabled = true; + } + + getPerformanceStats(): typeof this.frameStats { + return { ...this.frameStats }; + } +} +``` + +### Error Recovery System + +```ts +class RobustEngine { + private recoveryAttempts = 0; + private maxRecoveryAttempts = 3; + + constructor(private engine: Engine) { + this.setupErrorRecovery(); + } + + private setupErrorRecovery(): void { + this.engine.on("devicelost", this.handleDeviceLost); + this.engine.on("devicerestored", this.handleDeviceRestored); + + // Global error handling + window.addEventListener("error", this.handleGlobalError); + } + + private handleDeviceLost = (): void => { + console.log("Device lost - attempting recovery"); + this.recoveryAttempts++; + + if (this.recoveryAttempts > this.maxRecoveryAttempts) { + this.handleFatalError("Too many device recovery attempts"); + return; + } + + // Pause non-essential systems + this.pauseNonEssentialSystems(); + } + + private handleDeviceRestored = (): void => { + console.log("Device restored - resuming operation"); + this.recoveryAttempts = 0; + + // Resume all systems + this.resumeAllSystems(); + } + + private handleGlobalError = (event: ErrorEvent): void => { + console.error("Global error:", event.error); + + // Attempt graceful degradation + this.attemptGracefulDegradation(); + } + + private handleFatalError(message: string): void { + console.error("Fatal error:", message); + + // Notify user and provide recovery options + this.showErrorDialog(message); + } + + private pauseNonEssentialSystems(): void { + // Pause non-critical components + // Keep essential systems running + } + + private resumeAllSystems(): void { + // Resume all paused systems + } + + private attemptGracefulDegradation(): void { + // Reduce quality settings + // Disable non-essential features + } + + private showErrorDialog(message: string): void { + // Show user-friendly error message + // Provide reload/recovery options + } +} +``` + +## API Reference + +```apidoc +WebGLEngine: + Static Methods: + create(config: WebGLEngineConfiguration): Promise + - Asynchronously create engine instance with configuration. + + Properties: + canvas: WebCanvas + - The rendering canvas associated with this engine instance. + resourceManager: ResourceManager + - Manages asset loading, caching, and memory management. + sceneManager: SceneManager + - Manages scene creation, activation, and lifecycle. + inputManager: InputManager + - Handles mouse, keyboard, and touch input events. + physicsManager: PhysicsManager + - Manages physics engine integration and initialization. + xrManager: XRManager | undefined + - XR/VR manager if XR device was configured during creation. + time: Time + - Provides frame timing information (deltaTime, totalTime, frameCount). + isPaused: boolean + - Whether the engine frame loop is currently paused. + destroyed: boolean + - Whether the engine has been destroyed. + + Frame Control Properties: + vSyncCount: number + - Vertical sync frame interval. 0 = disabled, 1 = 60fps, 2 = 30fps (on 60Hz). + targetFrameRate: number + - Target FPS when vSyncCount = 0. Use Number.POSITIVE_INFINITY for unlimited. + + Methods: + run(): void + - Start the main engine loop with automatic frame updates. + pause(): void + - Pause the engine loop and input processing. + resume(): void + - Resume the engine from paused state. + update(): void + - Manually update one frame (use when not calling run()). + destroy(): void + - Shutdown engine and release all resources. Safe to call during frame. + + Device Management Methods: + forceLoseDevice(): void + - Simulate graphics device loss (for testing). + forceRestoreDevice(): void + - Simulate graphics device restoration (for testing). + + Events: + "devicelost": (engine: Engine) => void + - Fired when graphics device is lost. + "devicerestored": (engine: Engine) => void + - Fired when graphics device is restored. + +WebGLEngineConfiguration: + canvas: string | HTMLCanvasElement | OffscreenCanvas + - Canvas ID (string) or canvas object for rendering. + colorSpace?: ColorSpace + - Color space configuration (Gamma or Linear). + graphicDeviceOptions?: WebGLGraphicDeviceOptions + - Graphics device settings including WebGL mode and context options. + physics?: Physics + - Physics engine instance (e.g., new PhysXPhysics()). + input?: InputConfiguration + - Input system configuration including pointerTarget. + gltf?: GLTFConfiguration + - GLTF loader configuration including meshOpt settings. + ktx2Loader?: KTX2Configuration + - KTX2 loader configuration including worker count. + xrDevice?: XRDevice + - XR device for VR/AR support. +``` + +## Best Practices + +- **Single Engine Per Canvas**: Each canvas should have its own Engine instance +- **Proper Lifecycle Management**: Always call `destroy()` when done to prevent memory leaks +- **Frame Rate Configuration**: Use v-sync when possible for smooth rendering +- **Resource Preloading**: Load critical assets before starting the main loop +- **Error Handling**: Implement device loss/restore handlers for robust applications +- **Performance Monitoring**: Track frame times and implement quality scaling +- **Graceful Shutdown**: Pause or destroy engines when navigating away from pages +- **Memory Management**: Periodically call resource garbage collection in long-running apps + +## Common Issues + +**Engine Won't Start**: Ensure canvas exists and is accessible: +```ts +// Wait for DOM if needed +document.addEventListener("DOMContentLoaded", async () => { + const engine = await WebGLEngine.create({ canvas: "canvas" }); + engine.run(); +}); +``` + +**Performance Issues**: Monitor and optimize frame timing: +```ts +engine.on("update", () => { + if (engine.time.deltaTime > 0.033) { // 30fps threshold + console.warn("Frame time too high:", engine.time.deltaTime); + // Implement quality reduction + } +}); +``` + +**Memory Leaks**: Always destroy engines and clean up references: +```ts +class GameManager { + private engine?: Engine; + + async init(): Promise { + this.engine = await WebGLEngine.create({ canvas: "canvas" }); + } + + cleanup(): void { + if (this.engine && !this.engine.destroyed) { + this.engine.destroy(); + this.engine = undefined; + } + } +} + +// Cleanup on page unload +window.addEventListener("beforeunload", () => { + gameManager.cleanup(); +}); +``` + +**Device Context Loss**: Handle gracefully with automatic recovery: +```ts +engine.on("devicelost", () => { + // Show loading indicator + showLoadingScreen("Restoring graphics..."); +}); + +engine.on("devicerestored", () => { + // Hide loading indicator + hideLoadingScreen(); +}); +``` diff --git a/docs/scripting/entity.md b/docs/scripting/entity.md new file mode 100644 index 0000000000..49960a0232 --- /dev/null +++ b/docs/scripting/entity.md @@ -0,0 +1,233 @@ +# Entity + +Galacean's `Entity` class is the fundamental container that binds components, scripts, and child entities together inside a scene graph. Every entity owns a `Transform` component, tracks its active state across hierarchy/scene boundaries, and coordinates component lifecycles when it is reparented, cloned, or destroyed. + +## Create Entities + +```ts +import { WebGLEngine, Entity, Camera } from "@galacean/engine"; + +const engine = await WebGLEngine.create({ canvas: "canvas" }); +const scene = engine.sceneManager.activeScene; + +const root = scene.createRootEntity("Root"); +const hero = new Entity(engine, "Hero"); +root.addChild(hero); + +// Method 3: Create child entity (recommended) +const weapon = hero.createChild("Weapon"); + +// Method 4: Create entity and add components +const cameraEntity = root.createChild("Camera"); +const camera = cameraEntity.addComponent(Camera); + +// Method 5: Add existing entity to scene as root +const detachedEntity = new Entity(engine, "Detached"); +scene.addRootEntity(detachedEntity); +``` + +- `scene.createRootEntity()` creates and automatically adds entity to scene +- `new Entity()` creates detached entity that needs manual parenting +- `createChild()` is the recommended way to create child entities +- `addComponent()` is the correct way to attach components +- Each entity automatically gets a `Transform` component +- When you call `createChild`, the parent’s transform type is reused so the new entity always has a compatible `Transform` implementation. + +## Hierarchy Operations + +```ts +// Add the same child to a different parent (cross-scene safe) +const enemy = new Entity(engine, "Enemy"); +root.addChild(enemy); + +const bossRoom = scene.createRootEntity("BossRoom"); +bossRoom.addChild(enemy); // handles deactivation/reactivation across scenes + +// Indexed insert +bossRoom.addChild(0, enemy); + +// Remove without destroying +bossRoom.removeChild(enemy); + +// Clear but keep entities alive (they become scene-less) +bossRoom.clearChildren(); +``` + +- `addChild` automatically detaches the target from any previous parent (including root scenes), updates sibling indices, and propagates active state transitions. +- `removeChild` simply sets `child.parent = null`. Destroy the entity explicitly if you no longer need it. +- `clearChildren` walks children from last to first, deactivates them, and removes their scene reference. Use it before pooling or manual cleanup. + +## Search Utilities + +```ts +// Find by name (recursive search) +const weapon = hero.findByName("Weapon"); + +// Find by path (slash-delimited) +const hand = hero.findByPath("Arm/Hand"); + +// Scene-level search (more efficient for deep hierarchies) +const boss = scene.findEntityByPath("parent/child/grandson"); + +// Access children +const firstChild = hero.getChild(0); +const allChildren = hero.children; // ReadonlyArray + +// Find components in subtree +const renderers: MeshRenderer[] = []; +hero.getComponentsIncludeChildren(MeshRenderer, renderers); +``` + +- `findByName` performs depth-first search starting with current entity +- `findByPath` supports slash-delimited traversal with backtracking +- `scene.findEntityByPath` is more efficient for deep scene searches +- `children` property provides readonly access to child entities +- `getChild(index)` provides direct indexed access +- `getComponentsIncludeChildren` recursively collects components across subtree + +## Lifecycle & Activation + +```ts +hero.isActive = false; // disables hierarchy + scene participation +hero.isActive = true; // reactivates using parent/scene state + +if (!hero.isActiveInHierarchy) { + // entity or one of its ancestors is inactive +} + +bossRoom.isActive = false; // cascades to children via ActiveChangeFlag +``` + +- `isActive` toggles local activation; Galacean automatically propagates hierarchy +to components using `ActiveChangeFlag` masks. +- `isActiveInHierarchy` reflects whether all ancestors are active; `isActiveInScene` (internal) tracks scene-level activation. +- Reactivation across scenes triggers a deactivate/activate cycle so render and physics state stay consistent. + +> **Warning:** `removeChild` or setting `parent = null` does **not** destroy components. Call `entity.destroy()` when you need to fully release resources. + +## Transform Helpers + +Each entity owns exactly one `Transform`. Adding another `Transform` replaces the previous instance and silently destroys the old copy. + +```ts +const transform = hero.transform; +transform.setPosition(0, 0, 0); +transform.rotate(new Vector3(0, 45, 0)); + +const flag = hero.registerWorldChangeFlag(); +engine.on("update", () => { + if (flag.flag) { + flag.flag = false; + console.log("Hero moved", hero.transform.worldMatrix); + } +}); +``` + +- `registerWorldChangeFlag` returns a `BoolUpdateFlag` that flips when the world matrix changes, ideal for reactive systems or caching logic. +- Use `Transform` APIs for all spatial updates (`setPosition`, `translate`, `rotate`, `lookAt`, etc.). + +## Component Access + +```ts +import { DirectLight, Camera, Script, MeshRenderer } from "@galacean/engine"; + +// Add components +const light = hero.addComponent(DirectLight); +light.color.set(1, 1, 1); +light.intensity = 1.0; + +// Get single component +const camera = hero.getComponent(Camera); +if (camera) { + camera.fieldOfView = 60; +} + +// Get all components of a type +const scripts: Script[] = []; +hero.getComponents(Script, scripts); + +// Get components from entity and children +const renderers: MeshRenderer[] = []; +hero.getComponentsIncludeChildren(MeshRenderer, renderers); + +// Access entity from component +const entityFromComponent = light.entity; // Returns hero +``` + +- `addComponent` creates, attaches, and activates component if entity is active +- `getComponent` returns first component of specified type or null +- `getComponents` populates provided array with all matching components +- `getComponentsIncludeChildren` searches entity and all descendants +- Components have `entity` property to access their parent entity +- Component `enabled` property controls activation state independently + +## Cloning & Templates + +```ts +const clone = hero.clone(); +clone.name = "HeroClone"; +scene.addRootEntity(clone); +``` + +- `clone()` deep-copies components, scripts, and child entities, preserving active states, layers, and template resource references (`ReferResource`). +- Template entities registered via `_markAsTemplate` keep refer-counted resources aligned across clones. + +## API Reference + +```apidoc +Entity: + Properties: + name: string + - Arbitrary identifier used by `findByName` and editors. + layer: Layer + - Rendering layer mask shared with children created via `createChild`. + parent: Entity | null + - Getter/setter for hierarchy linkage. Setter revalidates scene membership. + children: ReadonlyArray + - Ordered list of direct children. + siblingIndex: number + - Controls draw/update order among siblings. Throws if entity is detached. + scene: Scene | null + - Owning scene or null when detached. + isActive: boolean + - Local activation flag; propagates via `_processActive`/`_processInActive`. + isActiveInHierarchy: boolean + - Readonly view of effective activation considering ancestors. + + Methods: + constructor(engine: Engine, name?: string): Entity + - Creates a detached entity with automatic Transform component. + addComponent(type: new() => T, ...args: any[]): T + - Instantiates, attaches, and activates a component. + getComponent(type: new() => T): T | null + - Returns first component of specified type or null. + getComponents(type: new() => T, results: T[]): void + - Populates provided array with all components of specified type. + getComponentsIncludeChildren(type: new() => T, results: T[]): void + - Recursively collects components from entity and all descendants. + addChild(child: Entity): void + addChild(index: number, child: Entity): void + - Reparents child entity and updates hierarchy. + removeChild(child: Entity): void + - Detaches child without destroying it. + createChild(name?: string): Entity + - Creates and parents a new child entity. + clearChildren(): void + - Removes all children, deactivating them. + clone(): Entity + - Deep clones entity with all components and children. + findByName(name: string): Entity | null + - Depth-first recursive search by name. + findByPath(path: string): Entity | null + - Slash-delimited hierarchical search. + getChild(index: number): Entity | undefined + - Direct indexed access to child entity. + destroy(): void + - Destroys entity and all its components and children. +``` + +## Best Practices + +- Keep entity trees shallow when possible; frequent `findByName` on deep hierarchies is recursive. +- Reuse entities via `clearChildren` and manual pooling to avoid GC churn from repeated construction. +- When moving entities between scenes, always rely on `addChild`/`parent` assignments instead of directly manipulating internal arrays; Galacean handles activation, layer sync, and scene reassignment for you. diff --git a/docs/scripting/environment-probe.md b/docs/scripting/environment-probe.md new file mode 100644 index 0000000000..45c24d0078 --- /dev/null +++ b/docs/scripting/environment-probe.md @@ -0,0 +1,450 @@ +# Environment Probe System + +Galacean's environment probe system provides dynamic environment mapping capabilities for realistic reflections, refractions, and image-based lighting (IBL). The system captures the surrounding environment from specific positions and generates cube textures that can be reused by ambient lighting, skyboxes, or custom materials. + +The environment probe system includes: +- **Probe**: Base class that configures render targets, resolution, and layer masks. +- **CubeProbe**: Captures a 360° cube texture from a world position. +- **IBL Integration**: Hooks for updating the scene's ambient light and sky. +- **Dynamic Updates**: Scripts can control when probes capture to balance quality and cost. + +## Quick Start + +```ts +import { + BackgroundMode, + CubeProbe, + Layer, + MeshRenderer, + PBRMaterial, + PrimitiveMesh, + SkyBoxMaterial, + TextureCube, + WebGLEngine +} from '@galacean/engine'; + +const engine = await WebGLEngine.create({ canvas: 'canvas' }); +const scene = engine.sceneManager.activeScene; + +// Reflective test mesh +const reflectiveEntity = scene.createRootEntity('ReflectiveSphere'); +const renderer = reflectiveEntity.addComponent(MeshRenderer); +renderer.mesh = PrimitiveMesh.createSphere(engine, 0.6); +const reflectiveMaterial = new PBRMaterial(engine); +reflectiveMaterial.metallic = 1.0; +reflectiveMaterial.roughness = 0.1; +renderer.setMaterial(reflectiveMaterial); + +// Probe host entity +const probeEntity = scene.createRootEntity('EnvironmentProbe'); +probeEntity.transform.setPosition(0, 1, 0); +const cubeProbe = probeEntity.addComponent(CubeProbe); +cubeProbe.width = 1024; +cubeProbe.height = 1024; +cubeProbe.probeLayer = Layer.Everything; + +let lastCapture: TextureCube | null = null; +cubeProbe.onTextureChange = (cubeTexture) => { + lastCapture = cubeTexture as TextureCube; // Cache for debugging or baking + + const ambientLight = scene.ambientLight; + ambientLight.specularTexture = cubeTexture; + ambientLight.specularIntensity = 1.0; + + if (scene.background.mode !== BackgroundMode.Sky) { + scene.background.mode = BackgroundMode.Sky; + scene.background.sky.material = new SkyBoxMaterial(engine); + } + const skyMaterial = scene.background.sky.material as SkyBoxMaterial | null; + skyMaterial && (skyMaterial.texture = cubeTexture); +}; + +cubeProbe.enabled = true; +engine.run(); +``` + +`CubeProbe.onTextureChange` fires once per render when the probe is enabled. Cache the latest `TextureCube` if you need it elsewhere (e.g., for baking or debugging). + +## CubeProbe Configuration + +```ts +import { CubeProbe, Layer, Vector3 } from '@galacean/engine'; + +const cubeProbe = probeEntity.addComponent(CubeProbe); + +// Resolution (power-of-two sizes avoid mip artefacts) +cubeProbe.width = 2048; +cubeProbe.height = 2048; + +// Capture masks (bitwise OR the layers you need) +cubeProbe.probeLayer = Layer.Layer0 | Layer.Layer1; + +// Sample position is independent of the entity transform +cubeProbe.position = new Vector3(0, 2, 0); + +// Hardware MSAA (WebGL2 only) +cubeProbe.antiAliasing = 4; + +// Respond when the capture is ready +cubeProbe.onTextureChange = (cubeTexture) => { + console.log('Environment captured', cubeTexture); +}; +``` + +### Probe Positioning + +```ts +// Absolute world position +cubeProbe.position = new Vector3(5, 3, -2); + +// Match an entity transform snapshot +cubeProbe.position.copyFrom(probeEntity.transform.worldPosition); + +// Spawn multiple probes +[ + { name: 'Center', pos: new Vector3(0, 1, 0) }, + { name: 'East', pos: new Vector3(10, 1, 0) }, + { name: 'West', pos: new Vector3(-10, 1, 0) } +].forEach(({ name, pos }) => { + const entity = scene.createRootEntity(name); + const probe = entity.addComponent(CubeProbe); + probe.position = pos; + probe.width = 1024; + probe.height = 1024; +}); +``` + +## Layer-Based Capture + +Define application-specific layers and exclude masks you do not want in the reflection: + +```ts +const LayerGameplay = Layer.Layer0; +const LayerUI = Layer.Layer4; +const LayerPostFX = Layer.Layer5; + +cubeProbe.probeLayer = Layer.Everything & ~LayerUI & ~LayerPostFX; + +const reflectiveObject = scene.createRootEntity('ReflectiveObject'); +reflectiveObject.layer = LayerGameplay; + +const uiElement = scene.createRootEntity('HudCanvas'); +uiElement.layer = LayerUI; +``` + +UI- or post-process entities render normally, but the probe omits them because their bits are removed from the mask. + +## IBL Integration + +### Ambient Lighting + +```ts +import { DiffuseMode } from '@galacean/engine'; + +cubeProbe.onTextureChange = (cubeTexture) => { + const ambientLight = scene.ambientLight; + + ambientLight.specularTexture = cubeTexture; + ambientLight.specularTextureDecodeRGBM = false; + ambientLight.specularIntensity = 1.0; + + // Optionally drive diffuse lighting with spherical harmonics (see below). + ambientLight.diffuseMode = DiffuseMode.SphericalHarmonics; +}; +``` + +The engine's PBR shaders automatically read `scene.ambientLight.specularTexture`, so every material that relies on IBL will pick up the reflection once it is assigned. + +### Material Adjustments + +Because reflections come from the scene ambient light, individual materials typically only need minor tweaks: + +```ts +const reflectiveMaterial = renderer.getMaterial() as PBRMaterial; +reflectiveMaterial.specularIntensity = 0.8; // Per-material reflection intensity +reflectiveMaterial.metallic = 1.0; +reflectiveMaterial.roughness = 0.05; +``` + +If you require unique environment maps per object, clone or create a custom shader that samples your cached `TextureCube`. + +## Dynamic Environment Updates + +Capture frequency has a direct performance cost. This controller script re-enables the probe on a timer and turns it back off after a capture completes: + +```ts +import { CubeProbe, Script, TextureCube } from '@galacean/engine'; + +class ProbeRefreshController extends Script { + private probe!: CubeProbe; + private interval = 1.0; // seconds + private timer = 0; + + onAwake(): void { + this.probe = this.entity.getComponent(CubeProbe)!; + + const previousHandler = this.probe.onTextureChange?.bind(this.probe); + this.probe.onTextureChange = (texture: TextureCube) => { + previousHandler?.(texture); + this.probe.enabled = false; // freeze until the next scheduled capture + }; + + this.probe.enabled = true; // force initial capture + } + + setFrequency(hz: number): void { + this.interval = hz > 0 ? 1 / hz : Number.POSITIVE_INFINITY; + } + + onUpdate(deltaTime: number): void { + this.timer += deltaTime; + if (this.timer >= this.interval) { + this.timer = 0; + if (!this.probe.enabled) { + this.probe.enabled = true; + } + } + } +} + +// Usage +const controller = probeEntity.addComponent(ProbeRefreshController); +controller.setFrequency(0.5); // Capture twice per second +``` + +Attach the script to the same entity that owns the probe. The wrapper preserves any existing `onTextureChange` logic. + +## Performance Optimization + +### Resolution Management + +```ts +function selectProbeResolution(distance: number): number { + if (distance < 10) return 2048; + if (distance < 50) return 1024; + return 512; +} + +const cameraPosition = camera.entity.transform.worldPosition; +const distance = Vector3.distance(cameraPosition, cubeProbe.position); +const value = selectProbeResolution(distance); +cubeProbe.width = cubeProbe.height = value; +``` + +### Update Strategies + +Use dedicated lists so ultra-static probes only capture once: + +```ts +class ProbeManager { + private staticProbes: CubeProbe[] = []; + private dynamicControllers: ProbeRefreshController[] = []; + + addStaticProbe(probe: CubeProbe): void { + probe.onTextureChange = (texture) => { + scene.ambientLight.specularTexture = texture; + probe.enabled = false; + }; + probe.enabled = true; // capture once + this.staticProbes.push(probe); + } + + addDynamicProbe(controller: ProbeRefreshController): void { + this.dynamicControllers.push(controller); + } + + onUpdate(deltaTime: number): void { + this.dynamicControllers.forEach((controller) => controller.onUpdate(deltaTime)); + } +} +``` + +### Memory Management + +```ts +import { CubeProbe, Entity } from '@galacean/engine'; + +const MAX_PROBE_MEMORY = 256 * 1024 * 1024; // 256 MB +let currentBudget = 0; + +function estimateProbeMemory(width: number, height: number): number { + return 6 * width * height * 4 * 1.33; // faces * pixels * RGBA * mip chain +} + +function createProbeWithBudget(host: Entity, width: number, height: number): CubeProbe | null { + const cost = estimateProbeMemory(width, height); + if (currentBudget + cost > MAX_PROBE_MEMORY) { + if (width <= 128) { + console.warn('Probe budget exhausted.'); + return null; + } + return createProbeWithBudget(host, width >> 1, height >> 1); + } + + currentBudget += cost; + const probe = host.addComponent(CubeProbe); + probe.width = width; + probe.height = height; + return probe; +} +``` + +## Spherical Harmonics Integration + +Use spherical harmonics to derive diffuse light from a cube map. The following helper uses a deterministic Fibonacci sampling pattern: + +```ts +import { Color, DiffuseMode, SphericalHarmonics3, TextureCube, Vector3 } from '@galacean/engine'; + +class SHGenerator { + generateFromCubeTexture(cubeTexture: TextureCube, sampleCount = 64): SphericalHarmonics3 { + const sh = new SphericalHarmonics3(); + const directions = this._generateDirections(sampleCount); + const solidAngle = (4 * Math.PI) / sampleCount; + + directions.forEach((direction) => { + const color = this._sampleCubeTexture(cubeTexture, direction); + sh.addLight(direction, color, solidAngle); + }); + + return sh; + } + + private _generateDirections(count: number): Vector3[] { + const directions: Vector3[] = []; + const golden = Math.PI * (3 - Math.sqrt(5)); + + for (let i = 0; i < count; i++) { + const y = 1 - (i / (count - 1)) * 2; + const radius = Math.sqrt(Math.max(0, 1 - y * y)); + const theta = golden * i; + const x = Math.cos(theta) * radius; + const z = Math.sin(theta) * radius; + directions.push(new Vector3(x, y, z)); + } + + return directions; + } + + private _sampleCubeTexture(_cubeTexture: TextureCube, _direction: Vector3): Color { + // GPU read-back is engine specific. Replace this placeholder with platform utilities. + return new Color(0.5, 0.5, 0.5, 1); + } +} + +cubeProbe.onTextureChange = (cubeTexture) => { + const sh = new SHGenerator().generateFromCubeTexture(cubeTexture); + scene.ambientLight.diffuseMode = DiffuseMode.SphericalHarmonics; + scene.ambientLight.diffuseSphericalHarmonics = sh; +}; +``` + +## Multi-Probe Blending + +Cache probe captures and choose the best texture per location. This example blends weights and applies the strongest match to the ambient light before each camera render: + +```ts +import { Camera, CubeProbe, Script, TextureCube, Vector3 } from '@galacean/engine'; + +interface ProbeRecord { + probe: CubeProbe; + position: Vector3; + radius: number; + texture: TextureCube | null; +} + +class ProbeBlendingSystem extends Script { + private records: ProbeRecord[] = []; + + registerProbe(probe: CubeProbe, position: Vector3, radius: number): void { + const record: ProbeRecord = { probe, position: position.clone(), radius, texture: null }; + + const previousHandler = probe.onTextureChange?.bind(probe); + probe.onTextureChange = (texture: TextureCube) => { + record.texture = texture; + previousHandler?.(texture); + }; + + this.records.push(record); + } + + onBeginRender(camera: Camera): void { + const strongest = this._getStrongest(camera.entity.transform.worldPosition); + if (strongest && strongest.texture) { + const ambient = this.scene.ambientLight; + ambient.specularTexture = strongest.texture; + ambient.specularIntensity = strongest.weight; + } + } + + private _getStrongest(worldPosition: Vector3): + | { texture: TextureCube | null; weight: number } + | null { + let best: { texture: TextureCube | null; weight: number } | null = null; + + for (const record of this.records) { + if (!record.texture) continue; + const distance = Vector3.distance(worldPosition, record.position); + if (distance >= record.radius) continue; + + const weight = 1 - distance / record.radius; + if (!best || weight > best.weight) { + best = { texture: record.texture, weight }; + } + } + + return best; + } +} +``` + +For finer blending you can mix the textures into a single cube map offline, or feed weights into a custom shader that samples multiple environment maps. + +## API Reference + +```apidoc +Probe (abstract) + Properties: + probeLayer: Layer - Bit mask that determines which entities are rendered. + width: number - Render target width (default 1024). + height: number - Render target height (default 1024). + antiAliasing: number - MSAA level when supported (default 1). + enabled: boolean - Component enable flag inherited from Script. + + Events: + onTextureChange(texture: Texture): void + Called after a capture. Assign one handler and forward to your own callbacks as needed. + + Methods: + onBeginRender(camera: Camera): void + Internal override; executed automatically when the probe is enabled. + +CubeProbe extends Probe + Properties: + position: Vector3 - World-space capture origin (defaults to [0, 0, 0]). + + Behavior: + - Captures the six faces of a cube map with a 90° field of view. + - Uses RenderTarget auto-mipmap generation so reflections respect roughness. + - Requires you to cache the latest TextureCube via onTextureChange; there is no direct `texture` getter. + +TextureCube + Properties: + width/height: number - Face size in pixels. + mipmapCount: number - Includes generated mip levels. + + Methods: + generateMipmaps(): void + Explicitly rebuilds the mip chain when you update pixel data manually. +``` + +## Best Practices + +- **Resolution**: Start with 1024×1024 captures; push to 2048 for hero assets and drop to 512 or 256 for distant zones. +- **Capture Budget**: Disable probes once a static environment is captured; re-enable only when something changes. +- **Layer Hygiene**: Reserve dedicated Layer bits (e.g., Layer4 for UI) so probes can filter non-physical elements cleanly. +- **Memory Watch**: Track combined face memory (6 × width × height × 4 bytes × mip multiplier) before spawning probes. +- **Placement**: Position probes at locations representative of the surrounding geometry, not inside occluded spaces. +- **Spherical Harmonics**: Precompute SH coefficients when you need high-quality diffuse IBL; reuse the cached result across probes where possible. +- **Blending**: For large scenes, store probe volumes and select the best match per camera or per region before rendering. +- **Validation**: Toggle the probe's cached `TextureCube` onto the skybox or a debug material to verify what was captured. diff --git a/docs/scripting/graphics-2d.md b/docs/scripting/graphics-2d.md new file mode 100644 index 0000000000..3edc29de93 --- /dev/null +++ b/docs/scripting/graphics-2d.md @@ -0,0 +1,696 @@ +# 2D Graphics + +Galacean's 2D graphics system provides comprehensive sprite and text rendering capabilities for 2D games and UI elements. Built on efficient batching and atlas systems, it supports multiple draw modes, advanced text rendering with font management, and sprite masking for complex 2D scenes. + +## Overview + +The 2D graphics system consists of several key components: +- **SpriteRenderer**: Renders sprites with various draw modes and batching optimization +- **Sprite**: Resource that defines texture regions, borders, and atlas mapping +- **TextRenderer**: Advanced text rendering with font support and layout options +- **Font**: Font resource management with atlas generation and character mapping +- **SpriteAtlas**: Texture atlas management for efficient sprite batching +- **Masking System**: SpriteMask for clipping and visual effects + +## Quick Start + +```ts +import { WebGLEngine, Entity, AssetType } from "@galacean/engine"; +import { SpriteRenderer, TextRenderer, Sprite, SpriteDrawMode } from "@galacean/engine"; +import { Vector2, Color } from "@galacean/engine-math"; + +const engine = await WebGLEngine.create({ canvas: "canvas" }); +const scene = engine.sceneManager.activeScene; + +// Create sprite entity +const spriteEntity = scene.createRootEntity("Sprite"); +const spriteRenderer = spriteEntity.addComponent(SpriteRenderer); + +// Load texture and create sprite +const texture = await engine.resourceManager.load({ + url: "path/to/sprite.png", + type: AssetType.Texture2D +}); + +const sprite = new Sprite(engine, texture); +spriteRenderer.sprite = sprite; +spriteRenderer.drawMode = SpriteDrawMode.Simple; + +// Create text entity +const textEntity = scene.createRootEntity("Text"); +const textRenderer = textEntity.addComponent(TextRenderer); +textRenderer.text = "Hello 2D Graphics!"; +textRenderer.fontSize = 32; +textRenderer.color = new Color(1, 1, 1, 1); + +engine.run(); +``` + +## Sprite System + +### Sprite Resource Management + +Sprites define how textures are rendered with support for regions, pivots, and borders: + +```ts +import { Sprite } from "@galacean/engine"; +import { Rect, Vector2, Vector4 } from "@galacean/engine-math"; + +// Basic sprite creation +const sprite = new Sprite(engine, texture); + +// Sprite with custom region (normalized coordinates) +const region = new Rect(0.25, 0.25, 0.5, 0.5); // Use center quarter of texture +const pivot = new Vector2(0.5, 0); // Bottom-center pivot +const sprite = new Sprite(engine, texture, region, pivot); + +// Sprite dimensions +sprite.width = 100; // Custom width in world units +sprite.height = 100; // Custom height in world units + +// Advanced sprite configuration +sprite.region = new Rect(0, 0, 1, 1); // Full texture region +sprite.pivot = new Vector2(0.5, 0.5); // Center pivot +sprite.border = new Vector4(10, 10, 10, 10); // 9-slice borders (left, bottom, right, top) + +// Atlas sprite configuration +sprite.atlasRegion = new Rect(0.1, 0.1, 0.3, 0.3); // Region within atlas +sprite.atlasRegionOffset = new Vector4(0.05, 0.05, 0.05, 0.05); // Padding offset +sprite.atlasRotated = false; // Whether sprite is rotated in atlas +``` + +### SpriteRenderer Component + +SpriteRenderer handles sprite display with various draw modes and optimization features: + +```ts +import { SpriteRenderer, SpriteDrawMode, SpriteTileMode } from "@galacean/engine"; +import { Color } from "@galacean/engine-math"; + +const spriteRenderer = entity.addComponent(SpriteRenderer); + +// Basic sprite configuration +spriteRenderer.sprite = sprite; +spriteRenderer.color = new Color(1, 0.5, 0.5, 0.8); // Tinted red, semi-transparent + +// Size control +spriteRenderer.width = 200; // Override sprite width +spriteRenderer.height = 150; // Override sprite height + +// Flipping +spriteRenderer.flipX = true; // Flip horizontally +spriteRenderer.flipY = false; // No vertical flip + +// Draw modes +spriteRenderer.drawMode = SpriteDrawMode.Simple; // Basic sprite rendering +spriteRenderer.drawMode = SpriteDrawMode.Sliced; // 9-slice scaling +spriteRenderer.drawMode = SpriteDrawMode.Tiled; // Tiled repetition + +// Tiled mode configuration +spriteRenderer.tileMode = SpriteTileMode.Continuous; // Seamless tiling +spriteRenderer.tileMode = SpriteTileMode.Adaptive; // Adaptive tiling +spriteRenderer.tiledAdaptiveThreshold = 0.5; // Stretch threshold for adaptive mode +``` + +### Sprite Draw Modes + +Different draw modes provide various scaling and rendering behaviors: + +```ts +// Simple Mode - Direct texture mapping +spriteRenderer.drawMode = SpriteDrawMode.Simple; +// - Stretches entire sprite to fit dimensions +// - Best for solid backgrounds, simple icons +// - Fastest rendering performance + +// Sliced Mode - 9-slice scaling +spriteRenderer.drawMode = SpriteDrawMode.Sliced; +spriteRenderer.sprite.border = new Vector4(20, 20, 20, 20); // Border sizes in pixels +// - Corners remain unscaled +// - Edges stretch in one direction +// - Center scales in both directions +// - Perfect for UI panels, buttons + +// Tiled Mode - Repetitive rendering +spriteRenderer.drawMode = SpriteDrawMode.Tiled; +spriteRenderer.tileMode = SpriteTileMode.Continuous; +// - Repeats sprite texture to fill area +// - Maintains sprite aspect ratio +// - Great for patterns, backgrounds + +// Adaptive Tiled Mode +spriteRenderer.tileMode = SpriteTileMode.Adaptive; +spriteRenderer.tiledAdaptiveThreshold = 0.4; // 40% stretch threshold +// - Combines stretching and tiling +// - Stretches when close to perfect fit +// - Tiles when stretching would be too extreme +``` + +## Text Rendering System + +### TextRenderer Component + +TextRenderer provides advanced text rendering with layout, styling, and internationalization support: + +```ts +import { TextRenderer, TextHorizontalAlignment, TextVerticalAlignment } from "@galacean/engine"; +import { FontStyle, OverflowMode } from "@galacean/engine"; +import { Color } from "@galacean/engine-math"; + +const textRenderer = entity.addComponent(TextRenderer); + +// Basic text properties +textRenderer.text = "Hello World!"; +textRenderer.fontSize = 24; +textRenderer.color = new Color(0, 0, 0, 1); // Black text + +// Font and styling +textRenderer.font = customFont; // Custom font resource +textRenderer.fontStyle = FontStyle.Bold | FontStyle.Italic; // Combined styles + +// Dimensions and layout +textRenderer.width = 300; // Text container width +textRenderer.height = 100; // Text container height + +// Text alignment +textRenderer.horizontalAlignment = TextHorizontalAlignment.Center; +textRenderer.verticalAlignment = TextVerticalAlignment.Middle; + +// Text wrapping and overflow +textRenderer.enableWrapping = true; // Wrap text to new lines +textRenderer.overflowMode = OverflowMode.Truncate; // Clip overflow text +textRenderer.lineSpacing = 5; // Additional space between lines (pixels) + +// Advanced layout options +textRenderer.horizontalAlignment = TextHorizontalAlignment.Left; +textRenderer.verticalAlignment = TextVerticalAlignment.Top; +textRenderer.overflowMode = OverflowMode.Overflow; // Allow text to extend beyond bounds +``` + +### Font Management + +Font resources manage character atlases and rendering metrics: + +```ts +import { Font, AssetType } from "@galacean/engine"; + +// Load font resource +const font = await engine.resourceManager.load({ + url: "fonts/custom-font.ttf", + type: AssetType.Font +}); + +// Apply to text renderer +textRenderer.font = font; + +// Font supports automatic atlas generation for different sizes and styles +// Each font size/style combination creates a SubFont with its own texture atlas +``` + +### Text Alignment and Layout + +Comprehensive text positioning and alignment options: + +```ts +import { + TextHorizontalAlignment, + TextVerticalAlignment, + OverflowMode +} from "@galacean/engine"; + +// Horizontal alignment options +textRenderer.horizontalAlignment = TextHorizontalAlignment.Left; // Left-aligned +textRenderer.horizontalAlignment = TextHorizontalAlignment.Center; // Centered +textRenderer.horizontalAlignment = TextHorizontalAlignment.Right; // Right-aligned + +// Vertical alignment options +textRenderer.verticalAlignment = TextVerticalAlignment.Top; // Top-aligned +textRenderer.verticalAlignment = TextVerticalAlignment.Center; // Vertically centered +textRenderer.verticalAlignment = TextVerticalAlignment.Bottom; // Bottom-aligned + +// Overflow handling +textRenderer.overflowMode = OverflowMode.Overflow; // Allow text beyond bounds +textRenderer.overflowMode = OverflowMode.Truncate; // Clip text at bounds + +// Wrapping behavior +textRenderer.enableWrapping = true; // Enable line wrapping +textRenderer.lineSpacing = 2; // Extra spacing between lines + +// Dynamic text sizing +textRenderer.width = 0; // Auto-width (no wrapping) +textRenderer.height = 0; // Auto-height (no truncation) +``` + +## Sprite Atlas System + +### SpriteAtlas Management + +Sprite atlases optimize rendering by combining multiple sprites into single textures: + +```ts +import { SpriteAtlas, AssetType } from "@galacean/engine"; + +// Load sprite atlas +const atlas = await engine.resourceManager.load({ + url: "atlases/game-sprites.json", + type: AssetType.SpriteAtlas +}); + +// Get sprites from atlas +const playerSprite = atlas.getSprite("player"); +const enemySprite = atlas.getSprite("enemy"); +const bulletSprite = atlas.getSprite("bullet"); + +// Use atlas sprites +spriteRenderer.sprite = playerSprite; + +// Atlas sprites automatically have correct regions and offsets configured +// This enables efficient batching when multiple atlas sprites are rendered together +``` + +### Font Atlas System + +Font atlases efficiently pack character glyphs for text rendering: + +```ts +import { FontAtlas } from "@galacean/engine"; + +// Font atlases are automatically managed by the Font system +// Each font size/style combination generates a FontAtlas +// Character glyphs are dynamically added to atlases as needed + +// Font atlas management is automatic—each font/size/style builds atlases on demand. +// Use the engine's debugging overlay or custom instrumentation to inspect atlas pages +// when optimizing memory usage (e.g., track `textRenderer.fontAtlas.texture` during development). +``` + +## Masking and Clipping + +### SpriteMask Component + +SpriteMask enables clipping and visual effects for 2D graphics: + +```ts +import { SpriteMask, SpriteMaskInteraction } from "@galacean/engine"; + +// Create mask entity +const maskEntity = scene.createRootEntity("Mask"); +const spriteMask = maskEntity.addComponent(SpriteMask); +spriteMask.sprite = maskSprite; // Sprite defining mask shape + +// Configure masked content +const maskedEntity = scene.createRootEntity("MaskedContent"); +const maskedRenderer = maskedEntity.addComponent(SpriteRenderer); +maskedRenderer.sprite = contentSprite; +maskedRenderer.maskInteraction = SpriteMaskInteraction.VisibleInsideMask; + +// Mask interaction modes +maskedRenderer.maskInteraction = SpriteMaskInteraction.None; // No masking +maskedRenderer.maskInteraction = SpriteMaskInteraction.VisibleInsideMask; // Show inside mask +maskedRenderer.maskInteraction = SpriteMaskInteraction.VisibleOutsideMask; // Show outside mask + +// Mask layers for complex masking setups +spriteMask.maskLayer = SpriteMaskLayer.Layer0; +maskedRenderer.maskLayer = SpriteMaskLayer.Layer0; // Must match for masking to apply +``` + +## Performance Optimization + +### Batching and Draw Calls + +Optimize 2D rendering performance through intelligent batching: + +```ts +// Batching is automatic but can be optimized through: + +// 1. Use same materials for multiple sprites +const sharedMaterial = new Material(engine, spriteShader); +spriteRenderer1.setMaterial(sharedMaterial); +spriteRenderer2.setMaterial(sharedMaterial); +spriteRenderer3.setMaterial(sharedMaterial); + +// 2. Use sprite atlases to combine textures +const atlas = await engine.resourceManager.load({ + url: "game-atlas.json", + type: AssetType.SpriteAtlas +}); + +// All sprites from same atlas can batch together +sprite1 = atlas.getSprite("player"); +sprite2 = atlas.getSprite("enemy"); +sprite3 = atlas.getSprite("powerup"); + +// 3. Group similar 2D objects in the scene hierarchy +const uiGroup = scene.createRootEntity("UI"); +const gameObjectGroup = scene.createRootEntity("GameObjects"); +const backgroundGroup = scene.createRootEntity("Backgrounds"); + +// 4. Use appropriate draw modes for different content +spriteRenderer.drawMode = SpriteDrawMode.Simple; // Fastest for simple sprites +spriteRenderer.drawMode = SpriteDrawMode.Sliced; // Good for UI with borders +spriteRenderer.drawMode = SpriteDrawMode.Tiled; // Efficient for repeating patterns +``` + +### Memory Management + +Efficient resource usage for 2D graphics: + +```ts +// Sprite and font resource pooling +class SpritePool { + private pool: Sprite[] = []; + + getSprite(texture: Texture2D): Sprite { + if (this.pool.length > 0) { + const sprite = this.pool.pop()!; + sprite.texture = texture; + return sprite; + } + return new Sprite(engine, texture); + } + + returnSprite(sprite: Sprite): void { + sprite.texture = null; + this.pool.push(sprite); + } +} + +// Minimize texture memory usage +// - Use power-of-2 texture dimensions when possible +// - Compress textures appropriately for platform +// - Use sprite atlases to reduce texture count +// - Dispose unused textures promptly + +// Font optimization +// - Preload commonly used character sets +// - Use appropriate font sizes (avoid extreme scaling) +// - Share fonts between text renderers when possible +``` + +## Animation and Effects + +### Sprite Animation + +Animate sprites through property changes and atlas switching: + +```ts +// Property-based animation +class SpriteAnimator extends Script { + private time: number = 0; + private spriteRenderer: SpriteRenderer; + + onAwake(): void { + this.spriteRenderer = this.entity.getComponent(SpriteRenderer); + } + + onUpdate(deltaTime: number): void { + this.time += deltaTime; + + // Animate color + const pulse = Math.sin(this.time * 2) * 0.5 + 0.5; + this.spriteRenderer.color.set(1, pulse, pulse, 1); + + // Animate scale + const scale = 1 + Math.sin(this.time) * 0.1; + this.entity.transform.setScale(scale, scale, 1); + + // Animate rotation + this.entity.transform.rotate(0, 0, deltaTime * 45); + } +} + +// Atlas-based sprite animation +class SpriteSequenceAnimator extends Script { + public sprites: Sprite[] = []; + public frameRate: number = 10; + + private currentFrame: number = 0; + private frameTime: number = 0; + private spriteRenderer: SpriteRenderer; + + onAwake(): void { + this.spriteRenderer = this.entity.getComponent(SpriteRenderer); + } + + onUpdate(deltaTime: number): void { + this.frameTime += deltaTime; + const frameDuration = 1 / this.frameRate; + + if (this.frameTime >= frameDuration) { + this.frameTime -= frameDuration; + this.currentFrame = (this.currentFrame + 1) % this.sprites.length; + this.spriteRenderer.sprite = this.sprites[this.currentFrame]; + } + } +} +``` + +### Text Effects + +Create dynamic text effects and animations: + +```ts +// Text typing effect +class TypewriterEffect extends Script { + public fullText: string = ""; + public typingSpeed: number = 50; // Characters per second + + private textRenderer: TextRenderer; + private currentLength: number = 0; + private timer: number = 0; + + onAwake(): void { + this.textRenderer = this.entity.getComponent(TextRenderer); + this.fullText = this.textRenderer.text; + this.textRenderer.text = ""; + } + + onUpdate(deltaTime: number): void { + if (this.currentLength < this.fullText.length) { + this.timer += deltaTime; + const charactersToShow = Math.floor(this.timer * this.typingSpeed); + this.currentLength = Math.min(charactersToShow, this.fullText.length); + this.textRenderer.text = this.fullText.substring(0, this.currentLength); + } + } +} + +// Text color wave effect +class TextWaveEffect extends Script { + private textRenderer: TextRenderer; + private time: number = 0; + + onAwake(): void { + this.textRenderer = this.entity.getComponent(TextRenderer); + } + + onUpdate(deltaTime: number): void { + this.time += deltaTime; + + // Create wave effect with color + const hue = (Math.sin(this.time * 2) + 1) * 0.5; + this.textRenderer.color.set(hue, 1 - hue, 0.5, 1); + + // Oscillate font size + const sizeVariation = Math.sin(this.time * 3) * 2; + this.textRenderer.fontSize = 24 + sizeVariation; + } +} +``` + +## API Reference + +```apidoc +SpriteRenderer: + Properties: + sprite: Sprite + - The sprite resource to render. + drawMode: SpriteDrawMode + - Simple, Sliced, or Tiled rendering mode. + color: Color + - Tint color and transparency for the sprite. + width: number + - Render width in world coordinates. Overrides sprite width if set. + height: number + - Render height in world coordinates. Overrides sprite height if set. + flipX: boolean + - Flip sprite horizontally. + flipY: boolean + - Flip sprite vertically. + tileMode: SpriteTileMode + - Continuous or Adaptive tiling mode (for Tiled draw mode). + tiledAdaptiveThreshold: number + - Stretch threshold for adaptive tiling (0-1 range). + maskInteraction: SpriteMaskInteraction + - How sprite interacts with sprite masks. + maskLayer: SpriteMaskLayer + - Mask layer for sprite mask interactions. + +Sprite: + Properties: + texture: Texture2D + - Source texture for the sprite. + width: number + - Sprite width in world coordinates. + height: number + - Sprite height in world coordinates. + region: Rect + - Texture region in normalized coordinates (0-1). + pivot: Vector2 + - Pivot point in normalized coordinates (0-1). + border: Vector4 + - 9-slice borders (left, bottom, right, top) in normalized coordinates. + atlasRegion: Rect + - Region within atlas texture in normalized coordinates. + atlasRegionOffset: Vector4 + - Padding offset within atlas region. + atlasRotated: boolean + - Whether sprite is rotated 90° in atlas. + + Methods: + clone(): Sprite + - Create a copy of the sprite. + +TextRenderer: + Properties: + text: string + - Text content to display. + font: Font + - Font resource for text rendering. + fontSize: number + - Font size in points. + fontStyle: FontStyle + - Font style flags (Bold, Italic, etc.). + color: Color + - Text color and transparency. + width: number + - Text container width in world coordinates. + height: number + - Text container height in world coordinates. + horizontalAlignment: TextHorizontalAlignment + - Left, Center, or Right alignment. + verticalAlignment: TextVerticalAlignment + - Top, Center, or Bottom alignment. + enableWrapping: boolean + - Whether text wraps to new lines. + overflowMode: OverflowMode + - How to handle text exceeding bounds. + lineSpacing: number + - Additional spacing between lines in pixels. + maskInteraction: SpriteMaskInteraction + - How text interacts with sprite masks. + maskLayer: SpriteMaskLayer + - Mask layer for sprite mask interactions. + +SpriteMask: + Properties: + sprite: Sprite + - Sprite defining the mask shape. + maskLayer: SpriteMaskLayer + - Layer this mask operates on. + +Font: + Properties: + name: string + - Font family name. + fontAtlas: FontAtlas + - Current atlas used for rendering. Inspect for debugging or preloading glyphs. +``` + +## Best Practices + +- **Atlas Usage**: Use sprite atlases to minimize draw calls and improve batching efficiency +- **Draw Mode Selection**: Choose appropriate draw modes - Simple for performance, Sliced for UI, Tiled for patterns +- **Font Management**: Limit font sizes and styles to reduce memory usage and atlas generation +- **Masking Performance**: Use sprite masks sparingly as they can break batching +- **Color Optimization**: Avoid frequent color changes that might break batching +- **Text Layout**: Pre-calculate text dimensions when possible rather than relying on auto-sizing +- **Resource Cleanup**: Properly dispose of sprites and fonts when no longer needed +- **Texture Formats**: Use appropriate texture compression for sprites and font atlases + +## Common Patterns + +### Animated Sprite Button + +```ts +class AnimatedButton extends Script { + private spriteRenderer: SpriteRenderer; + private originalScale: Vector3; + private isPressed: boolean = false; + + onAwake(): void { + this.spriteRenderer = this.entity.getComponent(SpriteRenderer); + this.originalScale = this.entity.transform.scale.clone(); + } + + onPointerDown(): void { + this.isPressed = true; + this.entity.transform.setScale( + this.originalScale.x * 0.95, + this.originalScale.y * 0.95, + this.originalScale.z + ); + this.spriteRenderer.color.set(0.8, 0.8, 0.8, 1); + } + + onPointerUp(): void { + this.isPressed = false; + this.entity.transform.scale = this.originalScale; + this.spriteRenderer.color.set(1, 1, 1, 1); + } +} +``` + +### Multi-Language Text System + +```ts +class LocalizedText extends Script { + public textKey: string = ""; + private textRenderer: TextRenderer; + + onAwake(): void { + this.textRenderer = this.entity.getComponent(TextRenderer); + this.updateText(); + } + + updateText(): void { + const localizedString = Localization.getString(this.textKey); + this.textRenderer.text = localizedString; + + // Adjust font if needed for different languages + const currentLanguage = Localization.getCurrentLanguage(); + if (currentLanguage === "chinese") { + this.textRenderer.font = chineseFont; + } else { + this.textRenderer.font = englishFont; + } + } +} +``` + +### Dynamic Sprite Loading + +```ts +class DynamicSpriteLoader extends Script { + public spriteUrl: string = ""; + private spriteRenderer: SpriteRenderer; + + onAwake(): void { + this.spriteRenderer = this.entity.getComponent(SpriteRenderer); + } + + async loadSprite(url: string): Promise { + try { + const texture = await this.engine.resourceManager.load({ + url: url, + type: AssetType.Texture2D + }); + + const sprite = new Sprite(this.engine, texture); + this.spriteRenderer.sprite = sprite; + } catch (error) { + console.error("Failed to load sprite:", error); + } + } +} +``` diff --git a/docs/scripting/layer-system.md b/docs/scripting/layer-system.md new file mode 100644 index 0000000000..c9f9d09e57 --- /dev/null +++ b/docs/scripting/layer-system.md @@ -0,0 +1,148 @@ +# Layer System + +Galacean exposes 32 bitmask layers for selectively rendering, lighting, and colliding entities. Layers are powers of two defined in `Layer.ts` and can be combined through bitwise operators. Cameras and lights accept masks (multiple layers), while entities and colliders belong to a single layer at a time. + +## Quick Reference + +```ts +import { + Camera, + CameraClearFlags, + DirectLight, + Entity, + Layer, + WebGLEngine +} from '@galacean/engine'; + +const engine = await WebGLEngine.create({ canvas }); +const scene = engine.sceneManager.activeScene; + +// Entities default to Layer.Layer0 +const player = scene.createRootEntity('Player'); +const enemies = scene.createRootEntity('Enemies'); +const uiRoot = scene.createRootEntity('UI'); + +enemies.layer = Layer.Layer1; +uiRoot.layer = Layer.Layer2; + +const cameraEntity = scene.createRootEntity('MainCamera'); +const camera = cameraEntity.addComponent(Camera); + +// Render gameplay but skip UI +camera.cullingMask = Layer.Layer0 | Layer.Layer1; + +// Secondary camera renders UI only +const uiCamera = scene.createRootEntity('UICamera').addComponent(Camera); +uiCamera.cullingMask = Layer.Layer2; +uiCamera.clearFlags = CameraClearFlags.Depth; +uiCamera.isOrthographic = true; +``` + +`Layer.Everything` represents all bits set; `Layer.Nothing` is zero. + +## Layer Enum + +The enum defines 32 single-bit values: + +- `Layer.Layer0` … `Layer.Layer31` +- `Layer.Everything` (0xffffffff) +- `Layer.Nothing` (0x0) + +Combine masks with `|`, exclude using `& ~layer`, test membership with `(mask & layer) !== 0`. + +## Entity Layers vs Masks + +Entities store one layer: + +```ts +entity.layer = Layer.Layer5; +``` + +Cameras, lights, physics queries, and other mask consumers accept any combination of bits: + +```ts +camera.cullingMask = Layer.Layer0 | Layer.Layer4; +directLight.cullingMask = Layer.Layer0 | Layer.Layer1; +``` + +Because entities are single-layer, you can use bitwise operations on masks without worrying about ambiguous overlaps. + +## Rendering and Lighting + +### Camera culling + +- `Camera.cullingMask` controls which layers render. +- `Camera.postProcessMask` gates which cameras feed post-processing. +- UI setups typically use two stacked cameras: one for 3D content, one for `Layer.Layer2` (or similar). + +### Light culling + +All light types expose `cullingMask`. The internal `LightManager` splits the 32-bit mask into two 16-bit integers, so large masks are supported. + +```ts +const lamp = scene.createRootEntity('Lamp').addComponent(DirectLight); +lamp.cullingMask = Layer.Layer0 | Layer.Layer3; // illuminate gameplay + props +``` + +### Probe capture + +`Probe.probeLayer` works exactly like a camera mask. Keep UI or debugging layers out by masking them off. + +## Physics Collision Layers + +Colliders also have a single layer. Assigning a combined mask throws because the engine verifies the value is a pure power-of-two. + +```ts +const collider = entity.addComponent(DynamicCollider); +collider.collisionLayer = Layer.Layer4; // one layer only +``` + +Control which layers interact through the `PhysicsScene` collision matrix. Internally, the physics backend stores a 32 × 32 boolean table. + +```ts +const physics = scene.physics; + +// Disable player <-> enemy collisions +physics.setColliderLayerCollision(Layer.Layer0, Layer.Layer1, false); + +const canHit = physics.getColliderLayerCollision(Layer.Layer0, Layer.Layer1); +console.log('Player collides with enemies?', canHit); +``` + +All physics queries support an optional layer mask to filter results: + +```ts +const ray = new Ray(origin, direction); +const hit = physics.raycast(ray, 100, Layer.Layer0 | Layer.Layer1, hitResult); +``` + +`raycast`, `boxCast`, `sphereCast`, etc. default to `Layer.Everything` when no mask is provided. + +## Layer Utilities + +```ts +function addLayer(mask: Layer, layer: Layer): Layer { + return mask | layer; +} + +function removeLayer(mask: Layer, layer: Layer): Layer { + return mask & ~layer; +} + +function hasLayer(mask: Layer, layer: Layer): boolean { + return (mask & layer) !== 0; +} +``` + +Cache frequently used masks to avoid recomputing bit expressions every frame. + +## Recommended Conventions + +- Reserve low indices (`Layer0`, `Layer1`, `Layer2`) for core systems (world, enemies, UI). Document the mapping for your team. +- Dedicate a layer to editor/debug helpers so you can toggle them off globally. +- Leave a block of higher bits for temporary/spawned items—masks are cheap to tweak at runtime. +- When authoring assets in the editor, give prefabs meaningful default layers to reduce errors at import time. +- For physics: build a collision matrix chart early in production, update alongside code, and enforce rules in prefab validation. +- Always use bitwise operations, not addition or subtraction, when composing masks. + +By keeping entity layers distinct and masks expressive, you can scale render, lighting, and physics selection logic without allocating additional marker components or tags. diff --git a/docs/scripting/lighting.md b/docs/scripting/lighting.md new file mode 100644 index 0000000000..385f864150 --- /dev/null +++ b/docs/scripting/lighting.md @@ -0,0 +1,894 @@ +# Lighting + +Galacean's lighting system provides comprehensive support for realistic 3D illumination including directional lights, point lights, spot lights, ambient lighting, and advanced features like shadows, ambient occlusion, and physically-based rendering. The system supports dynamic lighting with real-time shadows, image-based lighting with spherical harmonics, and performance optimization through light culling and batching. + +## Overview + +The lighting system consists of several core components: +- **Light Types**: DirectLight (sun/moon), PointLight (bulbs), SpotLight (flashlights), AmbientLight (environment) +- **Shadow System**: Real-time shadow mapping with configurable quality and bias settings +- **Ambient Occlusion**: Screen-space ambient occlusion (SSAO) for enhanced depth perception +- **Light Management**: Automatic light culling, batching, and performance optimization +- **Image-Based Lighting**: Environment maps and spherical harmonics for realistic ambient lighting + +The system integrates seamlessly with materials and shaders to provide physically-accurate lighting calculations, supporting both forward and deferred rendering pipelines. + +## Quick Start + +```ts +import { WebGLEngine, DirectLight, PointLight, SpotLight, AmbientLight, DiffuseMode, ShadowType } from "@galacean/engine"; + +const engine = await WebGLEngine.create({ canvas: "canvas" }); +const scene = engine.sceneManager.activeScene; + +// Create sun light (directional light) +const sunEntity = scene.createRootEntity("Sun"); +const sunLight = sunEntity.addComponent(DirectLight); +sunLight.color.set(1, 0.95, 0.8, 1); // Warm sunlight +sunLight.shadowType = ShadowType.SoftLow; +sunEntity.transform.setRotation(-30, 30, 0); + +// Create point light for indoor lighting +const lampEntity = scene.createRootEntity("Lamp"); +const pointLight = lampEntity.addComponent(PointLight); +pointLight.color.set(1, 0.8, 0.6, 1); // Warm indoor light +pointLight.distance = 10; +lampEntity.transform.setPosition(5, 3, 0); + +// Create spot light for focused illumination +const flashlightEntity = scene.createRootEntity("Flashlight"); +const spotLight = flashlightEntity.addComponent(SpotLight); +spotLight.color.set(1, 1, 1, 1); +spotLight.angle = Math.PI / 6; // Cone angle in radians (30 degrees) +spotLight.penumbra = Math.PI / 12; // Soft edge falloff in radians +spotLight.distance = 15; +flashlightEntity.transform.setPosition(0, 2, 5); +flashlightEntity.transform.lookAt(new Vector3(0, 0, 0)); + +// Setup ambient lighting +const ambientLight = scene.ambientLight; +ambientLight.diffuseMode = DiffuseMode.SolidColor; +ambientLight.diffuseSolidColor.set(0.2, 0.2, 0.3, 1); +ambientLight.diffuseIntensity = 0.4; +``` + +## Light Types + +### DirectLight (Directional Light) + +Directional lights simulate distant light sources like the sun or moon, providing parallel rays across the entire scene: + +```ts +// Create directional light +const sunEntity = scene.createRootEntity("Sun"); +const directLight = sunEntity.addComponent(DirectLight); + +// Configure light properties +directLight.color.set(1, 0.9, 0.7, 1); // Sunset color +directLight.shadowType = ShadowType.SoftHigh; // High quality shadows + +// Control light direction through entity rotation +sunEntity.transform.setRotation(-45, 45, 0); // Angled sunlight + +// Shadow configuration +directLight.shadowBias = 0.005; // Prevent shadow acne +directLight.shadowNormalBias = 0.02; // Reduce self-shadowing +directLight.shadowNearPlane = 1; // Near plane for shadow camera + +// Access computed direction +const lightDirection = directLight.direction; +const reverseDirection = directLight.reverseDirection; // Opposite direction +``` + +### PointLight + +Point lights emit light uniformly in all directions from a single point, like light bulbs: + +```ts +// Create point light +const bulbEntity = scene.createRootEntity("Bulb"); +const pointLight = bulbEntity.addComponent(PointLight); + +// Configure light properties +pointLight.color.set(1, 0.8, 0.6, 1); // Warm white +pointLight.distance = 8; // Light attenuation distance + +// Position the light +bulbEntity.transform.setPosition(2, 3, 1); + +// Access computed position +const lightPosition = pointLight.position; + +// Shadow configuration (if supported) +pointLight.shadowType = ShadowType.SoftLow; +pointLight.shadowBias = 0.01; +pointLight.shadowNormalBias = 0.05; +``` + +### SpotLight + +Spot lights emit light in a cone shape, perfect for flashlights, car headlights, or stage lighting: + +```ts +// Create spot light +const spotEntity = scene.createRootEntity("Spotlight"); +const spotLight = spotEntity.addComponent(SpotLight); + +// Configure cone properties +spotLight.color.set(1, 1, 0.9, 1); +spotLight.angle = Math.PI / 4; // Cone angle in radians (45 degrees) +spotLight.penumbra = Math.PI / 12; // Soft edge falloff in radians +spotLight.distance = 20; // Maximum range + +// Position and orient the light +spotEntity.transform.setPosition(0, 5, 5); +spotEntity.transform.lookAt(new Vector3(0, 0, 0)); + +// Access computed properties +const lightPosition = spotLight.position; +const lightDirection = spotLight.direction; +const reverseDirection = spotLight.reverseDirection; + +// Shadow configuration +spotLight.shadowType = ShadowType.SoftMedium; +spotLight.shadowBias = 0.003; +spotLight.shadowNormalBias = 0.02; + +// Advanced cone control +console.log(`Inner cone: ${spotLight.angle} radians (${spotLight.angle * 180 / Math.PI} degrees)`); +console.log(`Penumbra falloff: ${spotLight.penumbra} radians (${spotLight.penumbra * 180 / Math.PI} degrees)`); +``` + +### Light Properties and Shadow Configuration + +All light types share common properties for shadows and culling: + +```ts +// Common light properties +light.cullingMask = Layer.Everything; // Which layers to illuminate +light.shadowType = ShadowType.SoftHigh; // Shadow quality + +// Shadow bias settings (prevent shadow artifacts) +light.shadowBias = 0.005; // Depth bias +light.shadowNormalBias = 0.02; // Normal-based bias +light.shadowNearPlane = 0.1; // Shadow camera near plane + +// Shadow strength (how dark shadows appear) +light.shadowStrength = 0.8; // 0 = no shadows, 1 = full shadows + +// Access light's view matrix (for advanced shadow techniques) +const viewMatrix = light.viewMatrix; +const inverseViewMatrix = light.inverseViewMatrix; +``` + +## Ambient Lighting + +Ambient lighting provides global illumination that affects all surfaces, creating realistic environmental lighting: + +### Solid Color Ambient Light + +```ts +// Access scene's ambient light +const ambientLight = scene.ambientLight; + +// Set to solid color mode +ambientLight.diffuseMode = DiffuseMode.SolidColor; +ambientLight.diffuseSolidColor.set(0.3, 0.3, 0.4, 1); // Cool ambient +ambientLight.diffuseIntensity = 0.5; + +// Specular reflection control +ambientLight.specularTexture = environmentCubeMap; +ambientLight.specularIntensity = 1.0; +ambientLight.specularTextureDecodeRGBM = false; // Set to true for RGBM encoded textures +``` + +### Image-Based Lighting with Spherical Harmonics + +```ts +// Set to spherical harmonics mode for realistic ambient lighting +ambientLight.diffuseMode = DiffuseMode.SphericalHarmonics; + +// Create spherical harmonics from environment map +const sphericalHarmonics = new SphericalHarmonics3(); +// ... populate spherical harmonics coefficients ... + +ambientLight.diffuseSphericalHarmonics = sphericalHarmonics; +ambientLight.diffuseIntensity = 1.2; + +// Use environment cube map for specular reflections +ambientLight.specularTexture = environmentCubeMap; +ambientLight.specularIntensity = 0.8; + +// Enable RGBM decoding for HDR environment maps +ambientLight.specularTextureDecodeRGBM = true; +``` + +### Environment Map Setup + +```ts +// Create environment cube map for reflections +const envTexture = new TextureCube(engine, 512, TextureFormat.R8G8B8); + +// Load environment faces +const faceImages = [ + 'env_right.jpg', 'env_left.jpg', // +X, -X + 'env_top.jpg', 'env_bottom.jpg', // +Y, -Y + 'env_front.jpg', 'env_back.jpg' // +Z, -Z +]; + +Promise.all(faceImages.map(loadImage)).then(images => { + images.forEach((image, index) => { + envTexture.setImageSource(image, index as TextureCubeFace); + }); + envTexture.generateMipmaps(); + + // Apply to ambient lighting + ambientLight.specularTexture = envTexture; +}); + +// Precompute spherical harmonics for diffuse lighting +const sphericalHarmonics = SphericalHarmonics3.preComputeFromCubeMap(envTexture); +ambientLight.diffuseSphericalHarmonics = sphericalHarmonics; +``` + +## Shadow System + +The shadow system provides real-time shadow mapping with configurable quality levels: + +### Shadow Types and Quality + +```ts +import { ShadowType } from "@galacean/engine"; + +// Available shadow types +light.shadowType = ShadowType.None; // No shadows +light.shadowType = ShadowType.Hard; // Hard shadows (fastest) +light.shadowType = ShadowType.SoftLow; // Soft shadows (low quality) +light.shadowType = ShadowType.SoftMedium; // Soft shadows (medium quality) +light.shadowType = ShadowType.SoftHigh; // Soft shadows (high quality) + +// Configure shadow resolution (engine-wide setting) +engine.shadowManager.shadowMapSize = 2048; // Higher = better quality, worse performance + +// Shadow distance for directional lights +engine.shadowManager.maxDistance = 100; // Maximum shadow distance +``` + +### Shadow Bias Configuration + +```ts +// Prevent shadow acne and light leaking +light.shadowBias = 0.005; // Depth offset to prevent shadow acne +light.shadowNormalBias = 0.02; // Normal-based offset for curved surfaces +light.shadowNearPlane = 0.1; // Near plane for shadow camera + +// Shadow strength controls darkness +light.shadowStrength = 0.8; // 0.0 = no shadow, 1.0 = full black + +// Directional light specific settings +if (light instanceof DirectLight) { + light.shadowNearPlaneOffset = 1; // Additional near plane offset +} +``` + +### Shadow Casting and Receiving + +```ts +// Control which objects cast shadows +const meshRenderer = entity.getComponent(MeshRenderer); +meshRenderer.castShadows = true; // This object casts shadows +meshRenderer.receiveShadows = true; // This object receives shadows + +// Layer-based shadow culling +light.cullingMask = Layer.Default | Layer.Environment; // Only affect these layers + +// Material shadow configuration +const material = new PBRMaterial(engine); +material.receiveShadows = true; // Material-level shadow receiving +``` + +## Ambient Occlusion + +Screen-space ambient occlusion (SSAO) enhances depth perception and realism: + +### Basic SSAO Setup + +```ts +// Access ambient occlusion from scene +const ambientOcclusion = scene.ambientOcclusion; + +// Enable ambient occlusion +ambientOcclusion.enabled = true; +ambientOcclusion.quality = AmbientOcclusionQuality.Medium; + +// Configure SSAO parameters +ambientOcclusion.radius = 0.5; // Sampling radius in world units +ambientOcclusion.intensity = 1.0; // AO effect strength +ambientOcclusion.power = 2.0; // Contrast enhancement +ambientOcclusion.bias = 0.025; // Depth bias to prevent artifacts +ambientOcclusion.minHorizonAngle = 15; // Minimum angle for occlusion (degrees) + +// Bilateral blur settings for noise reduction +ambientOcclusion.bilateralThreshold = 0.1; // Edge preservation threshold +``` + +### SSAO Quality Levels + +```ts +import { AmbientOcclusionQuality } from "@galacean/engine"; + +// Available quality levels +ambientOcclusion.quality = AmbientOcclusionQuality.Low; // 4 samples +ambientOcclusion.quality = AmbientOcclusionQuality.Medium; // 8 samples +ambientOcclusion.quality = AmbientOcclusionQuality.High; // 16 samples +ambientOcclusion.quality = AmbientOcclusionQuality.Ultra; // 32 samples + +// Custom SSAO configuration per quality level +class SSAOManager extends Script { + setupSSAO(quality: 'mobile' | 'desktop' | 'high-end'): void { + const ao = this.scene.ambientOcclusion; + + switch (quality) { + case 'mobile': + ao.quality = AmbientOcclusionQuality.Low; + ao.radius = 0.3; + ao.intensity = 0.8; + break; + + case 'desktop': + ao.quality = AmbientOcclusionQuality.Medium; + ao.radius = 0.5; + ao.intensity = 1.0; + break; + + case 'high-end': + ao.quality = AmbientOcclusionQuality.High; + ao.radius = 0.8; + ao.intensity = 1.2; + break; + } + } +} +``` + +## Light Management + +The light manager handles automatic culling, batching, and optimization: + +### Light Inventory Helpers + +```ts +import { Scene, Entity, Component, DirectLight, PointLight, SpotLight } from "@galacean/engine"; + +// Collect lights using public component APIs +const collectLights = (entity: Entity, type: new (...args: any[]) => T, out: T[]): void => { + entity.getComponents(type, out); + for (const child of entity.children) { + collectLights(child, type, out); + } +}; + +const gatherLights = (scene: Scene, type: new (...args: any[]) => T): T[] => { + const result: T[] = []; + for (const root of scene.rootEntities) { + collectLights(root, type, result); + } + return result; +}; + +const directionalLights = gatherLights(scene, DirectLight); +const pointLights = gatherLights(scene, PointLight); +const spotLights = gatherLights(scene, SpotLight); + +console.log(`Directional lights: ${directionalLights.length}`); +console.log(`Point lights: ${pointLights.length}`); +console.log(`Spot lights: ${spotLights.length}`); + +// Access the designated sun light (if any) +const sunlight = scene.sun ?? directionalLights[0] ?? null; +``` + +### Dynamic Light Management + +```ts +class DynamicLightManager extends Script { + private lights: PointLight[] = []; + private maxActiveLights = 8; + + onUpdate(): void { + const cameraEntity = this.scene.findEntityByName("MainCamera"); + if (!cameraEntity) return; + const camera = cameraEntity.getComponent(Camera); + if (!camera) return; + const cameraPos = camera.entity.transform.worldPosition; + + // Sort lights by distance to camera + this.lights.sort((a, b) => { + const distA = Vector3.distance(a.position, cameraPos); + const distB = Vector3.distance(b.position, cameraPos); + return distA - distB; + }); + + // Enable only closest lights + this.lights.forEach((light, index) => { + light.entity.isActive = index < this.maxActiveLights; + }); + } + + addLight(light: PointLight): void { + this.lights.push(light); + } + + removeLight(light: PointLight): void { + const index = this.lights.indexOf(light); + if (index > -1) { + this.lights.splice(index, 1); + } + } +} +``` + +### Light Culling by Layers + +```ts +// Setup layer-based lighting +const playerLayer = Layer.Layer1; +const environmentLayer = Layer.Layer2; +const uiLayer = Layer.Layer3; + +// Main scene lighting affects player and environment +const sunLight = sunEntity.getComponent(DirectLight); +sunLight.cullingMask = playerLayer | environmentLayer; + +// UI lighting only affects UI elements +const uiLight = uiEntity.getComponent(DirectLight); +uiLight.cullingMask = uiLayer; + +// Point light for indoor areas only +const indoorLight = lampEntity.getComponent(PointLight); +indoorLight.cullingMask = playerLayer; // Only affect player objects +``` + +## Advanced Lighting Techniques + +### Day-Night Cycle + +```ts +class DayNightCycle extends Script { + private sunLight: DirectLight; + private moonLight: DirectLight; + private timeOfDay = 0; // 0-24 hours + + onAwake(): void { + this.sunLight = this.scene.findEntityByName("Sun").getComponent(DirectLight); + this.moonLight = this.scene.findEntityByName("Moon").getComponent(DirectLight); + } + + onUpdate(deltaTime: number): void { + this.timeOfDay += deltaTime * 0.1; // 10x speed + if (this.timeOfDay >= 24) this.timeOfDay = 0; + + this.updateSunMoon(); + this.updateAmbientLight(); + } + + private updateSunMoon(): void { + const sunAngle = (this.timeOfDay - 6) * 15; // Sun rises at 6 AM + const moonAngle = sunAngle + 180; + + // Update sun + this.sunLight.entity.transform.setRotation(sunAngle, 30, 0); + this.sunLight.entity.isActive = sunAngle > -30 && sunAngle < 210; + + // Update moon + this.moonLight.entity.transform.setRotation(moonAngle, 30, 0); + this.moonLight.entity.isActive = !this.sunLight.entity.isActive; + + // Adjust colors based on time + if (this.timeOfDay >= 5 && this.timeOfDay <= 7) { + // Sunrise - warm orange + this.sunLight.color.set(1, 0.6, 0.3, 1); + } else if (this.timeOfDay >= 17 && this.timeOfDay <= 19) { + // Sunset - warm red + this.sunLight.color.set(1, 0.4, 0.2, 1); + } else { + // Midday - neutral white + this.sunLight.color.set(1, 0.95, 0.8, 1); + } + } + + private updateAmbientLight(): void { + const ambientLight = this.scene.ambientLight; + + if (this.timeOfDay >= 6 && this.timeOfDay <= 18) { + // Daytime - blue sky ambient + ambientLight.diffuseSolidColor.set(0.4, 0.6, 1.0, 1); + ambientLight.diffuseIntensity = 0.8; + } else { + // Nighttime - dark blue ambient + ambientLight.diffuseSolidColor.set(0.1, 0.15, 0.3, 1); + ambientLight.diffuseIntensity = 0.3; + } + } +} +``` + +### Light Probes for Baked Lighting + +```ts +class LightProbeSystem extends Script { + private probePositions: Vector3[] = []; + private sphericalHarmonics: SphericalHarmonics3[] = []; + + onAwake(): void { + this.setupProbeGrid(); + this.bakeProbes(); + } + + private setupProbeGrid(): void { + // Create 3D grid of light probe positions + for (let x = -10; x <= 10; x += 5) { + for (let y = 0; y <= 10; y += 5) { + for (let z = -10; z <= 10; z += 5) { + this.probePositions.push(new Vector3(x, y, z)); + } + } + } + } + + private bakeProbes(): void { + // Simulate baking process (in real implementation, use raytracing) + this.probePositions.forEach(position => { + const sh = new SphericalHarmonics3(); + + // Sample environment in all directions from this position + // This is simplified - real implementation would raytrace + const environmentContribution = this.sampleEnvironment(position); + sh.addAmbientLight(environmentContribution); + + this.sphericalHarmonics.push(sh); + }); + } + + getProbeAtPosition(worldPos: Vector3): SphericalHarmonics3 { + // Find closest probe and interpolate + let closestIndex = 0; + let closestDistance = Infinity; + + this.probePositions.forEach((probePos, index) => { + const distance = Vector3.distance(worldPos, probePos); + if (distance < closestDistance) { + closestDistance = distance; + closestIndex = index; + } + }); + + return this.sphericalHarmonics[closestIndex]; + } + + private sampleEnvironment(position: Vector3): Color { + // Simplified environment sampling + const height = position.y / 10; // Normalize height + return new Color(0.3 + height * 0.4, 0.4 + height * 0.3, 0.6 + height * 0.2, 1); + } +} +``` + +### Volumetric Lighting + +```ts +class VolumetricLighting extends Script { + private volumetricMaterial: Material; + private lightShaftQuad: Entity; + + onAwake(): void { + this.createVolumetricQuad(); + this.setupVolumetricMaterial(); + } + + private createVolumetricQuad(): void { + this.lightShaftQuad = this.scene.createRootEntity("VolumetricLighting"); + const meshRenderer = this.lightShaftQuad.addComponent(MeshRenderer); + meshRenderer.mesh = PrimitiveMesh.createPlane(this.engine, 20, 20); + meshRenderer.material = this.volumetricMaterial; + } + + private setupVolumetricMaterial(): void { + // Create custom volumetric lighting shader + const volumetricShader = Shader.create("VolumetricLighting", + this.getVolumetricVertexShader(), + this.getVolumetricFragmentShader() + ); + + this.volumetricMaterial = new Material(this.engine, volumetricShader); + this.volumetricMaterial.isTransparent = true; + this.volumetricMaterial.blendMode = BlendMode.Additive; + } + + updateVolumetrics(lightPosition: Vector3, lightDirection: Vector3): void { + this.volumetricMaterial.shaderData.setVector3("lightPosition", lightPosition); + this.volumetricMaterial.shaderData.setVector3("lightDirection", lightDirection); + this.volumetricMaterial.shaderData.setFloat("scatteringStrength", 0.8); + this.volumetricMaterial.shaderData.setFloat("attenuationStrength", 0.5); + } + + private getVolumetricVertexShader(): string { + return ` + attribute vec3 POSITION; + attribute vec2 TEXCOORD_0; + + uniform mat4 camera_VPMat; + uniform mat4 renderer_ModelMat; + + varying vec2 v_uv; + varying vec3 v_worldPos; + + void main() { + vec4 worldPos = renderer_ModelMat * vec4(POSITION, 1.0); + v_worldPos = worldPos.xyz; + v_uv = TEXCOORD_0; + gl_Position = camera_VPMat * worldPos; + } + `; + } + + private getVolumetricFragmentShader(): string { + return ` + precision mediump float; + + uniform vec3 lightPosition; + uniform vec3 lightDirection; + uniform float scatteringStrength; + uniform float attenuationStrength; + uniform vec3 camera_Position; + + varying vec2 v_uv; + varying vec3 v_worldPos; + + void main() { + vec3 viewDir = normalize(camera_Position - v_worldPos); + vec3 lightDir = normalize(lightPosition - v_worldPos); + + float lightDistance = length(lightPosition - v_worldPos); + float attenuation = 1.0 / (1.0 + attenuationStrength * lightDistance); + + float scattering = pow(max(dot(viewDir, lightDir), 0.0), 4.0); + + vec3 color = vec3(1.0, 0.9, 0.7) * scattering * scatteringStrength * attenuation; + gl_FragColor = vec4(color, scattering * 0.1); + } + `; + } +} +``` + +## Performance Optimization + +### Light Batching and Culling + +```ts +class LightOptimizer extends Script { + private lightGroups: Map = new Map(); + private cullingDistance = 50; + + onUpdate(): void { + const camera = this.scene.findEntityByName("MainCamera").getComponent(Camera); + this.performFrustumCulling(camera); + this.updateLightLOD(camera); + } + + private performFrustumCulling(camera: Camera): void { + const cameraPos = camera.entity.transform.worldPosition; + + // Get all point lights in scene + const allLights = this.scene.findEntitiesWithTag("PointLight"); + + allLights.forEach(lightEntity => { + const pointLight = lightEntity.getComponent(PointLight); + const lightPos = lightEntity.transform.worldPosition; + const distance = Vector3.distance(cameraPos, lightPos); + + // Cull lights beyond maximum distance + const shouldCull = distance > this.cullingDistance; + lightEntity.isActive = !shouldCull; + + // Reduce light range for distant lights + if (!shouldCull && distance > 20) { + pointLight.distance = Math.max(1, pointLight.distance * 0.5); + } + }); + } + + private updateLightLOD(camera: Camera): void { + const cameraPos = camera.entity.transform.worldPosition; + + this.lightGroups.forEach((lights, groupName) => { + lights.forEach(light => { + const distance = Vector3.distance(cameraPos, light.position); + + // Adjust shadow quality based on distance + if (distance < 10) { + light.shadowType = ShadowType.SoftHigh; + } else if (distance < 25) { + light.shadowType = ShadowType.SoftMedium; + } else if (distance < 40) { + light.shadowType = ShadowType.SoftLow; + } else { + light.shadowType = ShadowType.None; + } + }); + }); + } + + groupLights(lights: PointLight[], groupName: string): void { + this.lightGroups.set(groupName, lights); + } +} +``` + + +## API Reference + +```apidoc +Light (Base Class): + Properties: + cullingMask: Layer + - Layer mask for selective lighting. + shadowType: ShadowType + - Shadow quality level (None, Hard, SoftLow, SoftMedium, SoftHigh). + shadowBias: number + - Depth bias to prevent shadow acne. + shadowNormalBias: number + - Normal-based bias for curved surfaces. + shadowNearPlane: number + - Near plane distance for shadow camera. + shadowStrength: number + - Shadow darkness (0-1, where 0 = no shadow, 1 = full black). + color: Color + - Light color and intensity. + + Methods: + viewMatrix: Matrix4 + - Get light's view matrix for shadow calculations. + inverseViewMatrix: Matrix4 + - Get inverse view matrix for light-space transformations. + +DirectLight extends Light: + Properties: + shadowNearPlaneOffset: number + - Additional near plane offset for shadow camera. + direction: Vector3 + - Light direction in world space (read-only). + reverseDirection: Vector3 + - Opposite of light direction (read-only). + +PointLight extends Light: + Properties: + distance: number + - Light attenuation distance. + position: Vector3 + - Light position in world space (read-only). + +SpotLight extends Light: + Properties: + distance: number + - Light attenuation distance. + angle: number + - Cone angle in degrees (0-90). + penumbra: number + - Soft edge falloff ratio (0-1). + position: Vector3 + - Light position in world space (read-only). + direction: Vector3 + - Light direction in world space (read-only). + reverseDirection: Vector3 + - Opposite of light direction (read-only). + +AmbientLight: + Properties: + diffuseMode: DiffuseMode + - Ambient light mode (SolidColor or SphericalHarmonics). + diffuseSolidColor: Color + - Solid color for ambient lighting. + diffuseIntensity: number + - Ambient light intensity multiplier. + diffuseSphericalHarmonics: SphericalHarmonics3 + - Spherical harmonics coefficients for environment lighting. + specularTexture: TextureCube + - Environment cube map for specular reflections. + specularIntensity: number + - Specular reflection intensity. + specularTextureDecodeRGBM: boolean + - Whether to decode RGBM format for HDR textures. + +AmbientOcclusion: + Properties: + enabled: boolean + - Whether ambient occlusion is active. + quality: AmbientOcclusionQuality + - SSAO quality level (Low, Medium, High, Ultra). + radius: number + - Sampling radius in world units. + intensity: number + - AO effect strength. + power: number + - Contrast enhancement factor. + bias: number + - Depth bias to prevent artifacts. + minHorizonAngle: number + - Minimum angle for occlusion detection (degrees). + bilateralThreshold: number + - Edge preservation threshold for blur. + +Enums: + ShadowType: + - None: No shadows + - Hard: Sharp shadows (fastest) + - SoftLow: Soft shadows (4 samples) + - SoftMedium: Soft shadows (9 samples) + - SoftHigh: Soft shadows (16 samples) + + DiffuseMode: + - SolidColor: Uniform ambient color + - SphericalHarmonics: Environment-based ambient lighting + + AmbientOcclusionQuality: + - Low: 4 samples per pixel + - Medium: 8 samples per pixel + - High: 16 samples per pixel + - Ultra: 32 samples per pixel +``` + +## Best Practices + +- **Light Optimization**: Limit the number of dynamic lights per scene (8-16 for mobile, 32+ for desktop) +- **Shadow Configuration**: Use appropriate shadow types based on importance (SoftHigh for hero objects, SoftLow for background) +- **Ambient Lighting**: Use spherical harmonics for realistic environment lighting instead of flat ambient color +- **Layer Culling**: Use culling masks to prevent lights from affecting unnecessary objects +- **Distance Culling**: Disable distant lights or reduce their quality to improve performance +- **Shadow Bias**: Adjust shadow bias values to prevent both shadow acne and light leaking +- **SSAO Parameters**: Tune SSAO settings per platform - use lower quality on mobile devices +- **Light Probes**: Pre-bake lighting information for static environments to reduce runtime calculations +- **Dynamic Shadows**: Reserve real-time shadows for important objects, use baked shadows for static geometry +- **HDR Workflow**: Use HDR environment maps and proper tone mapping for realistic lighting + +## Common Issues + +**Shadow Acne**: Adjust shadow bias settings to prevent self-shadowing artifacts: +```ts +// Increase depth bias for surfaces prone to shadow acne +light.shadowBias = 0.01; +light.shadowNormalBias = 0.05; +``` + +**Light Leaking**: Reduce shadow bias if light appears to leak through surfaces: +```ts +// Reduce bias but increase normal bias +light.shadowBias = 0.001; +light.shadowNormalBias = 0.02; +``` + +**Performance Issues**: Monitor light count and shadow complexity: +```ts +// Implement light LOD system +class PerformanceMonitor extends Script { + onUpdate(): void { + const pointLights = gatherLights(this.scene, PointLight); + const spotLights = gatherLights(this.scene, SpotLight); + const directionalLights = gatherLights(this.scene, DirectLight); + const lightCount = pointLights.length + spotLights.length + directionalLights.length; + + if (lightCount > 16) { + console.warn(`Too many lights: ${lightCount}`); + } + } +} +``` + +**SSAO Artifacts**: Adjust SSAO parameters to reduce noise and banding: +```ts +// Reduce artifacts with proper parameter tuning +ambientOcclusion.radius = 0.3; // Smaller radius for tighter contact shadows +ambientOcclusion.bias = 0.02; // Prevent surface banding +ambientOcclusion.bilateralThreshold = 0.05; // Preserve fine details +``` diff --git a/docs/scripting/material.md b/docs/scripting/material.md new file mode 100644 index 0000000000..fc330314f8 --- /dev/null +++ b/docs/scripting/material.md @@ -0,0 +1,814 @@ +# Material + +Galacean's `Material` class is the foundation for all material systems in the engine, defining how surfaces appear when rendered. Materials combine shaders with properties, textures, and render states to control every aspect of an object's visual appearance including colors, lighting, transparency, and surface details. The material system supports both simple artistic workflows and advanced technical rendering techniques. + +## Overview + +The Material system provides comprehensive surface definition capabilities: + +- **Shader Integration**: Seamless binding of shaders with material properties and textures +- **Property Management**: Type-safe shader property binding with automatic validation +- **Render State Control**: Fine-grained control over blending, depth testing, culling, and render queues +- **Texture Mapping**: Support for diffuse, normal, specular, emissive, and custom texture maps +- **Material Variants**: Multiple material types for different rendering approaches (PBR, Blinn-Phong, Unlit) +- **Instance Management**: Efficient material instancing for per-object customization +- **Resource Management**: Automatic reference counting and memory management + +Materials work closely with Renderers to define the final appearance of 3D objects in the scene. + +## Quick Start + +```ts +import { WebGLEngine, BlinnPhongMaterial, PBRMaterial, UnlitMaterial } from "@galacean/engine"; + +const engine = await WebGLEngine.create({ canvas: "canvas" }); + +// Create different material types +const blinnPhongMaterial = new BlinnPhongMaterial(engine); +const pbrMaterial = new PBRMaterial(engine); +const unlitMaterial = new UnlitMaterial(engine); + +// Configure basic properties +blinnPhongMaterial.baseColor.set(1, 0, 0, 1); // Red color +blinnPhongMaterial.specularColor.set(0.5, 0.5, 0.5, 1); // Gray specular +blinnPhongMaterial.shininess = 32; + +// Load and assign textures +const diffuseTexture = await engine.resourceManager.load({ + url: "textures/brick-diffuse.jpg", + type: AssetType.Texture2D +}); +blinnPhongMaterial.baseTexture = diffuseTexture; + +// Apply to renderer +const meshRenderer = entity.getComponent(MeshRenderer); +meshRenderer.setMaterial(blinnPhongMaterial); +``` + +## Material Types + +### Base Material Class + +All materials inherit from the base `Material` class which provides fundamental functionality: + +```ts +// Create custom material with specific shader +const customShader = Shader.create("CustomShader", vertexSource, fragmentSource); +const customMaterial = new Material(engine, customShader); + +// Access core material properties +console.log("Material name:", customMaterial.name); +console.log("Shader:", customMaterial.shader.name); +console.log("Render states count:", customMaterial.renderStates.length); + +// Access shader data for custom properties +customMaterial.shaderData.setFloat("customIntensity", 2.0); +customMaterial.shaderData.setColor("customTint", new Color(1, 0.5, 0, 1)); +customMaterial.shaderData.setTexture("customTexture", myTexture); +``` + +### Material Render States + +All materials provide render state control for transparency and culling: + +```ts +// Transparency control through render states +material.renderState.renderQueueType = RenderQueueType.Transparent; +material.renderState.depthState.writeEnabled = false; +material.renderState.blendState.enabled = true; + +// Configure alpha blending +const blendState = material.renderState.blendState; +const target = blendState.targetBlendState; +target.sourceColorBlendFactor = BlendFactor.SourceAlpha; +target.destinationColorBlendFactor = BlendFactor.OneMinusSourceAlpha; +target.colorBlendOperation = BlendOperation.Add; +``` + +### Blinn-Phong Material + +Traditional lighting model suitable for stylized and artistic rendering: + +```ts +const blinnPhongMaterial = new BlinnPhongMaterial(engine); + +// Basic surface properties +blinnPhongMaterial.baseColor.set(0.8, 0.2, 0.2, 1.0); // Diffuse color +blinnPhongMaterial.specularColor.set(1.0, 1.0, 1.0, 1.0); // Specular color +blinnPhongMaterial.shininess = 64; // Specular power (higher = more focused) + +// Texture maps +blinnPhongMaterial.baseTexture = diffuseTexture; +blinnPhongMaterial.specularTexture = specularTexture; +blinnPhongMaterial.normalTexture = normalTexture; +blinnPhongMaterial.emissiveTexture = emissiveTexture; + +// Texture properties +blinnPhongMaterial.normalIntensity = 1.0; // Normal map strength +blinnPhongMaterial.emissiveColor.set(0.1, 0.1, 0.1, 1.0); // Emissive glow + +// UV tiling and offset +blinnPhongMaterial.tilingOffset.set(2, 2, 0, 0); // Tile 2x2, no offset +``` + +### PBR Material + +Physically-based rendering for realistic materials: + +```ts +const pbrMaterial = new PBRMaterial(engine); + +// PBR workflow properties +pbrMaterial.baseColor.set(0.7, 0.7, 0.7, 1.0); +pbrMaterial.metallic = 0.0; // 0 = dielectric, 1 = metallic +pbrMaterial.roughness = 0.5; // 0 = mirror-like, 1 = completely rough + +// PBR texture maps +pbrMaterial.baseTexture = albedoTexture; +pbrMaterial.roughnessMetallicTexture = metallicRoughnessTexture; +pbrMaterial.normalTexture = normalTexture; +pbrMaterial.occlusionTexture = occlusionTexture; +pbrMaterial.emissiveTexture = emissiveTexture; + +// Advanced PBR properties +pbrMaterial.clearCoat = 0.0; // Clear coat layer intensity +pbrMaterial.clearCoatRoughness = 0.1; // Clear coat roughness +pbrMaterial.clearCoatTexture = clearCoatTexture; +pbrMaterial.clearCoatRoughnessTexture = clearCoatRoughnessTexture; +pbrMaterial.clearCoatNormalTexture = clearCoatNormalTexture; + +// Sheen properties (for fabric-like materials) +pbrMaterial.sheenColor.set(0.1, 0.1, 0.1, 1.0); +pbrMaterial.sheenColorTexture = sheenColorTexture; +pbrMaterial.sheenRoughness = 0.5; +pbrMaterial.sheenRoughnessTexture = sheenRoughnessTexture; + +// Anisotropy properties (for brushed metal, hair) +pbrMaterial.anisotropy = 0.0; // 0 = disabled +pbrMaterial.anisotropyRotation = 0.0; +pbrMaterial.anisotropyTexture = anisotropyTexture; + +// Transmission properties (for glass, liquids) +pbrMaterial.transmission = 0.0; // 0 = opaque, 1 = fully transparent +pbrMaterial.transmissionTexture = transmissionTexture; + +// Refraction properties (requires transmission > 0) +pbrMaterial.attenuationColor.set(1, 1, 1, 1); // Absorption color +pbrMaterial.attenuationDistance = 0.0; // Attenuation distance +pbrMaterial.thickness = 0.0; // Refraction thickness +pbrMaterial.thicknessTexture = thicknessTexture; +pbrMaterial.refractionMode = RefractionMode.Sphere; // or RefractionMode.Planar +``` + +### Unlit Material + +For objects that don't require lighting calculations: + +```ts +const unlitMaterial = new UnlitMaterial(engine); + +// Simple color and texture +unlitMaterial.baseColor.set(1, 1, 1, 1); +unlitMaterial.baseTexture = uiTexture; + +// Perfect for UI elements, effects, and stylized objects +unlitMaterial.tilingOffset.set(1, 1, 0, 0); +``` + +## Shader Properties and Data + +### Setting Shader Properties + +Materials provide type-safe access to shader properties: + +```ts +const material = new BlinnPhongMaterial(engine); +const shaderData = material.shaderData; + +// Scalar values +shaderData.setFloat("customIntensity", 2.5); +shaderData.setInt("tileCount", 4); +shaderData.setBool("enableEffect", true); + +// Vector values +shaderData.setVector2("uvScale", new Vector2(2, 2)); +shaderData.setVector3("worldOffset", new Vector3(0, 1, 0)); +shaderData.setVector4("tintColor", new Vector4(1, 0.5, 0.2, 1)); + +// Color values (Vector4 with automatic conversion) +shaderData.setColor("glowColor", new Color(1, 0, 0, 1)); + +// Matrix values +const transformMatrix = new Matrix(); +Matrix.translation(new Vector3(1, 0, 0), transformMatrix); +shaderData.setMatrix("customTransform", transformMatrix); + +// Texture values +shaderData.setTexture("diffuseMap", diffuseTexture); +shaderData.setTexture("normalMap", normalTexture); + +// Texture arrays +shaderData.setTextureArray("textureMaps", [tex1, tex2, tex3]); +``` + +### Shader Macros + +Enable or disable shader features using macros: + +```ts +const shaderData = material.shaderData; + +// Enable features +shaderData.enableMacro("USE_NORMAL_MAP"); +shaderData.enableMacro("USE_SPECULAR_MAP"); +shaderData.enableMacro("USE_VERTEX_COLOR"); + +// Disable features +shaderData.disableMacro("USE_SHADOW_MAP"); + +// Check macro state +if (shaderData.hasMacro("USE_NORMAL_MAP")) { + console.log("Normal mapping is enabled"); +} + +// Macros with values +shaderData.enableMacro(ShaderMacro.getByName("POINT_LIGHT_COUNT", "4")); +``` + +## Render States and Blending + +### Transparency and Blending + +```ts +const material = new BlinnPhongMaterial(engine); + +// Enable transparency +material.isTransparent = true; + +// Set blend mode +material.blendMode = BlendMode.Normal; // Standard alpha blending +material.blendMode = BlendMode.Additive; // Additive blending for effects + +// Advanced per-pass transparency control +material.setIsTransparent(0, true); // Enable transparency for pass 0 +material.setBlendMode(0, BlendMode.Additive); + +// Alpha testing (for cutout materials) +material.alphaCutoff = 0.5; // Discard pixels with alpha < 0.5 +material.isTransparent = false; // Use alpha test instead of blending +``` + +### Face Culling + +```ts +// Control which faces are rendered +material.renderFace = RenderFace.Front; // Default: only front faces +material.renderFace = RenderFace.Back; // Only back faces +material.renderFace = RenderFace.Double; // Both front and back faces + +// Useful for: +// - Front: Solid objects (most common) +// - Back: Inside of spheres, rooms +// - Double: Vegetation, cloth, transparent objects +``` + +### Custom Render States + +```ts +// Access individual render states for each shader pass +const renderState = material.renderStates[0]; // First pass + +// Depth state +renderState.depthState.enabled = true; +renderState.depthState.writeEnabled = true; +renderState.depthState.compareFunction = CompareFunction.Less; + +// Stencil state +renderState.stencilState.enabled = true; +renderState.stencilState.referenceValue = 1; +renderState.stencilState.compareFunctionFront = CompareFunction.Equal; + +// Raster state +renderState.rasterState.cullMode = CullMode.Back; +renderState.rasterState.fillMode = FillMode.Solid; + +// Render queue +renderState.renderQueueType = RenderQueueType.Transparent; +``` + +## Texture Management + +### Loading and Assigning Textures + +```ts +// Load textures +const diffuseTexture = await engine.resourceManager.load({ + url: "textures/brick_diffuse.jpg", + type: AssetType.Texture2D +}); + +const normalTexture = await engine.resourceManager.load({ + url: "textures/brick_normal.jpg", + type: AssetType.Texture2D +}); + +// Assign to material +const material = new BlinnPhongMaterial(engine); +material.baseTexture = diffuseTexture; +material.normalTexture = normalTexture; + +// Configure texture properties +material.tilingOffset.set(2, 2, 0.1, 0.1); // Scale 2x2, offset by 0.1 +material.normalIntensity = 1.5; // Increase normal map effect +``` + +### Texture Coordinates and Tiling + +```ts +const material = new PBRMaterial(engine); + +// UV tiling and offset (Vector4: tilingX, tilingY, offsetX, offsetY) +material.tilingOffset.set( + 3.0, 3.0, // Tile texture 3 times in both directions + 0.5, 0.5 // Offset by half a tile +); + +// For animated textures +class AnimatedTexture extends Script { + private material: PBRMaterial; + private scrollSpeed = 1.0; + + onUpdate(deltaTime: number): void { + const offset = this.engine.time.totalTime * this.scrollSpeed; + this.material.tilingOffset.set(1, 1, offset % 1, 0); + } +} +``` + +### Multiple Texture Maps + +```ts +const pbrMaterial = new PBRMaterial(engine); + +// Standard PBR texture set +pbrMaterial.baseTexture = albedoTexture; // Diffuse color +pbrMaterial.metallicRoughnessTexture = mrTexture; // Metallic(B) + Roughness(G) +pbrMaterial.normalTexture = normalTexture; // Normal map (tangent space) +pbrMaterial.occlusionTexture = aoTexture; // Ambient occlusion +pbrMaterial.emissiveTexture = emissiveTexture; // Emissive glow + +// Configure texture properties +pbrMaterial.normalIntensity = 1.0; // Normal map strength +pbrMaterial.occlusionIntensity = 1.0; // AO intensity +pbrMaterial.emissiveColor.set(1, 1, 1, 1); // Emissive color multiplier +``` + +## Material Instancing and Cloning + +### Creating Material Instances + +```ts +// Create base material +const baseMaterial = new BlinnPhongMaterial(engine); +baseMaterial.baseTexture = sharedTexture; +baseMaterial.normalTexture = sharedNormalMap; + +// Clone for customization +const redMaterial = baseMaterial.clone(); +redMaterial.baseColor.set(1, 0, 0, 1); // Red variant + +const blueMaterial = baseMaterial.clone(); +blueMaterial.baseColor.set(0, 0, 1, 1); // Blue variant + +// Apply to different objects +redRenderer.setMaterial(redMaterial); +blueRenderer.setMaterial(blueMaterial); +``` + +### Efficient Material Sharing + +```ts +// Share base material across multiple objects +const sharedMaterial = new PBRMaterial(engine); +sharedMaterial.baseTexture = commonTexture; + +// Use renderer's instance materials for per-object properties +for (let i = 0; i < entities.length; i++) { + const renderer = entities[i].getComponent(MeshRenderer); + renderer.setMaterial(sharedMaterial); + + // Get instance material for customization + const instanceMaterial = renderer.getInstanceMaterial(); + instanceMaterial.baseColor.set( + Math.random(), + Math.random(), + Math.random(), + 1 + ); +} +``` + +### Material Variants + +```ts +class MaterialVariantManager { + private baseMaterial: PBRMaterial; + private variants: Map = new Map(); + + constructor(engine: Engine, baseMaterial: PBRMaterial) { + this.baseMaterial = baseMaterial; + } + + createVariant(name: string, modifications: (material: PBRMaterial) => void): PBRMaterial { + const variant = this.baseMaterial.clone(); + variant.name = `${this.baseMaterial.name}_${name}`; + modifications(variant); + this.variants.set(name, variant); + return variant; + } + + getVariant(name: string): PBRMaterial | null { + return this.variants.get(name) || null; + } +} + +// Usage +const variantManager = new MaterialVariantManager(engine, basePBRMaterial); + +const damagedVariant = variantManager.createVariant("damaged", (material) => { + material.roughness = 0.8; // More rough when damaged + material.baseColor.set(0.6, 0.4, 0.3, 1); // Darker, rustier color +}); + +const newVariant = variantManager.createVariant("new", (material) => { + material.roughness = 0.2; // Shinier when new + material.metallic = 0.8; // More metallic +}); +``` + +## Advanced Material Techniques + +### Dynamic Material Properties + +```ts +class DynamicMaterial extends Script { + private material: BlinnPhongMaterial; + private animationSpeed = 2.0; + + onAwake(): void { + const renderer = this.entity.getComponent(MeshRenderer); + this.material = renderer.getInstanceMaterial() as BlinnPhongMaterial; + } + + onUpdate(deltaTime: number): void { + const time = this.engine.time.totalTime; + + // Animate emissive color + const intensity = (Math.sin(time * this.animationSpeed) + 1) * 0.5; + this.material.emissiveColor.set(intensity, intensity * 0.5, 0, 1); + + // Animate UV offset + const offset = time * 0.1; + this.material.tilingOffset.set(1, 1, offset % 1, 0); + + // Animate shininess + this.material.shininess = 16 + Math.sin(time) * 8; + } +} +``` + +### LOD Material System + +```ts +class LODMaterialSystem extends Script { + private materials: Material[] = []; + private lodDistances: number[] = [10, 50, 100]; + private renderer: MeshRenderer; + + onAwake(): void { + this.renderer = this.entity.getComponent(MeshRenderer); + this.setupLODMaterials(); + } + + private setupLODMaterials(): void { + // High detail - PBR with all texture maps + const highDetailMaterial = new PBRMaterial(this.engine); + highDetailMaterial.baseTexture = highResAlbedo; + highDetailMaterial.normalTexture = highResNormal; + highDetailMaterial.metallicRoughnessTexture = highResMR; + + // Medium detail - PBR with fewer textures + const mediumDetailMaterial = new PBRMaterial(this.engine); + mediumDetailMaterial.baseTexture = mediumResAlbedo; + mediumDetailMaterial.normalTexture = mediumResNormal; + + // Low detail - Unlit for distant objects + const lowDetailMaterial = new UnlitMaterial(this.engine); + lowDetailMaterial.baseTexture = lowResAlbedo; + + this.materials = [highDetailMaterial, mediumDetailMaterial, lowDetailMaterial]; + } + + onUpdate(): void { + const camera = this.scene.findCamera(); + if (!camera) return; + + const distance = Vector3.distance( + this.entity.transform.worldPosition, + camera.entity.transform.worldPosition + ); + + const lodLevel = this.calculateLODLevel(distance); + const currentMaterial = this.renderer.getMaterial(); + + if (currentMaterial !== this.materials[lodLevel]) { + this.renderer.setMaterial(this.materials[lodLevel]); + } + } + + private calculateLODLevel(distance: number): number { + for (let i = 0; i < this.lodDistances.length; i++) { + if (distance < this.lodDistances[i]) return i; + } + return this.lodDistances.length; + } +} +``` + +### Material Property Animation + +```ts +class MaterialAnimator extends Script { + private material: PBRMaterial; + private animationCurves: Map = new Map(); + + addPropertyAnimation(propertyName: string, curve: AnimationCurve): void { + this.animationCurves.set(propertyName, curve); + } + + onUpdate(deltaTime: number): void { + const time = this.engine.time.totalTime; + + for (const [propertyName, curve] of this.animationCurves) { + const value = curve.evaluate(time); + + switch (propertyName) { + case "metallic": + this.material.metallic = value; + break; + case "roughness": + this.material.roughness = value; + break; + case "emissiveIntensity": + const emissive = this.material.emissiveColor; + this.material.emissiveColor.set(value, value, value, emissive.w); + break; + } + } + } +} + +// Usage +const animator = entity.addComponent(MaterialAnimator); +animator.addPropertyAnimation("metallic", metallicCurve); +animator.addPropertyAnimation("roughness", roughnessCurve); +``` + +## Performance Optimization + +### Material Batching + +```ts +class MaterialBatcher { + private materialInstances: Map = new Map(); + + getSharedMaterial(materialId: string, factory: () => Material): Material { + if (!this.materialInstances.has(materialId)) { + this.materialInstances.set(materialId, factory()); + } + return this.materialInstances.get(materialId); + } + + createVariantMaterial(baseMaterialId: string, variantId: string, + modifier: (material: Material) => void): Material { + const key = `${baseMaterialId}_${variantId}`; + + if (!this.materialInstances.has(key)) { + const baseMaterial = this.materialInstances.get(baseMaterialId); + if (baseMaterial) { + const variant = baseMaterial.clone(); + modifier(variant); + this.materialInstances.set(key, variant); + } + } + + return this.materialInstances.get(key); + } +} +``` + +### Efficient Property Updates + +```ts +class OptimizedMaterialController extends Script { + private material: PBRMaterial; + private lastUpdateTime = 0; + private updateInterval = 1000 / 30; // Update at 30 FPS instead of every frame + + onUpdate(deltaTime: number): void { + const now = performance.now(); + + if (now - this.lastUpdateTime >= this.updateInterval) { + this.updateMaterialProperties(); + this.lastUpdateTime = now; + } + } + + private updateMaterialProperties(): void { + // Only update when values actually change + const newRoughness = this.calculateRoughness(); + if (Math.abs(this.material.roughness - newRoughness) > 0.001) { + this.material.roughness = newRoughness; + } + } +} +``` + +## API Reference + +```apidoc +Material: + Properties: + name: string + - Display name of the material. + shader: Shader + - Shader used by the material. Setting this updates render states automatically. + shaderData: ShaderData + - Container for shader properties, textures, and macros. + renderState: RenderState + - First render state (shortcut for renderStates[0]). + renderStates: Readonly + - All render states for multi-pass shaders. + + Methods: + constructor(engine: Engine, shader: Shader) + - Create material with specified shader. + clone(): Material + - Create a complete copy of the material. + cloneTo(target: Material): void + - Copy this material's properties to target material. + +BaseMaterial extends Material: + Properties: + isTransparent: boolean + - Whether material uses alpha blending. Affects render queue and depth writing. + blendMode: BlendMode + - Blending mode when transparent. Normal or Additive. + alphaCutoff: number + - Alpha threshold for alpha testing. 0 disables alpha testing. + renderFace: RenderFace + - Which faces to render. Front, Back, or Double. + + Methods: + setIsTransparent(passIndex: number, isTransparent: boolean): void + - Set transparency for specific shader pass. + setBlendMode(passIndex: number, blendMode: BlendMode): void + - Set blend mode for specific shader pass. + setRenderFace(passIndex: number, renderFace: RenderFace): void + - Set face culling for specific shader pass. + +BlinnPhongMaterial extends BaseMaterial: + Properties: + baseColor: Color + - Diffuse color of the material. + baseTexture: Texture2D + - Diffuse texture map. + specularColor: Color + - Specular reflection color. + specularTexture: Texture2D + - Specular intensity texture map. + normalTexture: Texture2D + - Normal map for surface detail. + normalIntensity: number + - Strength of normal map effect. + emissiveColor: Color + - Emissive glow color. + emissiveTexture: Texture2D + - Emissive texture map. + shininess: number + - Specular power (higher = more focused highlights). + tilingOffset: Vector4 + - UV tiling (xy) and offset (zw) for texture coordinates. + +PBRMaterial extends BaseMaterial: + Properties: + baseColor: Color + - Albedo color of the material. + baseTexture: Texture2D + - Albedo texture map. + metallic: number + - Metallic factor (0 = dielectric, 1 = metallic). + roughness: number + - Roughness factor (0 = mirror, 1 = completely rough). + metallicRoughnessTexture: Texture2D + - Combined metallic (B channel) and roughness (G channel) texture. + normalTexture: Texture2D + - Tangent-space normal map. + normalIntensity: number + - Normal map intensity. + occlusionTexture: Texture2D + - Ambient occlusion texture (R channel). + occlusionIntensity: number + - AO effect intensity. + emissiveColor: Color + - Emissive color. + emissiveTexture: Texture2D + - Emissive texture map. + clearCoat: number + - Clear coat layer intensity. + clearCoatRoughness: number + - Clear coat roughness. + tilingOffset: Vector4 + - UV tiling and offset. + +UnlitMaterial extends BaseMaterial: + Properties: + baseColor: Color + - Base color (unaffected by lighting). + baseTexture: Texture2D + - Base texture map. + tilingOffset: Vector4 + - UV tiling and offset. + +Enums: + BlendMode: + Normal: Standard alpha blending + Additive: Additive blending for effects + + RenderFace: + Front: Render only front faces + Back: Render only back faces + Double: Render both faces + + RenderQueueType: + Opaque: Solid objects (2000) + AlphaTest: Alpha-tested objects (2450) + Transparent: Transparent objects (3000) +``` + +## Best Practices + +- **Use Appropriate Material Types**: Choose PBR for realistic rendering, Blinn-Phong for stylized art, Unlit for UI and effects +- **Share Materials When Possible**: Use the same material instance across multiple objects with similar appearance +- **Optimize Texture Usage**: Use appropriate texture resolutions and formats for target platforms +- **Minimize Render State Changes**: Group objects with similar materials to reduce state switching +- **Use LOD Materials**: Switch to simpler materials at distance to improve performance +- **Cache Material Instances**: Avoid creating new materials every frame +- **Batch Property Updates**: Update material properties at lower frequencies when possible +- **Use Shader Macros**: Enable/disable features through macros rather than multiple shaders + +## Common Issues + +**Material Not Appearing**: Check that shader is valid and material is assigned: +```ts +if (!material.shader) { + console.error("Material has no shader assigned"); +} +if (renderer.getMaterial() !== material) { + console.error("Material not assigned to renderer"); +} +``` + +**Transparency Issues**: Ensure correct render state setup: +```ts +// For alpha blending +material.renderState.renderQueueType = RenderQueueType.Transparent; +material.renderState.depthState.writeEnabled = false; +material.renderState.blendState.enabled = true; + +// Configure blend factors +const target = material.renderState.blendState.targetBlendState; +target.sourceColorBlendFactor = BlendFactor.SourceAlpha; +target.destinationColorBlendFactor = BlendFactor.OneMinusSourceAlpha; +material.alphaCutoff = 0.5; // Enable alpha testing +``` + +**Texture Not Loading**: Verify texture assignment and loading: +```ts +if (!material.baseTexture) { + console.error("Base texture not assigned"); +} else if (material.baseTexture.destroyed) { + console.error("Base texture was destroyed"); +} +``` + +**Performance Problems**: Monitor material complexity: +```ts +// Check for expensive operations +if (material.renderStates.length > 1) { + console.warn("Multi-pass material may impact performance"); +} +if (material.isTransparent) { + console.warn("Transparent material requires sorting"); +} +``` diff --git a/docs/scripting/memory-management.md b/docs/scripting/memory-management.md new file mode 100644 index 0000000000..e0600298f7 --- /dev/null +++ b/docs/scripting/memory-management.md @@ -0,0 +1,503 @@ +# Memory Management + +Galacean Engine provides a comprehensive set of memory management tools designed to minimize garbage collection pressure and optimize performance in real-time applications. These tools include object pools, specialized arrays, and memory-efficient data structures. + +## Object Pool System + +### IPoolElement Interface + +All pooled objects must implement the `IPoolElement` interface: + +```ts +interface IPoolElement { + /** + * Called when the object is returned to the pool. + * Use this to reset the object state and release references. + */ + dispose?(): void; +} + +// Example implementation +class PooledObject implements IPoolElement { + data: any[] = []; + + dispose(): void { + // Clear references to prevent memory leaks + this.data.length = 0; + this.someReference = null; + } +} +``` + +### ObjectPool (Abstract Base) + +The base class for all object pool implementations: + +```ts +abstract class ObjectPool { + protected _type: new () => T; + protected _elements: T[]; + + constructor(type: new () => T) { + this._type = type; + } + + // Cleanup all pooled objects + garbageCollection(): void { + const elements = this._elements; + for (let i = elements.length - 1; i >= 0; i--) { + elements[i].dispose && elements[i].dispose(); + } + elements.length = 0; + } + + abstract get(): T; +} +``` + +### ClearableObjectPool + +Optimized for scenarios where objects are used in batches and then cleared all at once: + +```ts +class ClearableObjectPool extends ObjectPool { + private _usedElementCount: number = 0; + + constructor(type: new () => T) { + super(type); + this._elements = []; + } + + get(): T { + const { _usedElementCount: usedCount, _elements: elements } = this; + this._usedElementCount++; + + if (elements.length === usedCount) { + // Create new object if pool is exhausted + const element = new this._type(); + elements.push(element); + return element; + } else { + // Reuse existing object + return elements[usedCount]; + } + } + + clear(): void { + // Reset usage counter without destroying objects + this._usedElementCount = 0; + } +} + +// Usage example +const renderElementPool = new ClearableObjectPool(RenderElement); + +// During frame rendering +const element1 = renderElementPool.get(); +const element2 = renderElementPool.get(); + +// At end of frame +renderElementPool.clear(); // Reset for next frame +``` + +### ReturnableObjectPool + +Best for objects with unpredictable lifetimes that need explicit return: + +```ts +class ReturnableObjectPool extends ObjectPool { + private _lastElementIndex: number; + + constructor(type: new () => T, initializeCount: number = 1) { + super(type); + this._lastElementIndex = initializeCount - 1; + this._elements = new Array(initializeCount); + + // Pre-populate pool + for (let i = 0; i < initializeCount; ++i) { + this._elements[i] = new type(); + } + } + + get(): T { + if (this._lastElementIndex < 0) { + // Pool exhausted, create new object + return new this._type(); + } + return this._elements[this._lastElementIndex--]; + } + + return(element: T): void { + // Call dispose before returning to pool + element.dispose && element.dispose(); + this._elements[++this._lastElementIndex] = element; + } +} + +// Usage example +const vertexAreaPool = new ReturnableObjectPool(VertexArea, 10); + +function allocateVertexArea(): VertexArea { + const area = vertexAreaPool.get(); + area.start = 0; + area.size = 100; + return area; +} + +function freeVertexArea(area: VertexArea): void { + vertexAreaPool.return(area); // Automatically calls dispose() +} +``` + +## Specialized Array Types + +### DisorderedArray + +High-performance array that uses swap-delete for O(1) removal: + +```ts +class DisorderedArray { + length = 0; + private _elements: T[]; + private _loopCounter = 0; + private _blankCount = 0; + + constructor(count: number = 0) { + this._elements = new Array(count); + } + + get isLopping(): boolean { + return this._loopCounter > 0; + } + + add(element: T): void { + this._elements[this.length++] = element; + } + + // O(1) removal using swap with last element + deleteByIndex(index: number): void { + const elements = this._elements; + const lastIndex = --this.length; + + if (index !== lastIndex) { + elements[index] = elements[lastIndex]; + } + elements[lastIndex] = null; + } + + // Safe iteration with modification support + forEach( + callback: (element: T, index: number) => void, + endCallback?: (element: T, index: number) => void + ): void { + this._loopCounter++; + + for (let i = 0; i < this.length; i++) { + const element = this._elements[i]; + if (element) { + callback(element, i); + } + } + + if (endCallback) { + for (let i = 0; i < this.length; i++) { + const element = this._elements[i]; + if (element) { + endCallback(element, i); + } + } + } + + this._loopCounter--; + } +} + +// Usage example +class ComponentManager { + private _scripts = new DisorderedArray