Add skeleton animation (mesh skinning) (#328)

* Convert update verticies into external animator

* Add empty skeleton API calls

* Implement Skeleton

* Add rgl_mesh_set_bone_weights

* Bind skeleton to entity (+empty SkeletonAnimator class)

* Rename rgl_entity_set_pose -> rgl_entity_set_transform

* Fix GASBuilder

* BoneWeights: fix member's order & separate definition to avoid massive includes in cuda

* Add rgl_mesh_set_restposes

* Remove Skeleton API object

* Add rgl_entity_set_pose_world and SkeletionAnimator logic

* Validate mesh when creating SkeletonAnimator

* Calculate displacement for SkeletonAnimator

* Renaming

* Remove TODO verification (verified)

* Add test to validate velocity resulted from entity's animation

* Extract function for animation time update

* Delete GAS builders for static meshes if no needed

* Update docs

* Simplify std::visit

* Review changes

* Review changes

Author: msz-rai

CMakeLists.txt

@@ -104,11 +104,14 @@ set(RGL_SOURCE_FILES
     src/gpu/helpersKernels.cu
     src/gpu/gaussianNoiseKernels.cu
     src/gpu/nodeKernels.cu
+    src/gpu/sceneKernels.cu
     src/scene/Scene.cpp
     src/scene/Mesh.cpp
     src/scene/Entity.cpp
     src/scene/Texture.cpp
     src/scene/ASBuildScratchpad.cpp
+    src/scene/animator/ExternalAnimator.cpp
+    src/scene/animator/SkeletonAnimator.cpp
     src/graph/GraphRunCtx.cpp
     src/graph/Node.cpp
     src/graph/GaussianNoiseAngularHitpointNode.cpp

docs/Usage.md

@@ -95,7 +95,7 @@ rgl_mat3x4f entity_tf = {
     { 0, 1, 0, 0 },
     { 0, 0, 1, 5 }}
 };
-rgl_entity_set_pose(cube_entity, &entity_tf);
+rgl_entity_set_transform(cube_entity, &entity_tf);
 
 // Create a Graph representation of a lidar that sends 1 ray and is situated at (x,y,z) = (0, 0, 0), facing positive Z
 rgl_mat3x4f ray_tf = {

include/rgl/api/core.h

@@ -151,6 +151,31 @@ static_assert(std::is_trivial<rgl_radar_scope_t>::value);
 static_assert(std::is_standard_layout<rgl_radar_scope_t>::value);
 #endif
 
+/**
+ * Describes 4 bone weights affecting a mesh vertex.
+ * The sum of all weights for a given vertex should equal 1 (RGL do not normalize them).
+ * If a vertex is affected by fewer than 4 bones, each of the remaining weight values must be 0.
+ * bone_indexes for unused bones must still be valid (filled with the existing bone indexes).
+ */
+typedef struct
+{
+	/**
+	 * Array for skinning weights.
+	 * The weight at each index corresponds to the bone_idx with the same index.
+	 */
+	float weights[4];
+	/**
+	 * Array for 4 bone indexes affecting a vertex.
+	 */
+	int32_t bone_indexes[4];
+} rgl_bone_weights_t;
+
+#ifdef __cplusplus
+static_assert(sizeof(rgl_bone_weights_t) == 4 * sizeof(float) + 4 * sizeof(int32_t));
+static_assert(std::is_trivial<rgl_bone_weights_t>::value);
+static_assert(std::is_standard_layout<rgl_bone_weights_t>::value);
+#endif
+
 /**
  * Radar object class used in object tracking.
  */
@@ -369,7 +394,8 @@ typedef enum : int32_t
 	 * - linear velocity
 	 * - angular velocity
 	 * - mesh deformations (e.g. skinning)
-	 * The aforementioned are inferred from calls to `rgl_entity_set_pose`, `rgl_scene_set_time` and `rgl_mesh_update_vertices`.
+	 * The aforementioned are inferred from calls to
+	 *  `rgl_entity_set_transform`, `rgl_scene_set_time` and `rgl_entity_apply_external_animation`, `rgl_entity_set_pose_world`.
 	 */
 	RGL_FIELD_ABSOLUTE_VELOCITY_VEC3_F32,
 
@@ -512,23 +538,35 @@ RGL_API rgl_status_t rgl_mesh_create(rgl_mesh_t* out_mesh, const rgl_vec3f* vert
  */
 RGL_API rgl_status_t rgl_mesh_set_texture_coords(rgl_mesh_t mesh, const rgl_vec2f* uvs, int32_t uv_count);
 
+/**
+ * Assign bone weights to given Mesh.
+ *
+ * @param mesh Mesh to modify.
+ * @param bone_weights An array of rgl_bone_weights_t objects that associate vertices with the bones affecting them.
+                       The bone weights object at each index corresponds to the vertex with the same index (`vertices` array of `rgl_mesh_create` API call).
+ * @param bone_weights_count Number of elements in the bone_weights array. It must be equal to the vertex count of the Mesh!
+ */
+RGL_API rgl_status_t rgl_mesh_set_bone_weights(rgl_mesh_t mesh, const rgl_bone_weights_t* bone_weights,
+                                               int32_t bone_weights_count);
+
+/**
+ * Assign restposes to given Mesh.
+ *
+ * @param mesh Mesh to modify.
+ * @param restposes An array containing inverse of the transformation matrix of the bone in restpose for each bone.
+ *                  Restpose at each index in the array corresponds to the bone with the same index.
+ *                  Typically, the restpose is the same as the bindpose.
+ * @param bones_count Number of elements in the restposes array.
+ */
+RGL_API rgl_status_t rgl_mesh_set_restposes(rgl_mesh_t mesh, const rgl_mat3x4f* restposes, int32_t bones_count);
+
 /**
  * Informs that the given Mesh will be no longer used.
  * The Mesh will be destroyed after all referring Entities are destroyed.
  * @param mesh Mesh to be marked as no longer needed
  */
 RGL_API rgl_status_t rgl_mesh_destroy(rgl_mesh_t mesh);
 
-/**
- * Updates Mesh vertex data. The number of vertices must not change.
- * This function is intended to update animated Meshes.
- * Should be called after rgl_scene_set_time to ensure proper velocity computation.
- * @param mesh Mesh to modify
- * @param vertices An array of rgl_vec3f or binary-compatible data representing Mesh vertices
- * @param vertex_count Number of elements in the vertices array. It must be equal to the original vertex count!
- */
-RGL_API rgl_status_t rgl_mesh_update_vertices(rgl_mesh_t mesh, const rgl_vec3f* vertices, int32_t vertex_count);
-
 /**
  * Assigns value true to out_alive if the given mesh is known and has not been destroyed,
  * assigns value false otherwise.
@@ -561,7 +599,20 @@ RGL_API rgl_status_t rgl_entity_destroy(rgl_entity_t entity);
  * @param entity Entity to modify
  * @param transform Pointer to rgl_mat3x4f (or binary-compatible data) representing desired (Entity -> world) coordinate system transform.
  */
-RGL_API rgl_status_t rgl_entity_set_pose(rgl_entity_t entity, const rgl_mat3x4f* transform);
+RGL_API rgl_status_t rgl_entity_set_transform(rgl_entity_t entity, const rgl_mat3x4f* transform);
+
+/**
+ * Set the current pose of the given Entity in world coordinates.
+ * The pose stands for bone transforms used in skeleton animation.
+ * The mesh associated with this entity must have bone weights and restposes assigned.
+ * Since it is expected the pose is already in world coordinates, the API call `rgl_entity_set_transform` should no longer be called on this entity.
+ * Should be called after rgl_scene_set_time to ensure proper velocity computation.
+ * @param entity Entity to modify.
+ * @param pose An array containing transformation matrices of the bones in world coordinates.
+ *             Bone transform at each index corresponds to the bone with the same index.
+ * @param bones_count Number of elements in the pose array. It must be equal to restposes count in the associated mesh!
+ */
+RGL_API rgl_status_t rgl_entity_set_pose_world(rgl_entity_t entity, const rgl_mat3x4f* pose, int32_t bones_count);
 
 /**
  * Set instance ID of the given Entity.
@@ -586,6 +637,17 @@ RGL_API rgl_status_t rgl_entity_set_intensity_texture(rgl_entity_t entity, rgl_t
  */
 RGL_API rgl_status_t rgl_entity_set_laser_retro(rgl_entity_t entity, float retro);
 
+/**
+ * Provides updated vertices to the Entity resulted from external animation system.
+ * It does not modify Mesh API object bound to the Entity.
+ * The number of vertices must not change.
+ * Should be called after rgl_scene_set_time to ensure proper velocity computation.
+ * @param entity Entity to modify
+ * @param vertices An array of rgl_vec3f or binary-compatible data representing Mesh vertices
+ * @param vertex_count Number of elements in the vertices array. It must be equal to the original vertex count!
+ */
+RGL_API rgl_status_t rgl_entity_apply_external_animation(rgl_entity_t entity, const rgl_vec3f* vertices, int32_t vertex_count);
+
 /**
  * Assigns value true to out_alive if the given entity is known and has not been destroyed,
  * assigns value false otherwise.
@@ -723,7 +785,7 @@ RGL_API rgl_status_t rgl_node_raytrace(rgl_node_t* node, rgl_scene_t scene);
  * Necessary for velocity distortion or calculating fields: RGL_FIELD_RELATIVE_VELOCITY_VEC3_F32 and RGL_FIELD_RADIAL_SPEED_F32.
  * Relative velocity calculation:
  * To calculate relative velocity the pipeline must allow to compute absolute velocities. For more details refer to API calls documentation:
- * `rgl_scene_set_time`, `rgl_entity_set_pose`, and `rgl_mesh_update_vertices`
+ * `rgl_scene_set_time`, `rgl_entity_set_transform`, `rgl_entity_set_pose_world`, and `rgl_entity_apply_external_animation`
  * @param node RaytraceNode to modify
  * @param linear_velocity 3D vector for linear velocity in units per second.
  * @param angular_velocity 3D vector for angular velocity in radians per second (roll, pitch, yaw).

src/api/apiCore.cpp

@@ -217,43 +217,65 @@ void TapeCore::tape_mesh_set_texture_coords(const YAML::Node& yamlNode, Playback
 	                            yamlNode[2].as<int32_t>());
 }
 
-RGL_API rgl_status_t rgl_mesh_destroy(rgl_mesh_t mesh)
+RGL_API rgl_status_t rgl_mesh_set_bone_weights(rgl_mesh_t mesh, const rgl_bone_weights_t* bone_weights,
+                                               int32_t bone_weights_count)
 {
 	auto status = rglSafeCall([&]() {
-		RGL_API_LOG("rgl_mesh_destroy(mesh={})", (void*) mesh);
+		RGL_API_LOG("rgl_mesh_set_bone_weights(mesh={}, bone_weights={})", (void*) mesh,
+		            repr(bone_weights, bone_weights_count));
 		CHECK_ARG(mesh != nullptr);
+		CHECK_ARG(bone_weights != nullptr);
+		CHECK_ARG(bone_weights_count > 0);
 		GraphRunCtx::synchronizeAll(); // Prevent races with graph threads
-		Mesh::release(mesh);
+		Mesh::validatePtr(mesh)->setBoneWeights(bone_weights, bone_weights_count);
 	});
-	TAPE_HOOK(mesh);
+	TAPE_HOOK(mesh, TAPE_ARRAY(bone_weights, bone_weights_count), bone_weights_count);
 	return status;
 }
 
-void TapeCore::tape_mesh_destroy(const YAML::Node& yamlNode, PlaybackState& state)
+void TapeCore::tape_mesh_set_bone_weights(const YAML::Node& yamlNode, PlaybackState& state)
 {
-	auto meshId = yamlNode[0].as<TapeAPIObjectID>();
-	rgl_mesh_destroy(state.meshes.at(meshId));
-	state.meshes.erase(meshId);
+	rgl_mesh_set_bone_weights(state.meshes.at(yamlNode[0].as<TapeAPIObjectID>()),
+	                          state.getPtr<const rgl_bone_weights_t>(yamlNode[1]), yamlNode[2].as<int32_t>());
 }
 
-RGL_API rgl_status_t rgl_mesh_update_vertices(rgl_mesh_t mesh, const rgl_vec3f* vertices, int32_t vertex_count)
+RGL_API rgl_status_t rgl_mesh_set_restposes(rgl_mesh_t mesh, const rgl_mat3x4f* restposes, int32_t restposes_count)
 {
 	auto status = rglSafeCall([&]() {
-		RGL_API_LOG("rgl_mesh_update_vertices(mesh={}, vertices={})", (void*) mesh, repr(vertices, vertex_count));
+		RGL_API_LOG("rgl_mesh_set_restposes(mesh={}, restposes={})", (void*) mesh, repr(restposes, restposes_count));
 		CHECK_ARG(mesh != nullptr);
-		CHECK_ARG(vertices != nullptr);
-		CHECK_ARG(vertex_count > 0);
+		CHECK_ARG(restposes != nullptr);
+		CHECK_ARG(restposes_count > 0);
 		GraphRunCtx::synchronizeAll(); // Prevent races with graph threads
-		Mesh::validatePtr(mesh)->updateVertices(reinterpret_cast<const Vec3f*>(vertices), vertex_count);
+		Mesh::validatePtr(mesh)->setRestposes(reinterpret_cast<const Mat3x4f*>(restposes), restposes_count);
 	});
-	TAPE_HOOK(mesh, TAPE_ARRAY(vertices, vertex_count), vertex_count);
+	TAPE_HOOK(mesh, TAPE_ARRAY(restposes, restposes_count), restposes_count);
 	return status;
 }
 
-void TapeCore::tape_mesh_update_vertices(const YAML::Node& yamlNode, PlaybackState& state)
+void TapeCore::tape_mesh_set_restposes(const YAML::Node& yamlNode, PlaybackState& state)
 {
-	rgl_mesh_update_vertices(state.meshes.at(yamlNode[0].as<TapeAPIObjectID>()), state.getPtr<const rgl_vec3f>(yamlNode[1]),
-	                         yamlNode[2].as<int32_t>());
+	rgl_mesh_set_restposes(state.meshes.at(yamlNode[0].as<TapeAPIObjectID>()), state.getPtr<const rgl_mat3x4f>(yamlNode[1]),
+	                       yamlNode[2].as<int32_t>());
+}
+
+RGL_API rgl_status_t rgl_mesh_destroy(rgl_mesh_t mesh)
+{
+	auto status = rglSafeCall([&]() {
+		RGL_API_LOG("rgl_mesh_destroy(mesh={})", (void*) mesh);
+		CHECK_ARG(mesh != nullptr);
+		GraphRunCtx::synchronizeAll(); // Prevent races with graph threads
+		Mesh::release(mesh);
+	});
+	TAPE_HOOK(mesh);
+	return status;
+}
+
+void TapeCore::tape_mesh_destroy(const YAML::Node& yamlNode, PlaybackState& state)
+{
+	auto meshId = yamlNode[0].as<TapeAPIObjectID>();
+	rgl_mesh_destroy(state.meshes.at(meshId));
+	state.meshes.erase(meshId);
 }
 
 rgl_status_t rgl_mesh_is_alive(rgl_mesh_t mesh, bool* out_alive)
@@ -308,10 +330,10 @@ void TapeCore::tape_entity_destroy(const YAML::Node& yamlNode, PlaybackState& st
 	state.entities.erase(entityId);
 }
 
-RGL_API rgl_status_t rgl_entity_set_pose(rgl_entity_t entity, const rgl_mat3x4f* transform)
+RGL_API rgl_status_t rgl_entity_set_transform(rgl_entity_t entity, const rgl_mat3x4f* transform)
 {
 	auto status = rglSafeCall([&]() {
-		RGL_API_LOG("rgl_entity_set_pose(entity={}, transform={})", (void*) entity, repr(transform, 1));
+		RGL_API_LOG("rgl_entity_set_transform(entity={}, transform={})", (void*) entity, repr(transform, 1));
 		CHECK_ARG(entity != nullptr);
 		CHECK_ARG(transform != nullptr);
 		GraphRunCtx::synchronizeAll(); // Prevent races with graph threads
@@ -322,9 +344,30 @@ RGL_API rgl_status_t rgl_entity_set_pose(rgl_entity_t entity, const rgl_mat3x4f*
 	return status;
 }
 
-void TapeCore::tape_entity_set_pose(const YAML::Node& yamlNode, PlaybackState& state)
+void TapeCore::tape_entity_set_transform(const YAML::Node& yamlNode, PlaybackState& state)
+{
+	rgl_entity_set_transform(state.entities.at(yamlNode[0].as<TapeAPIObjectID>()),
+	                         state.getPtr<const rgl_mat3x4f>(yamlNode[1]));
+}
+
+RGL_API rgl_status_t rgl_entity_set_pose_world(rgl_entity_t entity, const rgl_mat3x4f* pose, int32_t bones_count)
+{
+	auto status = rglSafeCall([&]() {
+		RGL_API_LOG("rgl_entity_set_pose_world(entity={}, pose={})", (void*) entity, repr(pose, bones_count));
+		CHECK_ARG(entity != nullptr);
+		CHECK_ARG(pose != nullptr);
+		CHECK_ARG(bones_count > 0);
+		GraphRunCtx::synchronizeAll(); // Prevent races with graph threads
+		Entity::validatePtr(entity)->setPoseAndAnimate(reinterpret_cast<const Mat3x4f*>(pose), bones_count);
+	});
+	TAPE_HOOK(entity, TAPE_ARRAY(pose, bones_count), bones_count);
+	return status;
+}
+
+void TapeCore::tape_entity_set_pose_world(const YAML::Node& yamlNode, PlaybackState& state)
 {
-	rgl_entity_set_pose(state.entities.at(yamlNode[0].as<TapeAPIObjectID>()), state.getPtr<const rgl_mat3x4f>(yamlNode[1]));
+	rgl_entity_set_pose_world(state.entities.at(yamlNode[0].as<TapeAPIObjectID>()),
+	                          state.getPtr<const rgl_mat3x4f>(yamlNode[1]), yamlNode[2].as<int32_t>());
 }
 
 RGL_API rgl_status_t rgl_entity_set_id(rgl_entity_t entity, int32_t id)
@@ -382,6 +425,27 @@ void TapeCore::tape_entity_set_laser_retro(const YAML::Node& yamlNode, PlaybackS
 	                           yamlNode[1].as<Field<LASER_RETRO_F32>::type>());
 }
 
+RGL_API rgl_status_t rgl_entity_apply_external_animation(rgl_entity_t entity, const rgl_vec3f* vertices, int32_t vertex_count)
+{
+	auto status = rglSafeCall([&]() {
+		RGL_API_LOG("rgl_entity_apply_external_animation(entity={}, vertices={})", (void*) entity,
+		            repr(vertices, vertex_count));
+		CHECK_ARG(entity != nullptr);
+		CHECK_ARG(vertices != nullptr);
+		CHECK_ARG(vertex_count > 0);
+		GraphRunCtx::synchronizeAll(); // Prevent races with graph threads
+		Entity::validatePtr(entity)->applyExternalAnimation(reinterpret_cast<const Vec3f*>(vertices), vertex_count);
+	});
+	TAPE_HOOK(entity, TAPE_ARRAY(vertices, vertex_count), vertex_count);
+	return status;
+}
+
+void TapeCore::tape_entity_apply_external_animation(const YAML::Node& yamlNode, PlaybackState& state)
+{
+	rgl_entity_apply_external_animation(state.entities.at(yamlNode[0].as<TapeAPIObjectID>()),
+	                                    state.getPtr<const rgl_vec3f>(yamlNode[1]), yamlNode[2].as<int32_t>());
+}
+
 rgl_status_t rgl_entity_is_alive(rgl_entity_t entity, bool* out_alive)
 {
 	auto status = rglSafeCall([&]() {

src/gpu/helpersKernels.cu

@@ -31,19 +31,3 @@ void gpuSetupRandomNumberGenerator(cudaStream_t stream, size_t elementsCount, un
 {
 	run(kSetupRandomNumberGenerator, stream, elementsCount, seed, outPHILOXStates);
 }
-// Updates vertices and calculates their displacement.
-// Input: newVertices and oldVertices
-// Output: verticesDisplacement and newVertices
-__global__ void kUpdateVertices(size_t vertexCount, Vec3f* newVerticesToDisplacement, Vec3f* oldToNewVertices)
-{
-	LIMIT(vertexCount);
-	// See Mesh::updateVertices to understand the logic here.
-	Vec3f newVertex = newVerticesToDisplacement[tid];
-	newVerticesToDisplacement[tid] -= oldToNewVertices[tid];
-	oldToNewVertices[tid] = newVertex;
-}
-
-void gpuUpdateVertices(cudaStream_t stream, size_t vertexCount, Vec3f* newVerticesToDisplacement, Vec3f* oldToNewVertices)
-{
-	run(kUpdateVertices, stream, vertexCount, newVerticesToDisplacement, oldToNewVertices);
-}

src/gpu/helpersKernels.hpp

@@ -20,5 +20,3 @@
 
 void gpuSetupRandomNumberGenerator(cudaStream_t stream, size_t elementsCount, unsigned int seed,
                                    curandStatePhilox4_32_10_t* outPHILOXStates);
-
-void gpuUpdateVertices(cudaStream_t stream, size_t vertexCount, Vec3f* newVerticesToDisplacement, Vec3f* oldToNewVertices);

src/gpu/optixPrograms.cu

@@ -239,7 +239,6 @@ extern "C" __global__ void __closesthit__()
 		if (wasSkinned) {
 			Mat3x4f objectToWorld;
 			optixGetObjectToWorldTransformMatrix(reinterpret_cast<float*>(objectToWorld.rc));
-			// TODO(msz-rai): To verify if rotation is needed (in some tests it produces more realistic results)
 			const Vec3f& vA = objectToWorld.rotation() * entityData.vertexDisplacementSincePrevFrame[triangleIndices.x()];
 			const Vec3f& vB = objectToWorld.rotation() * entityData.vertexDisplacementSincePrevFrame[triangleIndices.y()];
 			const Vec3f& vC = objectToWorld.rotation() * entityData.vertexDisplacementSincePrevFrame[triangleIndices.z()];