Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions crates/spirv-std/src/arch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ pub fn kill() -> ! {
/// <https://htmlpreview.github.io/?https://github.com/KhronosGroup/SPIRV-Registry/blob/master/extensions/KHR/SPV_KHR_shader_clock.html>
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpReadClockKHR")]
pub unsafe fn read_clock_khr<const SCOPE: u32>() -> u64 {
pub fn read_clock_khr<const SCOPE: u32>() -> u64 {
unsafe {
let mut result: u64;

Expand All @@ -175,7 +175,7 @@ pub unsafe fn read_clock_khr<const SCOPE: u32>() -> u64 {
/// bits and the second component containing the 32 most significant bits.'
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpReadClockKHR")]
pub unsafe fn read_clock_uvec2_khr<V: Vector<u32, 2>, const SCOPE: u32>() -> V {
pub fn read_clock_uvec2_khr<V: Vector<u32, 2>, const SCOPE: u32>() -> V {
unsafe {
let mut result = V::default();

Expand Down
130 changes: 59 additions & 71 deletions crates/spirv-std/src/arch/barrier.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ use core::arch::asm;
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpControlBarrier")]
#[inline]
pub unsafe fn control_barrier<
pub fn control_barrier<
const EXECUTION: u32, // Scope
const MEMORY: u32, // Scope
const SEMANTICS: u32, // Semantics
Expand Down Expand Up @@ -70,7 +70,7 @@ pub unsafe fn control_barrier<
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpMemoryBarrier")]
#[inline]
pub unsafe fn memory_barrier<
pub fn memory_barrier<
const MEMORY: u32, // Scope
const SEMANTICS: u32, // Semantics
>() {
Expand All @@ -93,16 +93,14 @@ pub unsafe fn memory_barrier<
/// From <https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/groupmemorybarrier>
#[spirv_std_macros::gpu_only]
#[inline]
pub unsafe fn workgroup_memory_barrier() {
unsafe {
memory_barrier::<
{ crate::memory::Scope::Workgroup as u32 },
{
crate::memory::Semantics::WORKGROUP_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}
pub fn workgroup_memory_barrier() {
memory_barrier::<
{ crate::memory::Scope::Workgroup as u32 },
{
crate::memory::Semantics::WORKGROUP_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}

/// Blocks execution of all threads in a group until all group shared accesses have been completed and all threads in the group have reached this call.
Expand All @@ -112,17 +110,15 @@ pub unsafe fn workgroup_memory_barrier() {
/// From <https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/groupmemorybarrierwithgroupsync>
#[spirv_std_macros::gpu_only]
#[inline]
pub unsafe fn workgroup_memory_barrier_with_group_sync() {
unsafe {
control_barrier::<
{ crate::memory::Scope::Workgroup as u32 },
{ crate::memory::Scope::Workgroup as u32 },
{
crate::memory::Semantics::WORKGROUP_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}
pub fn workgroup_memory_barrier_with_group_sync() {
control_barrier::<
{ crate::memory::Scope::Workgroup as u32 },
{ crate::memory::Scope::Workgroup as u32 },
{
crate::memory::Semantics::WORKGROUP_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}

/// Blocks execution of all threads in a group until all device memory accesses have been completed.
Expand All @@ -132,17 +128,15 @@ pub unsafe fn workgroup_memory_barrier_with_group_sync() {
/// From <https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/devicememorybarrier>
#[spirv_std_macros::gpu_only]
#[inline]
pub unsafe fn device_memory_barrier() {
unsafe {
memory_barrier::<
{ crate::memory::Scope::Device as u32 },
{
crate::memory::Semantics::IMAGE_MEMORY.bits()
| crate::memory::Semantics::UNIFORM_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}
pub fn device_memory_barrier() {
memory_barrier::<
{ crate::memory::Scope::Device as u32 },
{
crate::memory::Semantics::IMAGE_MEMORY.bits()
| crate::memory::Semantics::UNIFORM_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}

/// Blocks execution of all threads in a group until all device memory accesses have been completed and all threads in the group have reached this call.
Expand All @@ -152,18 +146,16 @@ pub unsafe fn device_memory_barrier() {
/// From <https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/devicememorybarrierwithgroupsync>
#[spirv_std_macros::gpu_only]
#[inline]
pub unsafe fn device_memory_barrier_with_group_sync() {
unsafe {
control_barrier::<
{ crate::memory::Scope::Workgroup as u32 },
{ crate::memory::Scope::Device as u32 },
{
crate::memory::Semantics::IMAGE_MEMORY.bits()
| crate::memory::Semantics::UNIFORM_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}
pub fn device_memory_barrier_with_group_sync() {
control_barrier::<
{ crate::memory::Scope::Workgroup as u32 },
{ crate::memory::Scope::Device as u32 },
{
crate::memory::Semantics::IMAGE_MEMORY.bits()
| crate::memory::Semantics::UNIFORM_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}

/// Blocks execution of all threads in a group until all memory accesses have been completed.
Expand All @@ -173,18 +165,16 @@ pub unsafe fn device_memory_barrier_with_group_sync() {
/// From <https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/allmemorybarrier>
#[spirv_std_macros::gpu_only]
#[inline]
pub unsafe fn all_memory_barrier() {
unsafe {
memory_barrier::<
{ crate::memory::Scope::Device as u32 },
{
crate::memory::Semantics::WORKGROUP_MEMORY.bits()
| crate::memory::Semantics::IMAGE_MEMORY.bits()
| crate::memory::Semantics::UNIFORM_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}
pub fn all_memory_barrier() {
memory_barrier::<
{ crate::memory::Scope::Device as u32 },
{
crate::memory::Semantics::WORKGROUP_MEMORY.bits()
| crate::memory::Semantics::IMAGE_MEMORY.bits()
| crate::memory::Semantics::UNIFORM_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}

/// Blocks execution of all threads in a group until all memory accesses have been completed and all threads in the group have reached this call.
Expand All @@ -194,17 +184,15 @@ pub unsafe fn all_memory_barrier() {
/// From <https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/allmemorybarrierwithgroupsync>
#[spirv_std_macros::gpu_only]
#[inline]
pub unsafe fn all_memory_barrier_with_group_sync() {
unsafe {
control_barrier::<
{ crate::memory::Scope::Workgroup as u32 },
{ crate::memory::Scope::Device as u32 },
{
crate::memory::Semantics::WORKGROUP_MEMORY.bits()
| crate::memory::Semantics::IMAGE_MEMORY.bits()
| crate::memory::Semantics::UNIFORM_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}
pub fn all_memory_barrier_with_group_sync() {
control_barrier::<
{ crate::memory::Scope::Workgroup as u32 },
{ crate::memory::Scope::Device as u32 },
{
crate::memory::Semantics::WORKGROUP_MEMORY.bits()
| crate::memory::Semantics::IMAGE_MEMORY.bits()
| crate::memory::Semantics::UNIFORM_MEMORY.bits()
| crate::memory::Semantics::ACQUIRE_RELEASE.bits()
},
>();
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use core::arch::asm;
/// of [demote_to_helper_invocation].
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpDemoteToHelperInvocationEXT", alias = "discard")]
pub unsafe fn demote_to_helper_invocation() {
pub fn demote_to_helper_invocation() {
unsafe {
asm!("OpDemoteToHelperInvocationEXT");
}
Expand Down
13 changes: 13 additions & 0 deletions crates/spirv-std/src/arch/mesh_shading.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ use core::arch::asm;
/// by this instruction.
///
/// This instruction is only valid in the *`MeshEXT`* Execution Model.
///
/// # Safety
/// * Must be called **exactly once** in mesh shaders
/// * Must be called in uniform control flow
/// * Must not write any output before this instruction in invoked
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpSetMeshOutputsEXT")]
#[inline]
Expand Down Expand Up @@ -54,6 +59,10 @@ pub unsafe fn set_mesh_outputs_ext(vertex_count: u32, primitive_count: u32) {
/// This instruction must be the last instruction in a block.
///
/// This instruction is only valid in the *`TaskEXT`* Execution Model.
///
/// # Safety
/// * Must be called **exactly once** in task shaders
/// * Must be called in uniform control flow
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpEmitMeshTasksEXT")]
#[inline]
Expand Down Expand Up @@ -93,6 +102,10 @@ pub unsafe fn emit_mesh_tasks_ext(group_count_x: u32, group_count_y: u32, group_
/// This instruction must be the last instruction in a block.
///
/// This instruction is only valid in the *`TaskEXT`* Execution Model.
///
/// # Safety
/// * Must be called **exactly once** in task shaders
/// * Must be called in uniform control flow
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpEmitMeshTasksEXT")]
#[inline]
Expand Down
72 changes: 31 additions & 41 deletions crates/spirv-std/src/arch/subgroup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,18 +64,16 @@ pub enum GroupOperation {
#[doc(alias = "subgroupBarrier")]
#[inline]
pub fn subgroup_barrier() {
unsafe {
barrier::control_barrier::<
SUBGROUP,
SUBGROUP,
{
Semantics::ACQUIRE_RELEASE.bits()
| Semantics::UNIFORM_MEMORY.bits()
| Semantics::WORKGROUP_MEMORY.bits()
| Semantics::IMAGE_MEMORY.bits()
},
>();
}
barrier::control_barrier::<
SUBGROUP,
SUBGROUP,
{
Semantics::ACQUIRE_RELEASE.bits()
| Semantics::UNIFORM_MEMORY.bits()
| Semantics::WORKGROUP_MEMORY.bits()
| Semantics::IMAGE_MEMORY.bits()
},
>();
}

/// The function `subgroupMemoryBarrier()` enforces the ordering of all memory
Expand All @@ -87,17 +85,15 @@ pub fn subgroup_barrier() {
#[doc(alias = "subgroupMemoryBarrier")]
#[inline]
pub fn subgroup_memory_barrier() {
unsafe {
barrier::memory_barrier::<
SUBGROUP,
{
Semantics::ACQUIRE_RELEASE.bits()
| Semantics::UNIFORM_MEMORY.bits()
| Semantics::WORKGROUP_MEMORY.bits()
| Semantics::IMAGE_MEMORY.bits()
},
>();
}
barrier::memory_barrier::<
SUBGROUP,
{
Semantics::ACQUIRE_RELEASE.bits()
| Semantics::UNIFORM_MEMORY.bits()
| Semantics::WORKGROUP_MEMORY.bits()
| Semantics::IMAGE_MEMORY.bits()
},
>();
}

/// The function `subgroupMemoryBarrierBuffer()` enforces the ordering of all
Expand All @@ -109,12 +105,10 @@ pub fn subgroup_memory_barrier() {
#[doc(alias = "subgroupMemoryBarrierBuffer")]
#[inline]
pub fn subgroup_memory_barrier_buffer() {
unsafe {
barrier::memory_barrier::<
SUBGROUP,
{ Semantics::ACQUIRE_RELEASE.bits() | Semantics::UNIFORM_MEMORY.bits() },
>();
}
barrier::memory_barrier::<
SUBGROUP,
{ Semantics::ACQUIRE_RELEASE.bits() | Semantics::UNIFORM_MEMORY.bits() },
>();
}

/// The function `subgroupMemoryBarrierShared()` enforces the ordering of all
Expand All @@ -128,12 +122,10 @@ pub fn subgroup_memory_barrier_buffer() {
#[doc(alias = "subgroupMemoryBarrierShared")]
#[inline]
pub fn subgroup_memory_barrier_shared() {
unsafe {
barrier::memory_barrier::<
SUBGROUP,
{ Semantics::ACQUIRE_RELEASE.bits() | Semantics::WORKGROUP_MEMORY.bits() },
>();
}
barrier::memory_barrier::<
SUBGROUP,
{ Semantics::ACQUIRE_RELEASE.bits() | Semantics::WORKGROUP_MEMORY.bits() },
>();
}

/// The function `subgroupMemoryBarrierImage()` enforces the ordering of all
Expand All @@ -145,12 +137,10 @@ pub fn subgroup_memory_barrier_shared() {
#[doc(alias = "subgroupMemoryBarrierImage")]
#[inline]
pub fn subgroup_memory_barrier_image() {
unsafe {
barrier::memory_barrier::<
SUBGROUP,
{ Semantics::ACQUIRE_RELEASE.bits() | Semantics::IMAGE_MEMORY.bits() },
>();
}
barrier::memory_barrier::<
SUBGROUP,
{ Semantics::ACQUIRE_RELEASE.bits() | Semantics::IMAGE_MEMORY.bits() },
>();
}

/// Result is true only in the active invocation with the lowest id in the group, otherwise result is false.
Expand Down
2 changes: 1 addition & 1 deletion examples/shaders/reduce/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ pub fn main(
if subgroup_local_invocation_id == 0 {
shared[subgroup_id as usize] = sum;
}
unsafe { spirv_std::arch::workgroup_memory_barrier_with_group_sync() };
spirv_std::arch::workgroup_memory_barrier_with_group_sync();
let mut sum = 0;
if subgroup_id == 0 {
if subgroup_local_invocation_id < num_subgroups {
Expand Down
4 changes: 2 additions & 2 deletions tests/compiletests/ui/arch/subgroup/subgroup_ballot.stderr
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
%1 = OpFunction %2 None %3
%4 = OpFunctionParameter %2
%5 = OpLabel
OpLine %6 377 13
OpLine %6 367 13
%7 = OpGroupNonUniformBallot %8 %9 %4
OpLine %6 416 13
OpLine %6 406 13
%10 = OpGroupNonUniformInverseBallot %2 %9 %7
OpNoLine
OpReturnValue %10
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
%1 = OpFunction %2 None %3
%4 = OpFunctionParameter %5
%6 = OpLabel
OpLine %7 511 0
OpLine %7 501 0
%8 = OpGroupNonUniformBallotBitCount %2 %9 Reduce %4
OpNoLine
OpReturnValue %8
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
%1 = OpFunction %2 None %3
%4 = OpFunctionParameter %2
%5 = OpLabel
OpLine %6 344 13
OpLine %6 334 13
%7 = OpGroupNonUniformBroadcastFirst %2 %8 %4
OpNoLine
OpReturnValue %7
Expand Down
Loading