Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions llvm/include/llvm-c/Core.h
Original file line number Diff line number Diff line change
Expand Up @@ -386,19 +386,19 @@ typedef enum {
LLVMAtomicRMWBinOpFMin, /**< Sets the value if it's smaller than the
original using an floating point comparison and
return the old one */
LLVMAtomicRMWBinOpFMaximum, /**< Sets the value if it's greater than the
original using an floating point comparison and
return the old one */
LLVMAtomicRMWBinOpFMinimum, /**< Sets the value if it's smaller than the
original using an floating point comparison and
return the old one */
LLVMAtomicRMWBinOpUIncWrap, /**< Increments the value, wrapping back to zero
when incremented above input value */
LLVMAtomicRMWBinOpUDecWrap, /**< Decrements the value, wrapping back to
the input value when decremented below zero */
LLVMAtomicRMWBinOpUSubCond, /**<Subtracts the value only if no unsigned
overflow */
LLVMAtomicRMWBinOpUSubSat, /**<Subtracts the value, clamping to zero */
LLVMAtomicRMWBinOpFMaximum, /**< Sets the value if it's greater than the
original using an floating point comparison and
return the old one */
LLVMAtomicRMWBinOpFMinimum, /**< Sets the value if it's smaller than the
original using an floating point comparison and
return the old one */
} LLVMAtomicRMWBinOp;

typedef enum {
Expand Down
6 changes: 0 additions & 6 deletions llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,6 @@ bool isSaturating(AtomicRMWInst& RMWI) {
case AtomicRMWInst::FMin:
// minnum(x, -inf) -> +inf
return CF->isNegative() && CF->isInfinity();
case AtomicRMWInst::FMaximum:
// maximum(x, +inf) -> +inf
return !CF->isNegative() && CF->isInfinity();
case AtomicRMWInst::FMinimum:
// minimum(x, -inf) -> +inf
return CF->isNegative() && CF->isInfinity();
case AtomicRMWInst::FAdd:
case AtomicRMWInst::FSub:
return CF->isNaN();
Expand Down
6 changes: 6 additions & 0 deletions llvm/test/Assembler/atomic.ll
Original file line number Diff line number Diff line change
Expand Up @@ -108,5 +108,11 @@ define void @fp_vector_atomicrmw(ptr %x, <2 x half> %val) {
; CHECK: %atomic.fmin = atomicrmw fmin ptr %x, <2 x half> %val seq_cst
%atomic.fmin = atomicrmw fmin ptr %x, <2 x half> %val seq_cst

; CHECK: %atomic.fmaximum = atomicrmw fmaximum ptr %x, <2 x half> %val seq_cst
%atomic.fmaximum = atomicrmw fmaximum ptr %x, <2 x half> %val seq_cst

; CHECK: %atomic.fminimum = atomicrmw fminimum ptr %x, <2 x half> %val seq_cst
%atomic.fminimum = atomicrmw fminimum ptr %x, <2 x half> %val seq_cst

ret void
}
Original file line number Diff line number Diff line change
Expand Up @@ -585,7 +585,7 @@
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_FMINIMUM (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. opcode 212 is aliased to 213
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_FMAXIMUM (opcode {{[0-9]+}}): 1 type index
Expand Down
167 changes: 167 additions & 0 deletions llvm/test/Transforms/AtomicExpand/AArch64/atomicrmw-fp.ll
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,170 @@ define float @test_atomicrmw_fsub_f32(ptr %ptr, float %value) {
ret float %res
}

define float @atomicrmw_fmin_float(ptr %ptr, float %value) {
; CHECK-LABEL: @atomicrmw_fmin_float(
; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VALUE:%.*]])
; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: ret float [[TMP6]]
;
%res = atomicrmw fmin ptr %ptr, float %value seq_cst
ret float %res
}

define float @atomicrmw_fmax_float(ptr %ptr, float %value) {
; CHECK-LABEL: @atomicrmw_fmax_float(
; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VALUE:%.*]])
; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: ret float [[TMP6]]
;
%res = atomicrmw fmax ptr %ptr, float %value seq_cst
ret float %res
}

define double @atomicrmw_fmin_double(ptr %ptr, double %value) {
; CHECK-LABEL: @atomicrmw_fmin_double(
; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR:%.*]], align 8
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE:%.*]])
; CHECK-NEXT: [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
; CHECK-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: ret double [[TMP6]]
;
%res = atomicrmw fmin ptr %ptr, double %value seq_cst
ret double %res
}

define double @atomicrmw_fmax_double(ptr %ptr, double %value) {
; CHECK-LABEL: @atomicrmw_fmax_double(
; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR:%.*]], align 8
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE:%.*]])
; CHECK-NEXT: [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
; CHECK-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: ret double [[TMP6]]
;
%res = atomicrmw fmax ptr %ptr, double %value seq_cst
ret double %res
}

define float @atomicrmw_fminimum_float(ptr %ptr, float %value) {
; CHECK-LABEL: @atomicrmw_fminimum_float(
; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minimum.f32(float [[LOADED]], float [[VALUE:%.*]])
; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: ret float [[TMP6]]
;
%res = atomicrmw fminimum ptr %ptr, float %value seq_cst
ret float %res
}

define float @atomicrmw_fmaximum_float(ptr %ptr, float %value) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also test half and bfloat

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, now added.

; CHECK-LABEL: @atomicrmw_fmaximum_float(
; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maximum.f32(float [[LOADED]], float [[VALUE:%.*]])
; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: ret float [[TMP6]]
;
%res = atomicrmw fmaximum ptr %ptr, float %value seq_cst
ret float %res
}

define double @atomicrmw_fminimum_double(ptr %ptr, double %value) {
; CHECK-LABEL: @atomicrmw_fminimum_double(
; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR:%.*]], align 8
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call double @llvm.minimum.f64(double [[LOADED]], double [[VALUE:%.*]])
; CHECK-NEXT: [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
; CHECK-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: ret double [[TMP6]]
;
%res = atomicrmw fminimum ptr %ptr, double %value seq_cst
ret double %res
}

define double @atomicrmw_fmaximum_double(ptr %ptr, double %value) {
; CHECK-LABEL: @atomicrmw_fmaximum_double(
; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR:%.*]], align 8
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call double @llvm.maximum.f64(double [[LOADED]], double [[VALUE:%.*]])
; CHECK-NEXT: [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
; CHECK-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: ret double [[TMP6]]
;
%res = atomicrmw fmaximum ptr %ptr, double %value seq_cst
ret double %res
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test a vector case like <2 x half> and bfloat

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, now added.

8 changes: 4 additions & 4 deletions llvm/test/Transforms/InstCombine/atomicrmw.ll
Original file line number Diff line number Diff line change
Expand Up @@ -399,7 +399,7 @@ define double @no_sat_fmin_inf(ptr %addr) {

define double @sat_fmaximum_inf(ptr %addr) {
; CHECK-LABEL: @sat_fmaximum_inf(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[ADDR:%.*]], double 0x7FF0000000000000 monotonic, align 8
; CHECK-NEXT: [[RES:%.*]] = atomicrmw fmaximum ptr [[ADDR:%.*]], double 0x7FF0000000000000 monotonic, align 8
; CHECK-NEXT: ret double [[RES]]
;
%res = atomicrmw fmaximum ptr %addr, double 0x7FF0000000000000 monotonic
Expand All @@ -417,7 +417,7 @@ define double @no_sat_fmaximum_inf(ptr %addr) {

define double @sat_fminimum_inf(ptr %addr) {
; CHECK-LABEL: @sat_fminimum_inf(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[ADDR:%.*]], double 0xFFF0000000000000 monotonic, align 8
; CHECK-NEXT: [[RES:%.*]] = atomicrmw fminimum ptr [[ADDR:%.*]], double 0xFFF0000000000000 monotonic, align 8
; CHECK-NEXT: ret double [[RES]]
;
%res = atomicrmw fminimum ptr %addr, double 0xFFF0000000000000 monotonic
Expand Down Expand Up @@ -826,7 +826,7 @@ define double @no_sat_fmin_inf_preserve_md(ptr %addr) {

define double @sat_fmaximum_inf_preserve_md(ptr %addr) {
; CHECK-LABEL: @sat_fmaximum_inf_preserve_md(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[ADDR:%.*]], double 0x7FF0000000000000 syncscope("agent") monotonic, align 8, !mmra [[META0]], !amdgpu.no.fine.grained.host.memory [[META1]], !amdgpu.no.remote.memory.access [[META1]]
; CHECK-NEXT: [[RES:%.*]] = atomicrmw fmaximum ptr [[ADDR:%.*]], double 0x7FF0000000000000 syncscope("agent") monotonic, align 8, !mmra [[META0]], !amdgpu.no.fine.grained.host.memory [[META1]], !amdgpu.no.remote.memory.access [[META1]]
; CHECK-NEXT: ret double [[RES]]
;
%res = atomicrmw fmaximum ptr %addr, double 0x7FF0000000000000 syncscope("agent") monotonic, !amdgpu.no.fine.grained.host.memory !0, !amdgpu.no.remote.memory.access !0, !mmra !1
Expand All @@ -844,7 +844,7 @@ define double @no_sat_fmaximum_inf_preserve_md(ptr %addr) {

define double @sat_fminimum_inf_preserve_md(ptr %addr) {
; CHECK-LABEL: @sat_fminimum_inf_preserve_md(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[ADDR:%.*]], double 0xFFF0000000000000 syncscope("agent") monotonic, align 8, !mmra [[META0]], !amdgpu.no.fine.grained.host.memory [[META1]], !amdgpu.no.remote.memory.access [[META1]]
; CHECK-NEXT: [[RES:%.*]] = atomicrmw fminimum ptr [[ADDR:%.*]], double 0xFFF0000000000000 syncscope("agent") monotonic, align 8, !mmra [[META0]], !amdgpu.no.fine.grained.host.memory [[META1]], !amdgpu.no.remote.memory.access [[META1]]
; CHECK-NEXT: ret double [[RES]]
;
%res = atomicrmw fminimum ptr %addr, double 0xFFF0000000000000 syncscope("agent") monotonic, !amdgpu.no.fine.grained.host.memory !0, !amdgpu.no.remote.memory.access !0, !mmra !1
Expand Down
21 changes: 21 additions & 0 deletions llvm/test/Transforms/LowerAtomic/atomic-load.ll
Original file line number Diff line number Diff line change
Expand Up @@ -100,3 +100,24 @@ define float @fminimum() {
ret float %j
; CHECK: ret float [[INST]]
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Test a vector case too, e.g <2 x half>

Copy link
Contributor Author

@jthackray jthackray Apr 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the suggestion, done.


define <2 x half> @fmaximum_2xhalf(<2 x half> %val) {
; CHECK-LABEL: @fmaximum_2xhalf(
%i = alloca <2 x half>, align 4
%j = atomicrmw fmaximum ptr %i, <2 x half> %val monotonic
; CHECK: [[INST:%[a-z0-9]+]] = load
; CHECK-NEXT: call <2 x half> @llvm.maximum.v2f16
; CHECK-NEXT: store
ret <2 x half> %j
}

define <2 x half> @fminimum_2xhalf(<2 x half> %val) {
; CHECK-LABEL: @fminimum_2xhalf(
%i = alloca <2 x half>, align 4
%j = atomicrmw fminimum ptr %i, <2 x half> %val monotonic
; CHECK: [[INST:%[a-z0-9]+]] = load
; CHECK-NEXT: call <2 x half> @llvm.minimum.v2f16
; CHECK-NEXT: store
ret <2 x half> %j
}

Loading