Skip to content

Commit cfa627e

Browse files
authored
Merge branch 'main' into amdgpu-loopalign
2 parents 74c5549 + e0f5e28 commit cfa627e

File tree

27 files changed

+342
-258
lines changed

27 files changed

+342
-258
lines changed

clang/lib/CIR/CodeGen/CIRGenCall.cpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,17 +42,17 @@ CIRGenFunctionInfo::create(CanQualType resultType,
4242
return fi;
4343
}
4444

45-
cir::FuncType CIRGenTypes::getFunctionType(const CIRGenFunctionInfo &fi) {
46-
mlir::Type resultType = convertType(fi.getReturnType());
45+
cir::FuncType CIRGenTypes::getFunctionType(const CIRGenFunctionInfo &info) {
46+
mlir::Type resultType = convertType(info.getReturnType());
4747
SmallVector<mlir::Type, 8> argTypes;
48-
argTypes.reserve(fi.getNumRequiredArgs());
48+
argTypes.reserve(info.getNumRequiredArgs());
4949

50-
for (const CanQualType &argType : fi.requiredArguments())
50+
for (const CanQualType &argType : info.requiredArguments())
5151
argTypes.push_back(convertType(argType));
5252

5353
return cir::FuncType::get(argTypes,
5454
(resultType ? resultType : builder.getVoidTy()),
55-
fi.isVariadic());
55+
info.isVariadic());
5656
}
5757

5858
CIRGenCallee CIRGenCallee::prepareConcreteCallee(CIRGenFunction &cgf) const {

clang/lib/CIR/CodeGen/CIRGenExpr.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1169,7 +1169,8 @@ static void pushTemporaryCleanup(CIRGenFunction &cgf,
11691169
->getBaseElementTypeUnsafe()
11701170
->getAs<clang::RecordType>()) {
11711171
// Get the destructor for the reference temporary.
1172-
auto *classDecl = cast<CXXRecordDecl>(rt->getOriginalDecl());
1172+
auto *classDecl =
1173+
cast<CXXRecordDecl>(rt->getOriginalDecl()->getDefinitionOrSelf());
11731174
if (!classDecl->hasTrivialDestructor())
11741175
referenceTemporaryDtor =
11751176
classDecl->getDefinitionOrSelf()->getDestructor();

clang/lib/CIR/CodeGen/CIRGenFunction.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -356,8 +356,8 @@ static bool mayDropFunctionReturn(const ASTContext &astContext,
356356
// destructor or a non-trivially copyable type.
357357
if (const RecordType *recordType =
358358
returnType.getCanonicalType()->getAs<RecordType>()) {
359-
if (const auto *classDecl =
360-
dyn_cast<CXXRecordDecl>(recordType->getOriginalDecl()))
359+
if (const auto *classDecl = dyn_cast<CXXRecordDecl>(
360+
recordType->getOriginalDecl()->getDefinitionOrSelf()))
361361
return classDecl->hasTrivialDestructor();
362362
}
363363
return returnType.isTriviallyCopyableType(astContext);

clang/lib/CIR/CodeGen/CIRGenTypes.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,8 @@ isSafeToConvert(const RecordDecl *rd, CIRGenTypes &cgt,
139139
if (!alreadyChecked.insert(rd).second)
140140
return true;
141141

142+
assert(rd->isCompleteDefinition() &&
143+
"Expect RecordDecl to be CompleteDefinition");
142144
const Type *key = cgt.getASTContext().getCanonicalTagType(rd).getTypePtr();
143145

144146
// If this type is already laid out, converting it is a noop.

clang/lib/CIR/CodeGen/TargetInfo.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,13 @@ bool clang::CIRGen::isEmptyRecordForLayout(const ASTContext &context,
1717
if (cxxrd->isDynamicClass())
1818
return false;
1919

20-
for (const auto &I : cxxrd->bases())
21-
if (!isEmptyRecordForLayout(context, I.getType()))
20+
for (const auto &i : cxxrd->bases())
21+
if (!isEmptyRecordForLayout(context, i.getType()))
2222
return false;
2323
}
2424

25-
for (const auto *I : rd->fields())
26-
if (!isEmptyFieldForLayout(context, I))
25+
for (const auto *i : rd->fields())
26+
if (!isEmptyFieldForLayout(context, i))
2727
return false;
2828

2929
return true;

libc/shared/math.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "math/cbrt.h"
3434
#include "math/cbrtf.h"
3535
#include "math/cos.h"
36+
#include "math/cosf.h"
3637
#include "math/erff.h"
3738
#include "math/exp.h"
3839
#include "math/exp10.h"

libc/shared/math/cosf.h

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
//===-- Shared cosf function ------------------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_LIBC_SHARED_MATH_COSF_H
10+
#define LLVM_LIBC_SHARED_MATH_COSF_H
11+
12+
#include "shared/libc_common.h"
13+
#include "src/__support/math/cosf.h"
14+
15+
namespace LIBC_NAMESPACE_DECL {
16+
namespace shared {
17+
18+
using math::cosf;
19+
20+
} // namespace shared
21+
} // namespace LIBC_NAMESPACE_DECL
22+
23+
#endif // LLVM_LIBC_SHARED_MATH_COSF_H

libc/src/__support/GPU/allocator.cpp

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,11 @@ static inline uint32_t get_leader_id(uint64_t ballot, uint32_t id) {
166166

167167
// We use a sentinal value to indicate a failed or in-progress allocation.
168168
template <typename T> bool is_sentinel(const T &x) {
169-
return x == cpp::numeric_limits<T>::max();
169+
if constexpr (cpp::is_pointer_v<T>)
170+
return reinterpret_cast<uintptr_t>(x) ==
171+
cpp::numeric_limits<uintptr_t>::max();
172+
else
173+
return x == cpp::numeric_limits<T>::max();
170174
}
171175

172176
} // namespace impl
@@ -446,7 +450,13 @@ struct GuardPtr {
446450
return new (raw) Slab(cpp::forward<Args>(args)...);
447451
}
448452

449-
if (!expected || impl::is_sentinel(reinterpret_cast<uintptr_t>(expected)))
453+
// If there is a slab allocation in progress we retry a few times.
454+
for (uint32_t t = 0; impl::is_sentinel(expected) && t < MAX_TRIES; ++t) {
455+
sleep_briefly();
456+
expected = ptr.load(cpp::MemoryOrder::RELAXED);
457+
}
458+
459+
if (!expected || impl::is_sentinel(expected))
450460
return nullptr;
451461

452462
if (!ref.acquire(n, count))
@@ -557,16 +567,6 @@ static Slab *find_slab(uint32_t chunk_size, uint64_t &uniform,
557567
Slab *slab = slots[index].try_lock(lane_mask, uniform & lane_mask,
558568
reserved, chunk_size, index);
559569

560-
// If there is a slab allocation in progress we retry a few times.
561-
for (uint32_t retries = 0;
562-
!slab && !impl::is_sentinel(reserved) && retries < MAX_TRIES;
563-
retries++) {
564-
uint64_t lane_mask = gpu::get_lane_mask();
565-
slab = slots[index].try_lock(lane_mask, uniform & lane_mask, reserved,
566-
chunk_size, index);
567-
sleep_briefly();
568-
}
569-
570570
// If we find a slab with a matching chunk size then we store the result.
571571
// Otherwise, we need to free the claimed lock and continue. In the case
572572
// of out-of-memory we receive a sentinel value and return a failure.

libc/src/__support/math/CMakeLists.txt

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,21 @@ add_header_library(
374374
libc.src.__support.macros.optimization
375375
)
376376

377+
add_header_library(
378+
cosf
379+
HDRS
380+
cosf.h
381+
DEPENDS
382+
.sincosf_utils
383+
libc.src.errno.errno
384+
libc.src.__support.FPUtil.basic_operations
385+
libc.src.__support.FPUtil.fenv_impl
386+
libc.src.__support.FPUtil.fp_bits
387+
libc.src.__support.FPUtil.except_value_utils
388+
libc.src.__support.FPUtil.fma
389+
libc.src.__support.FPUtil.polyeval
390+
libc.src.__support.macros.optimization
391+
)
377392

378393
add_header_library(
379394
erff
@@ -649,6 +664,19 @@ add_header_library(
649664
libc.src.__support.integer_literals
650665
)
651666

667+
add_header_library(
668+
range_reduction
669+
HDRS
670+
range_reduction.h
671+
range_reduction_fma.h
672+
DEPENDS
673+
libc.src.__support.FPUtil.fp_bits
674+
libc.src.__support.FPUtil.fma
675+
libc.src.__support.FPUtil.multiply_add
676+
libc.src.__support.FPUtil.nearest_integer
677+
libc.src.__support.common
678+
)
679+
652680
add_header_library(
653681
sincos_eval
654682
HDRS
@@ -660,3 +688,14 @@ add_header_library(
660688
libc.src.__support.FPUtil.polyeval
661689
libc.src.__support.integer_literals
662690
)
691+
692+
add_header_library(
693+
sincosf_utils
694+
HDRS
695+
sincosf_utils.h
696+
DEPENDS
697+
.range_reduction
698+
libc.src.__support.FPUtil.fp_bits
699+
libc.src.__support.FPUtil.polyeval
700+
libc.src.__support.common
701+
)

libc/src/__support/math/cosf.h

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
//===-- Implementation header for cosf --------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LIBC_SRC___SUPPORT_MATH_COSF_H
10+
#define LIBC_SRC___SUPPORT_MATH_COSF_H
11+
12+
#include "sincosf_utils.h"
13+
#include "src/__support/FPUtil/FEnvImpl.h"
14+
#include "src/__support/FPUtil/FPBits.h"
15+
#include "src/__support/FPUtil/except_value_utils.h"
16+
#include "src/__support/FPUtil/multiply_add.h"
17+
#include "src/__support/macros/config.h"
18+
#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
19+
#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
20+
21+
namespace LIBC_NAMESPACE_DECL {
22+
23+
namespace math {
24+
25+
LIBC_INLINE static constexpr float cosf(float x) {
26+
27+
#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
28+
// Exceptional cases for cosf.
29+
constexpr size_t N_EXCEPTS = 6;
30+
31+
constexpr fputil::ExceptValues<float, N_EXCEPTS> COSF_EXCEPTS{{
32+
// (inputs, RZ output, RU offset, RD offset, RN offset)
33+
// x = 0x1.64a032p43, cos(x) = 0x1.9d4ba4p-1 (RZ)
34+
{0x55325019, 0x3f4ea5d2, 1, 0, 0},
35+
// x = 0x1.4555p51, cos(x) = 0x1.115d7cp-1 (RZ)
36+
{0x5922aa80, 0x3f08aebe, 1, 0, 1},
37+
// x = 0x1.48a858p54, cos(x) = 0x1.f48148p-2 (RZ)
38+
{0x5aa4542c, 0x3efa40a4, 1, 0, 0},
39+
// x = 0x1.3170fp63, cos(x) = 0x1.fe2976p-1 (RZ)
40+
{0x5f18b878, 0x3f7f14bb, 1, 0, 0},
41+
// x = 0x1.2b9622p67, cos(x) = 0x1.f0285cp-1 (RZ)
42+
{0x6115cb11, 0x3f78142e, 1, 0, 1},
43+
// x = 0x1.ddebdep120, cos(x) = 0x1.114438p-1 (RZ)
44+
{0x7beef5ef, 0x3f08a21c, 1, 0, 0},
45+
}};
46+
#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
47+
48+
using FPBits = typename fputil::FPBits<float>;
49+
50+
FPBits xbits(x);
51+
xbits.set_sign(Sign::POS);
52+
53+
uint32_t x_abs = xbits.uintval();
54+
double xd = static_cast<double>(xbits.get_val());
55+
56+
// Range reduction:
57+
// For |x| > pi/16, we perform range reduction as follows:
58+
// Find k and y such that:
59+
// x = (k + y) * pi/32
60+
// k is an integer
61+
// |y| < 0.5
62+
// For small range (|x| < 2^45 when FMA instructions are available, 2^22
63+
// otherwise), this is done by performing:
64+
// k = round(x * 32/pi)
65+
// y = x * 32/pi - k
66+
// For large range, we will omit all the higher parts of 16/pi such that the
67+
// least significant bits of their full products with x are larger than 63,
68+
// since cos((k + y + 64*i) * pi/32) = cos(x + i * 2pi) = cos(x).
69+
//
70+
// When FMA instructions are not available, we store the digits of 32/pi in
71+
// chunks of 28-bit precision. This will make sure that the products:
72+
// x * THIRTYTWO_OVER_PI_28[i] are all exact.
73+
// When FMA instructions are available, we simply store the digits of 32/pi in
74+
// chunks of doubles (53-bit of precision).
75+
// So when multiplying by the largest values of single precision, the
76+
// resulting output should be correct up to 2^(-208 + 128) ~ 2^-80. By the
77+
// worst-case analysis of range reduction, |y| >= 2^-38, so this should give
78+
// us more than 40 bits of accuracy. For the worst-case estimation of range
79+
// reduction, see for instances:
80+
// Elementary Functions by J-M. Muller, Chapter 11,
81+
// Handbook of Floating-Point Arithmetic by J-M. Muller et. al.,
82+
// Chapter 10.2.
83+
//
84+
// Once k and y are computed, we then deduce the answer by the cosine of sum
85+
// formula:
86+
// cos(x) = cos((k + y)*pi/32)
87+
// = cos(y*pi/32) * cos(k*pi/32) - sin(y*pi/32) * sin(k*pi/32)
88+
// The values of sin(k*pi/32) and cos(k*pi/32) for k = 0..63 are precomputed
89+
// and stored using a vector of 32 doubles. Sin(y*pi/32) and cos(y*pi/32) are
90+
// computed using degree-7 and degree-6 minimax polynomials generated by
91+
// Sollya respectively.
92+
93+
// |x| < 0x1.0p-12f
94+
if (LIBC_UNLIKELY(x_abs < 0x3980'0000U)) {
95+
// When |x| < 2^-12, the relative error of the approximation cos(x) ~ 1
96+
// is:
97+
// |cos(x) - 1| < |x^2 / 2| = 2^-25 < epsilon(1)/2.
98+
// So the correctly rounded values of cos(x) are:
99+
// = 1 - eps(x) if rounding mode = FE_TOWARDZERO or FE_DOWWARD,
100+
// = 1 otherwise.
101+
// To simplify the rounding decision and make it more efficient and to
102+
// prevent compiler to perform constant folding, we use
103+
// fma(x, -2^-25, 1) instead.
104+
// Note: to use the formula 1 - 2^-25*x to decide the correct rounding, we
105+
// do need fma(x, -2^-25, 1) to prevent underflow caused by -2^-25*x when
106+
// |x| < 2^-125. For targets without FMA instructions, we simply use
107+
// double for intermediate results as it is more efficient than using an
108+
// emulated version of FMA.
109+
#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
110+
return fputil::multiply_add(xbits.get_val(), -0x1.0p-25f, 1.0f);
111+
#else
112+
return static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, 1.0));
113+
#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
114+
}
115+
116+
#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
117+
if (auto r = COSF_EXCEPTS.lookup(x_abs); LIBC_UNLIKELY(r.has_value()))
118+
return r.value();
119+
#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
120+
121+
// x is inf or nan.
122+
if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) {
123+
if (xbits.is_signaling_nan()) {
124+
fputil::raise_except_if_required(FE_INVALID);
125+
return FPBits::quiet_nan().get_val();
126+
}
127+
128+
if (x_abs == 0x7f80'0000U) {
129+
fputil::set_errno_if_required(EDOM);
130+
fputil::raise_except_if_required(FE_INVALID);
131+
}
132+
return x + FPBits::quiet_nan().get_val();
133+
}
134+
135+
// Combine the results with the sine of sum formula:
136+
// cos(x) = cos((k + y)*pi/32)
137+
// = cos(y*pi/32) * cos(k*pi/32) - sin(y*pi/32) * sin(k*pi/32)
138+
// = cosm1_y * cos_k + sin_y * sin_k
139+
// = (cosm1_y * cos_k + cos_k) + sin_y * sin_k
140+
double sin_k = 0, cos_k = 0, sin_y = 0, cosm1_y = 0;
141+
142+
sincosf_eval(xd, x_abs, sin_k, cos_k, sin_y, cosm1_y);
143+
144+
return static_cast<float>(fputil::multiply_add(
145+
sin_y, -sin_k, fputil::multiply_add(cosm1_y, cos_k, cos_k)));
146+
}
147+
148+
} // namespace math
149+
150+
} // namespace LIBC_NAMESPACE_DECL
151+
152+
#endif // LIBC_SRC___SUPPORT_MATH_COSF_H

0 commit comments

Comments
 (0)