Skip to content

GH-46739: [C++] Fix Float16 signed zero/NaN equality comparisons #46973

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pitrou merged 15 commits into from
Sep 4, 2025
Merged

GH-46739: [C++] Fix Float16 signed zero/NaN equality comparisons #46973

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
6195178
Add example tests mentioned in issue
benibus Jul 2, 2025
93b8a0a
Fix Float16 zero/NaN comparisons
benibus Jul 2, 2025
187877b
Exclude Float16 NaNs from being randomly generated
benibus Jul 4, 2025
3ee9649
Update scalar/array tests
benibus Jul 5, 2025
42cc849
Move/revise GetFloat test helper
benibus Jul 8, 2025
7a730c5
Enable constructing HalfFloatScalar from Float16
benibus Jul 10, 2025
08d45d1
Test HalfFloat in TestNumericScalar
benibus Jul 10, 2025
8c8ad06
Fix params for RandomArrayGenerator::Float16
benibus Jul 10, 2025
e3a3688
Use is_half_float_type
benibus Jul 10, 2025
109fe8c
Fix MSVC warnings
benibus Jul 11, 2025
4e69d1c
Improve random HalfFloat generation
benibus Sep 2, 2025
2e53f4a
Address additional review points
benibus Sep 2, 2025
d038d96
Remove RealToCType
benibus Sep 2, 2025
fb60726
Replace NumericHelper
benibus Sep 4, 2025
116b975
Remove leftover debug logs
benibus Sep 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 30 additions & 20 deletions cpp/src/arrow/array/array_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2120,16 +2120,21 @@ void CheckSliceApproxEquals() {
ASSERT_TRUE(slice1->ApproxEquals(slice2));
}

template <typename ArrowType>
using NumericArgType = std::conditional_t<is_half_float_type<ArrowType>::value, Float16,
typename ArrowType::c_type>;

template <typename TYPE>
void CheckFloatingNanEquality() {
using V = NumericArgType<TYPE>;
std::shared_ptr<Array> a, b;
std::shared_ptr<DataType> type = TypeTraits<TYPE>::type_singleton();

const auto nan_value = static_cast<typename TYPE::c_type>(NAN);
const auto nan_value = std::numeric_limits<V>::quiet_NaN();

// NaN in a null entry
ArrayFromVector<TYPE>(type, {true, false}, {0.5, nan_value}, &a);
ArrayFromVector<TYPE>(type, {true, false}, {0.5, nan_value}, &b);
ArrayFromVector<TYPE, V>(type, {true, false}, {V(0.5), nan_value}, &a);
ArrayFromVector<TYPE, V>(type, {true, false}, {V(0.5), nan_value}, &b);
ASSERT_TRUE(a->Equals(b));
ASSERT_TRUE(b->Equals(a));
ASSERT_TRUE(a->ApproxEquals(b));
Expand All @@ -2140,8 +2145,8 @@ void CheckFloatingNanEquality() {
ASSERT_TRUE(b->RangeEquals(a, 1, 2, 1));

// NaN in a valid entry
ArrayFromVector<TYPE>(type, {false, true}, {0.5, nan_value}, &a);
ArrayFromVector<TYPE>(type, {false, true}, {0.5, nan_value}, &b);
ArrayFromVector<TYPE, V>(type, {false, true}, {V(0.5), nan_value}, &a);
ArrayFromVector<TYPE, V>(type, {false, true}, {V(0.5), nan_value}, &b);
ASSERT_FALSE(a->Equals(b));
ASSERT_FALSE(b->Equals(a));
ASSERT_TRUE(a->Equals(b, EqualOptions().nans_equal(true)));
Expand All @@ -2160,8 +2165,8 @@ void CheckFloatingNanEquality() {
ASSERT_TRUE(b->RangeEquals(a, 0, 1, 0));

// NaN != non-NaN
ArrayFromVector<TYPE>(type, {false, true}, {0.5, nan_value}, &a);
ArrayFromVector<TYPE>(type, {false, true}, {0.5, 0.0}, &b);
ArrayFromVector<TYPE, V>(type, {false, true}, {V(0.5), nan_value}, &a);
ArrayFromVector<TYPE, V>(type, {false, true}, {V(0.5), V(0.0)}, &b);
ASSERT_FALSE(a->Equals(b));
ASSERT_FALSE(b->Equals(a));
ASSERT_FALSE(a->Equals(b, EqualOptions().nans_equal(true)));
Expand All @@ -2182,15 +2187,16 @@ void CheckFloatingNanEquality() {

template <typename TYPE>
void CheckFloatingInfinityEquality() {
using V = NumericArgType<TYPE>;
std::shared_ptr<Array> a, b;
std::shared_ptr<DataType> type = TypeTraits<TYPE>::type_singleton();

const auto infinity = std::numeric_limits<typename TYPE::c_type>::infinity();
const auto infinity = std::numeric_limits<V>::infinity();

for (auto nans_equal : {false, true}) {
// Infinity in a null entry
ArrayFromVector<TYPE>(type, {true, false}, {0.5, infinity}, &a);
ArrayFromVector<TYPE>(type, {true, false}, {0.5, -infinity}, &b);
ArrayFromVector<TYPE, V>(type, {true, false}, {V(0.5), infinity}, &a);
ArrayFromVector<TYPE, V>(type, {true, false}, {V(0.5), -infinity}, &b);
ASSERT_TRUE(a->Equals(b));
ASSERT_TRUE(b->Equals(a));
ASSERT_TRUE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
Expand All @@ -2201,8 +2207,8 @@ void CheckFloatingInfinityEquality() {
ASSERT_TRUE(b->RangeEquals(a, 1, 2, 1));

// Infinity in a valid entry
ArrayFromVector<TYPE>(type, {false, true}, {0.5, infinity}, &a);
ArrayFromVector<TYPE>(type, {false, true}, {0.5, infinity}, &b);
ArrayFromVector<TYPE, V>(type, {false, true}, {V(0.5), infinity}, &a);
ArrayFromVector<TYPE, V>(type, {false, true}, {V(0.5), infinity}, &b);
ASSERT_TRUE(a->Equals(b));
ASSERT_TRUE(b->Equals(a));
ASSERT_TRUE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
Expand All @@ -2219,17 +2225,17 @@ void CheckFloatingInfinityEquality() {
ASSERT_TRUE(b->RangeEquals(a, 0, 1, 0));

// Infinity != non-infinity
ArrayFromVector<TYPE>(type, {false, true}, {0.5, -infinity}, &a);
ArrayFromVector<TYPE>(type, {false, true}, {0.5, 0.0}, &b);
ArrayFromVector<TYPE, V>(type, {false, true}, {V(0.5), -infinity}, &a);
ArrayFromVector<TYPE, V>(type, {false, true}, {V(0.5), V(0.0)}, &b);
ASSERT_FALSE(a->Equals(b));
ASSERT_FALSE(b->Equals(a));
ASSERT_FALSE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
ASSERT_FALSE(b->ApproxEquals(a));
ASSERT_FALSE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
ASSERT_FALSE(b->ApproxEquals(a, EqualOptions().atol(1e-5).nans_equal(nans_equal)));
// Infinity != Negative infinity
ArrayFromVector<TYPE>(type, {true, true}, {0.5, -infinity}, &a);
ArrayFromVector<TYPE>(type, {true, true}, {0.5, infinity}, &b);
ArrayFromVector<TYPE, V>(type, {true, true}, {V(0.5), -infinity}, &a);
ArrayFromVector<TYPE, V>(type, {true, true}, {V(0.5), infinity}, &b);
ASSERT_FALSE(a->Equals(b));
ASSERT_FALSE(b->Equals(a));
ASSERT_FALSE(a->ApproxEquals(b));
Expand All @@ -2249,11 +2255,12 @@ void CheckFloatingInfinityEquality() {

template <typename TYPE>
void CheckFloatingZeroEquality() {
using V = NumericArgType<TYPE>;
std::shared_ptr<Array> a, b;
std::shared_ptr<DataType> type = TypeTraits<TYPE>::type_singleton();

ArrayFromVector<TYPE>(type, {true, false}, {0.0, 1.0}, &a);
ArrayFromVector<TYPE>(type, {true, false}, {0.0, 1.0}, &b);
ArrayFromVector<TYPE, V>(type, {true, false}, {V(0.0), V(1.0)}, &a);
ArrayFromVector<TYPE, V>(type, {true, false}, {V(0.0), V(1.0)}, &b);
ASSERT_TRUE(a->Equals(b));
ASSERT_TRUE(b->Equals(a));
for (auto nans_equal : {false, true}) {
Expand All @@ -2269,8 +2276,8 @@ void CheckFloatingZeroEquality() {
}
}

ArrayFromVector<TYPE>(type, {true, false}, {0.0, 1.0}, &a);
ArrayFromVector<TYPE>(type, {true, false}, {-0.0, 1.0}, &b);
ArrayFromVector<TYPE, V>(type, {true, false}, {V(0.0), V(1.0)}, &a);
ArrayFromVector<TYPE, V>(type, {true, false}, {V(-0.0), V(1.0)}, &b);
for (auto nans_equal : {false, true}) {
auto opts = EqualOptions().nans_equal(nans_equal);
ASSERT_TRUE(a->Equals(b, opts));
Expand Down Expand Up @@ -2306,16 +2313,19 @@ TEST(TestPrimitiveAdHoc, FloatingSliceApproxEquals) {
TEST(TestPrimitiveAdHoc, FloatingNanEquality) {
CheckFloatingNanEquality<FloatType>();
CheckFloatingNanEquality<DoubleType>();
CheckFloatingNanEquality<HalfFloatType>();
}

TEST(TestPrimitiveAdHoc, FloatingInfinityEquality) {
CheckFloatingInfinityEquality<FloatType>();
CheckFloatingInfinityEquality<DoubleType>();
CheckFloatingInfinityEquality<HalfFloatType>();
}

TEST(TestPrimitiveAdHoc, FloatingZeroEquality) {
CheckFloatingZeroEquality<FloatType>();
CheckFloatingZeroEquality<DoubleType>();
CheckFloatingZeroEquality<HalfFloatType>();
}

// ----------------------------------------------------------------------
Expand Down
5 changes: 3 additions & 2 deletions cpp/src/arrow/compare.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ struct FloatingEquality<uint16_t, Flags> {
bool operator()(uint16_t x, uint16_t y) const {
Float16 f_x = Float16::FromBits(x);
Float16 f_y = Float16::FromBits(y);
if (x == y) {
if (f_x == f_y) {
return Flags::signed_zeros_equal || (f_x.signbit() == f_y.signbit());
}
if (Flags::nans_equal && f_x.is_nan() && f_y.is_nan()) {
Expand Down Expand Up @@ -171,7 +171,8 @@ void VisitFloatingEquality(const EqualOptions& options, bool floating_approximat
}

inline bool IdentityImpliesEqualityNansNotEqual(const DataType& type) {
if (type.id() == Type::FLOAT || type.id() == Type::DOUBLE) {
if (type.id() == Type::FLOAT || type.id() == Type::DOUBLE ||
type.id() == Type::HALF_FLOAT) {
return false;
}
for (const auto& child : type.fields()) {
Expand Down
19 changes: 19 additions & 0 deletions cpp/src/arrow/scalar.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
#include "arrow/type_traits.h"
#include "arrow/util/compare.h"
#include "arrow/util/decimal.h"
#include "arrow/util/float16.h"
#include "arrow/util/visibility.h"
#include "arrow/visit_type_inline.h"

Expand Down Expand Up @@ -245,6 +246,12 @@ struct ARROW_EXPORT UInt64Scalar : public NumericScalar<UInt64Type> {

struct ARROW_EXPORT HalfFloatScalar : public NumericScalar<HalfFloatType> {
using NumericScalar<HalfFloatType>::NumericScalar;

explicit HalfFloatScalar(util::Float16 value)
: NumericScalar(value.bits(), float16()) {}

HalfFloatScalar(util::Float16 value, std::shared_ptr<DataType> type)
: NumericScalar(value.bits(), std::move(type)) {}
};

struct ARROW_EXPORT FloatScalar : public NumericScalar<FloatType> {
Expand Down Expand Up @@ -969,6 +976,18 @@ struct MakeScalarImpl {
return Status::OK();
}

// This isn't captured by the generic case above because `util::Float16` isn't implicity
// convertible to `uint16_t` (HalfFloat's ValueType)
template <typename T>
std::enable_if_t<std::is_same_v<std::decay_t<ValueRef>, util::Float16> &&
is_half_float_type<T>::value,
Status>
Visit(const T& t) {
out_ = std::make_shared<HalfFloatScalar>(static_cast<ValueRef>(value_),
std::move(type_));
return Status::OK();
}

Status Visit(const ExtensionType& t) {
ARROW_ASSIGN_OR_RAISE(auto storage,
MakeScalar(t.storage_type(), static_cast<ValueRef>(value_)));
Expand Down
84 changes: 51 additions & 33 deletions cpp/src/arrow/scalar_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,15 @@
#include "arrow/testing/random.h"
#include "arrow/testing/util.h"
#include "arrow/type_traits.h"
#include "arrow/util/float16.h"

namespace arrow {

using compute::Cast;
using compute::CastOptions;
using internal::checked_cast;
using internal::checked_pointer_cast;
using util::Float16;

std::shared_ptr<Scalar> CheckMakeNullScalar(const std::shared_ptr<DataType>& type) {
const auto scalar = MakeNullScalar(type);
Expand Down Expand Up @@ -201,22 +203,33 @@ TEST(TestScalar, IdentityCast) {
*/
}

template <typename ArrowType>
using NumericArgType = std::conditional_t<is_half_float_type<ArrowType>::value, Float16,
typename ArrowType::c_type>;

template <typename T>
class TestNumericScalar : public ::testing::Test {
public:
TestNumericScalar() = default;
};

TYPED_TEST_SUITE(TestNumericScalar, NumericArrowTypes);
using NumericArrowTypesPlusHalfFloat =
testing::Types<UInt8Type, UInt16Type, UInt32Type, UInt64Type, Int8Type, Int16Type,
Int32Type, Int64Type, FloatType, DoubleType, HalfFloatType>;
TYPED_TEST_SUITE(TestNumericScalar, NumericArrowTypesPlusHalfFloat);

TYPED_TEST(TestNumericScalar, Basics) {
using T = typename TypeParam::c_type;
using T = NumericArgType<TypeParam>;
using ScalarType = typename TypeTraits<TypeParam>::ScalarType;

T value = static_cast<T>(1);

auto scalar_val = std::make_shared<ScalarType>(value);
ASSERT_EQ(value, scalar_val->value);
Copy link
Copy Markdown
Member

@pitrou pitrou Aug 26, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should probably keep the check here by using if constexpr as below?

if constexpr (is_half_float_type<TypeParam>::value) {
ASSERT_EQ(value, Float16::FromBits(scalar_val->value));
} else {
ASSERT_EQ(value, scalar_val->value);
}
ASSERT_TRUE(scalar_val->is_valid);
ASSERT_OK(scalar_val->ValidateFull());

Expand All @@ -227,8 +240,13 @@ TYPED_TEST(TestNumericScalar, Basics) {
auto scalar_other = std::make_shared<ScalarType>(other_value);
ASSERT_NE(*scalar_other, *scalar_val);

scalar_val->value = other_value;
ASSERT_EQ(other_value, scalar_val->value);
if constexpr (is_half_float_type<TypeParam>::value) {
scalar_val->value = other_value.bits();
ASSERT_EQ(other_value, Float16::FromBits(scalar_val->value));
} else {
scalar_val->value = other_value;
ASSERT_EQ(other_value, scalar_val->value);
}
ASSERT_EQ(*scalar_other, *scalar_val);

ScalarType stack_val;
Expand All @@ -255,72 +273,72 @@ TYPED_TEST(TestNumericScalar, Basics) {
ASSERT_OK(two->ValidateFull());

ASSERT_TRUE(null->Equals(*null_value));
ASSERT_TRUE(one->Equals(ScalarType(1)));
ASSERT_FALSE(one->Equals(ScalarType(2)));
ASSERT_TRUE(two->Equals(ScalarType(2)));
ASSERT_FALSE(two->Equals(ScalarType(3)));
ASSERT_TRUE(one->Equals(ScalarType(static_cast<T>(1))));
ASSERT_FALSE(one->Equals(ScalarType(static_cast<T>(2))));
ASSERT_TRUE(two->Equals(ScalarType(static_cast<T>(2))));
ASSERT_FALSE(two->Equals(ScalarType(static_cast<T>(3))));

ASSERT_TRUE(null->ApproxEquals(*null_value));
ASSERT_TRUE(one->ApproxEquals(ScalarType(1)));
ASSERT_FALSE(one->ApproxEquals(ScalarType(2)));
ASSERT_TRUE(two->ApproxEquals(ScalarType(2)));
ASSERT_FALSE(two->ApproxEquals(ScalarType(3)));
ASSERT_TRUE(one->ApproxEquals(ScalarType(static_cast<T>(1))));
ASSERT_FALSE(one->ApproxEquals(ScalarType(static_cast<T>(2))));
ASSERT_TRUE(two->ApproxEquals(ScalarType(static_cast<T>(2))));
ASSERT_FALSE(two->ApproxEquals(ScalarType(static_cast<T>(3))));
}

TYPED_TEST(TestNumericScalar, Hashing) {
using T = typename TypeParam::c_type;
using T = NumericArgType<TypeParam>;
using ScalarType = typename TypeTraits<TypeParam>::ScalarType;

std::unordered_set<std::shared_ptr<Scalar>, Scalar::Hash, Scalar::PtrsEqual> set;
set.emplace(std::make_shared<ScalarType>());
for (T i = 0; i < 10; ++i) {
set.emplace(std::make_shared<ScalarType>(i));
for (int i = 0; i < 10; ++i) {
ASSERT_TRUE(set.emplace(std::make_shared<ScalarType>(static_cast<T>(i))).second);
}

ASSERT_FALSE(set.emplace(std::make_shared<ScalarType>()).second);
for (T i = 0; i < 10; ++i) {
ASSERT_FALSE(set.emplace(std::make_shared<ScalarType>(i)).second);
for (int i = 0; i < 10; ++i) {
ASSERT_FALSE(set.emplace(std::make_shared<ScalarType>(static_cast<T>(i))).second);
}
}

TYPED_TEST(TestNumericScalar, MakeScalar) {
using T = typename TypeParam::c_type;
using T = NumericArgType<TypeParam>;
using ScalarType = typename TypeTraits<TypeParam>::ScalarType;
auto type = TypeTraits<TypeParam>::type_singleton();

std::shared_ptr<Scalar> three = MakeScalar(static_cast<T>(3));
ASSERT_OK(three->ValidateFull());
ASSERT_EQ(ScalarType(3), *three);
ASSERT_EQ(ScalarType(static_cast<T>(3)), *three);

AssertMakeScalar(ScalarType(3), type, static_cast<T>(3));
AssertMakeScalar(ScalarType(static_cast<T>(3)), type, static_cast<T>(3));

AssertParseScalar(type, "3", ScalarType(3));
AssertParseScalar(type, "3", ScalarType(static_cast<T>(3)));
}

template <typename T>
class TestRealScalar : public ::testing::Test {
public:
using CType = typename T::c_type;
using ValueType = NumericArgType<T>;
using ScalarType = typename TypeTraits<T>::ScalarType;

void SetUp() {
type_ = TypeTraits<T>::type_singleton();

scalar_val_ = std::make_shared<ScalarType>(static_cast<CType>(1));
scalar_val_ = std::make_shared<ScalarType>(static_cast<ValueType>(1));
ASSERT_TRUE(scalar_val_->is_valid);

scalar_other_ = std::make_shared<ScalarType>(static_cast<CType>(1.1));
scalar_other_ = std::make_shared<ScalarType>(static_cast<ValueType>(1.1));
ASSERT_TRUE(scalar_other_->is_valid);

scalar_zero_ = std::make_shared<ScalarType>(static_cast<CType>(0.0));
scalar_other_zero_ = std::make_shared<ScalarType>(static_cast<CType>(0.0));
scalar_neg_zero_ = std::make_shared<ScalarType>(static_cast<CType>(-0.0));
scalar_zero_ = std::make_shared<ScalarType>(static_cast<ValueType>(0.0));
scalar_other_zero_ = std::make_shared<ScalarType>(static_cast<ValueType>(0.0));
scalar_neg_zero_ = std::make_shared<ScalarType>(static_cast<ValueType>(-0.0));

const CType nan_value = std::numeric_limits<CType>::quiet_NaN();
const auto nan_value = std::numeric_limits<ValueType>::quiet_NaN();
scalar_nan_ = std::make_shared<ScalarType>(nan_value);
ASSERT_TRUE(scalar_nan_->is_valid);

const CType other_nan_value = std::numeric_limits<CType>::quiet_NaN();
const auto other_nan_value = std::numeric_limits<ValueType>::quiet_NaN();
scalar_other_nan_ = std::make_shared<ScalarType>(other_nan_value);
ASSERT_TRUE(scalar_other_nan_->is_valid);
}
Expand Down Expand Up @@ -522,7 +540,9 @@ class TestRealScalar : public ::testing::Test {
scalar_zero_, scalar_other_zero_, scalar_neg_zero_;
};

TYPED_TEST_SUITE(TestRealScalar, RealArrowTypes);
using RealArrowTypesPlusHalfFloat =
::testing::Types<FloatType, DoubleType, HalfFloatType>;
TYPED_TEST_SUITE(TestRealScalar, RealArrowTypesPlusHalfFloat);

TYPED_TEST(TestRealScalar, NanEquals) { this->TestNanEquals(); }

Expand Down Expand Up @@ -1181,8 +1201,6 @@ TEST(TestDayTimeIntervalScalars, Basics) {
ASSERT_TRUE(first->Equals(ts_val2));
}

// TODO test HalfFloatScalar

TYPED_TEST(TestNumericScalar, Cast) {
auto type = TypeTraits<TypeParam>::type_singleton();

Expand Down
Loading
Loading