Skip to content

Commit dc51835

Browse files
bob80905bob80905
authored
Add comparison rule, tolerance if applicable, and hex64 view to test failure text output (#291)
This PR implements some extra quality of life to address confusion in seemingly identical actual and expected results. Fixes #289 Here's some sample output of what I see when I force a test to fail, manually changing expected values. <img width="1092" height="755" alt="image" src="https://github.com/user-attachments/assets/238cef2a-728b-4b9d-9b7c-d591c703761f" />
1 parent a60d95f commit dc51835

File tree

4 files changed

+202
-40
lines changed

4 files changed

+202
-40
lines changed

‎lib/Support/Check.cpp

Lines changed: 94 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@
1010
//===----------------------------------------------------------------------===//
1111

1212
#include "Support/Check.h"
13+
#include "Support/Pipeline.h"
1314
#include "llvm/ADT/APFloat.h"
1415
#include "llvm/ADT/APInt.h"
1516
#include "llvm/Support/Error.h"
1617
#include "llvm/Support/raw_ostream.h"
1718
#include <cmath>
19+
#include <sstream>
1820

1921
constexpr uint16_t Float16BitSign = 0x8000;
2022
constexpr uint16_t Float16BitExp = 0x7c00;
@@ -267,30 +269,119 @@ static bool testBufferFloatULP(offloadtest::Buffer *B1, offloadtest::Buffer *B2,
267269
return false;
268270
}
269271

272+
template <typename T>
273+
static std::string bitPatternAsHex64(const T &Val,
274+
offloadtest::Rule ComparisonRule) {
275+
static_assert(sizeof(T) <= sizeof(uint64_t), "Type too large for Hex64");
276+
277+
std::ostringstream Oss;
278+
if (ComparisonRule == offloadtest::Rule::BufferExact)
279+
Oss << "0x" << std::hex << Val;
280+
else
281+
Oss << std::hexfloat << Val;
282+
return Oss.str();
283+
}
284+
285+
template <typename T>
286+
static std::string formatBuffer(offloadtest::Buffer *B,
287+
offloadtest::Rule Rule) {
288+
const llvm::MutableArrayRef<T> Arr(reinterpret_cast<T *>(B->Data.get()),
289+
B->Size / sizeof(T));
290+
if (Arr.empty())
291+
return "";
292+
293+
std::string Result = "[ " + bitPatternAsHex64(Arr[0], Rule);
294+
for (size_t I = 1; I < Arr.size(); ++I)
295+
Result += ", " + bitPatternAsHex64(Arr[I], Rule);
296+
Result += " ]";
297+
return Result;
298+
}
299+
300+
static const std::string getBufferStr(offloadtest::Buffer *B,
301+
offloadtest::Rule Rule) {
302+
using DF = offloadtest::DataFormat;
303+
switch (B->Format) {
304+
case DF::Hex8:
305+
return formatBuffer<llvm::yaml::Hex8>(B, Rule);
306+
case DF::Hex16:
307+
return formatBuffer<llvm::yaml::Hex16>(B, Rule);
308+
case DF::Hex32:
309+
return formatBuffer<llvm::yaml::Hex32>(B, Rule);
310+
case DF::Hex64:
311+
return formatBuffer<llvm::yaml::Hex64>(B, Rule);
312+
case DF::UInt16:
313+
return formatBuffer<uint16_t>(B, Rule);
314+
case DF::UInt32:
315+
return formatBuffer<uint32_t>(B, Rule);
316+
case DF::UInt64:
317+
return formatBuffer<uint64_t>(B, Rule);
318+
case DF::Int16:
319+
return formatBuffer<int16_t>(B, Rule);
320+
case DF::Int32:
321+
return formatBuffer<int32_t>(B, Rule);
322+
case DF::Int64:
323+
return formatBuffer<int64_t>(B, Rule);
324+
case DF::Float16:
325+
return formatBuffer<llvm::yaml::Hex16>(B,
326+
Rule); // assuming no native float16
327+
case DF::Float32:
328+
return formatBuffer<float>(B, Rule);
329+
case DF::Float64:
330+
return formatBuffer<double>(B, Rule);
331+
case DF::Bool:
332+
return formatBuffer<uint32_t>(B,
333+
Rule); // Because sizeof(bool) is 1 but HLSL
334+
// represents a bool using 4 bytes.
335+
}
336+
}
337+
270338
llvm::Error verifyResult(offloadtest::Result R) {
339+
llvm::SmallString<256> Str;
340+
llvm::raw_svector_ostream OS(Str);
341+
OS << "Test failed: " << R.Name << "\n";
342+
271343
switch (R.ComparisonRule) {
272344
case offloadtest::Rule::BufferExact: {
273345
if (testBufferExact(R.ActualPtr, R.ExpectedPtr))
274346
return llvm::Error::success();
347+
OS << "Comparison Rule: BufferExact\n";
275348
break;
276349
}
277350
case offloadtest::Rule::BufferFloatULP: {
278351
if (testBufferFloatULP(R.ActualPtr, R.ExpectedPtr, R.ULPT, R.DM))
279352
return llvm::Error::success();
353+
OS << "Comparison Rule: BufferFloatULP\nULP: " << R.ULPT << "\n";
280354
break;
281355
}
282356
case offloadtest::Rule::BufferFloatEpsilon: {
283357
if (testBufferFloatEpsilon(R.ActualPtr, R.ExpectedPtr, R.Epsilon, R.DM))
284358
return llvm::Error::success();
359+
360+
std::ostringstream Oss;
361+
Oss << std::defaultfloat << R.Epsilon;
362+
OS << "Comparison Rule: BufferFloatEpsilon\nEpsilon: " << Oss.str() << "\n";
285363
break;
286364
}
287365
}
288-
llvm::SmallString<256> Str;
289-
llvm::raw_svector_ostream OS(Str);
290-
OS << "Test failed: " << R.Name << "\nExpected:\n";
366+
367+
OS << "Expected:\n";
291368
llvm::yaml::Output YAMLOS(OS);
292369
YAMLOS << *R.ExpectedPtr;
293370
OS << "Got:\n";
294371
YAMLOS << *R.ActualPtr;
372+
373+
// Now print exact hex64 representations of each element of the
374+
// actual and expected buffers.
375+
376+
const std::string ExpectedBufferStr =
377+
getBufferStr(R.ExpectedPtr, R.ComparisonRule);
378+
const std::string ActualBufferStr =
379+
getBufferStr(R.ActualPtr, R.ComparisonRule);
380+
381+
OS << "Full Hex 64bit representation of Expected Buffer Values:\n"
382+
<< ExpectedBufferStr << "\n";
383+
OS << "Full Hex 64bit representation of Actual Buffer Values:\n"
384+
<< ActualBufferStr << "\n";
385+
295386
return llvm::createStringError(Str.c_str());
296387
}

‎lib/Support/Pipeline.cpp

Lines changed: 55 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,31 @@ void MappingTraits<offloadtest::DescriptorSet>::mapping(
9393
I.mapRequired("Resources", D.Resources);
9494
}
9595

96+
template <typename Type> static void setData(IO &I, offloadtest::Buffer &B) {
97+
if (I.outputting()) {
98+
llvm::MutableArrayRef<Type> Arr(reinterpret_cast<Type *>(B.Data.get()),
99+
B.Size / sizeof(Type));
100+
I.mapRequired("Data", Arr);
101+
} else {
102+
int64_t ZeroInitSize;
103+
I.mapOptional("ZeroInitSize", ZeroInitSize, 0);
104+
if (ZeroInitSize > 0) {
105+
B.Size = ZeroInitSize;
106+
B.Data.reset(new char[B.Size]);
107+
memset(B.Data.get(), 0, B.Size);
108+
I.mapOptional("OutputProps", B.OutputProps);
109+
return;
110+
}
111+
llvm::SmallVector<Type, 64> Arr;
112+
I.mapRequired("Data", Arr);
113+
B.Size = Arr.size() * sizeof(Type);
114+
B.Data.reset(new char[B.Size]);
115+
memcpy(B.Data.get(), Arr.data(), B.Size);
116+
}
117+
118+
I.mapOptional("OutputProps", B.OutputProps);
119+
}
120+
96121
void MappingTraits<offloadtest::Buffer>::mapping(IO &I,
97122
offloadtest::Buffer &B) {
98123
I.mapRequired("Name", B.Name);
@@ -102,44 +127,37 @@ void MappingTraits<offloadtest::Buffer>::mapping(IO &I,
102127
I.mapOptional("Counter", B.Counter, 0);
103128
if (!I.outputting() && B.Stride != 0 && B.Channels != 1)
104129
I.setError("Cannot set a structure stride and more than one channel.");
130+
using DF = offloadtest::DataFormat;
105131
switch (B.Format) {
106-
#define DATA_CASE(Enum, Type) \
107-
case DataFormat::Enum: { \
108-
if (I.outputting()) { \
109-
llvm::MutableArrayRef<Type> Arr(reinterpret_cast<Type *>(B.Data.get()), \
110-
B.Size / sizeof(Type)); \
111-
I.mapRequired("Data", Arr); \
112-
} else { \
113-
int64_t ZeroInitSize; \
114-
I.mapOptional("ZeroInitSize", ZeroInitSize, 0); \
115-
if (ZeroInitSize > 0) { \
116-
B.Size = ZeroInitSize; \
117-
B.Data.reset(new char[B.Size]); \
118-
memset(B.Data.get(), 0, B.Size); \
119-
break; \
120-
} \
121-
llvm::SmallVector<Type, 64> Arr; \
122-
I.mapRequired("Data", Arr); \
123-
B.Size = Arr.size() * sizeof(Type); \
124-
B.Data.reset(new char[B.Size]); \
125-
memcpy(B.Data.get(), Arr.data(), B.Size); \
126-
} \
127-
break; \
128-
}
129-
DATA_CASE(Hex8, llvm::yaml::Hex8)
130-
DATA_CASE(Hex16, llvm::yaml::Hex16)
131-
DATA_CASE(Hex32, llvm::yaml::Hex32)
132-
DATA_CASE(Hex64, llvm::yaml::Hex64)
133-
DATA_CASE(UInt16, uint16_t)
134-
DATA_CASE(UInt32, uint32_t)
135-
DATA_CASE(UInt64, uint64_t)
136-
DATA_CASE(Int16, int16_t)
137-
DATA_CASE(Int32, int32_t)
138-
DATA_CASE(Int64, int64_t)
139-
DATA_CASE(Float16, llvm::yaml::Hex16)
140-
DATA_CASE(Float32, float)
141-
DATA_CASE(Float64, double)
142-
DATA_CASE(Bool, uint32_t) // Because sizeof(bool) is 1 but HLSL represents a bool using 4 bytes.
132+
case DF::Hex8:
133+
return setData<llvm::yaml::Hex8>(I, B);
134+
case DF::Hex16:
135+
return setData<llvm::yaml::Hex16>(I, B);
136+
case DF::Hex32:
137+
return setData<llvm::yaml::Hex32>(I, B);
138+
case DF::Hex64:
139+
return setData<llvm::yaml::Hex64>(I, B);
140+
case DF::UInt16:
141+
return setData<uint16_t>(I, B);
142+
case DF::UInt32:
143+
return setData<uint32_t>(I, B);
144+
case DF::UInt64:
145+
return setData<uint64_t>(I, B);
146+
case DF::Int16:
147+
return setData<int16_t>(I, B);
148+
case DF::Int32:
149+
return setData<int32_t>(I, B);
150+
case DF::Int64:
151+
return setData<int64_t>(I, B);
152+
case DF::Float16:
153+
return setData<llvm::yaml::Hex16>(I, B); // assuming no native float16
154+
case DF::Float32:
155+
return setData<float>(I, B);
156+
case DF::Float64:
157+
return setData<double>(I, B);
158+
case DF::Bool:
159+
return setData<uint32_t>(I, B); // Because sizeof(bool) is 1 but HLSL
160+
// represents a bool using 4 bytes.
143161
}
144162

145163
I.mapOptional("OutputProps", B.OutputProps);

‎test/Tools/Offloader/BufferExact-error.test

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ DescriptorSets:
4848
# RUN: not %offloader %t/pipeline.yaml %t.o 2>&1 | FileCheck %s
4949

5050
# CHECK: Test failed: Test1
51+
# CHECK: Comparison Rule: BufferExact
5152
# CHECK: Expected:
5253
# CHECK: ---
5354
# CHECK: Name: Expected1
@@ -69,3 +70,7 @@ DescriptorSets:
6970
# CHECK: Height: 0
7071
# CHECK: Width: 0
7172
# CHECK: Depth: 0
73+
# CHECK: Full Hex 64bit representation of Expected Buffer Values:
74+
# CHECK: [ 0x1, 0x2, 0x3, 0x4 ]
75+
# CHECK: Full Hex 64bit representation of Actual Buffer Values:
76+
# CHECK: [ 0x14, 0x1e, 0x28, 0x32 ]

‎test/Tools/Offloader/BufferFloat-error-64bit.test

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,8 @@ DescriptorSets:
110110
# RUN: not %offloader %t/pipeline.yaml %t.o 2>&1 | FileCheck %s
111111

112112
# CHECK: Test failed: Test1
113+
# CHECK: Comparison Rule: BufferFloatULP
114+
# CHECK: ULP: 1
113115
# CHECK: Expected:
114116
# CHECK: ---
115117
# CHECK: Name: Expected1
@@ -131,8 +133,18 @@ DescriptorSets:
131133
# CHECK: Height: 0
132134
# CHECK: Width: 0
133135
# CHECK: Depth: 0
136+
# CHECK: ...
137+
# CHECK-NEXT: Full Hex 64bit representation of Expected Buffer Values:
138+
# CHECK-NEXT: [ 0x1.8000000000000p+0, 0x1.4000000000000p+1 ]
139+
# CHECK-NEXT: Full Hex 64bit representation of Actual Buffer Values:
140+
# CHECK-NEXT: [ 0x1.44cccc
141+
# The rest is #ccccccdp+4, 0x1.4000000000000p+2 ], but some implementations
142+
# have trailing 0's for the remaining hex64 data. So, we resume checking from p+4
143+
# CHECK: p+4, 0x1.4000000000000p+2 ]
134144

135145
# CHECK: Test failed: Test2
146+
# CHECK: Comparison Rule: BufferFloatULP
147+
# CHECK: ULP: 1
136148
# CHECK: Expected:
137149
# CHECK: ---
138150
# CHECK: Name: Expected2
@@ -154,8 +166,15 @@ DescriptorSets:
154166
# CHECK: Height: 0
155167
# CHECK: Width: 0
156168
# CHECK: Depth: 0
169+
# CHECK: ...
170+
# CHECK-NEXT: Full Hex 64bit representation of Expected Buffer Values:
171+
# CHECK-NEXT: [ 0x0.fffffffffffffp-1022 ]
172+
# CHECK-NEXT: Full Hex 64bit representation of Actual Buffer Values:
173+
# CHECK-NEXT: [ 0x0.0000000000000p+0 ]
157174

158175
# CHECK: Test failed: Test3
176+
# CHECK: Comparison Rule: BufferFloatULP
177+
# CHECK: ULP: 0
159178
# CHECK: Expected:
160179
# CHECK: ---
161180
# CHECK: Name: Expected3
@@ -177,8 +196,15 @@ DescriptorSets:
177196
# CHECK: Height: 0
178197
# CHECK: Width: 0
179198
# CHECK: Depth: 0
199+
# CHECK: ...
200+
# CHECK-NEXT: Full Hex 64bit representation of Expected Buffer Values:
201+
# CHECK-NEXT: [ 0x0.0000000000000p+0 ]
202+
# CHECK-NEXT: Full Hex 64bit representation of Actual Buffer Values:
203+
# CHECK-NEXT: [ nan ]
180204

181205
# CHECK: Test failed: Test4
206+
# CHECK: Comparison Rule: BufferFloatEpsilon
207+
# CHECK: Epsilon: 0.1
182208
# CHECK: Expected:
183209
# CHECK: ---
184210
# CHECK: Name: Expected1
@@ -200,8 +226,18 @@ DescriptorSets:
200226
# CHECK: Height: 0
201227
# CHECK: Width: 0
202228
# CHECK: Depth: 0
229+
# CHECK: ...
230+
# CHECK-NEXT: Full Hex 64bit representation of Expected Buffer Values:
231+
# CHECK-NEXT: [ 0x1.8000000000000p+0, 0x1.4000000000000p+1 ]
232+
# CHECK-NEXT: Full Hex 64bit representation of Actual Buffer Values:
233+
# CHECK-NEXT: [ 0x1.44cccc
234+
# The rest is #ccccccdp+4, 0x1.4000000000000p+2 ], but some implementations
235+
# have trailing 0's for the remaining hex64 data. So, we resume checking from p+4
236+
# CHECK: p+4, 0x1.4000000000000p+2 ]
203237

204238
# CHECK: Test failed: Test5
239+
# CHECK: Comparison Rule: BufferFloatEpsilon
240+
# CHECK: Epsilon: 0
205241
# CHECK: Expected:
206242
# CHECK: ---
207243
# CHECK: Name: Expected2
@@ -223,8 +259,15 @@ DescriptorSets:
223259
# CHECK: Height: 0
224260
# CHECK: Width: 0
225261
# CHECK: Depth: 0
262+
# CHECK: ...
263+
# CHECK-NEXT: Full Hex 64bit representation of Expected Buffer Values:
264+
# CHECK-NEXT: [ 0x0.fffffffffffffp-1022 ]
265+
# CHECK-NEXT: Full Hex 64bit representation of Actual Buffer Values:
266+
# CHECK-NEXT: [ 0x0.0000000000000p+0 ]
226267

227268
# CHECK: Test failed: Test6
269+
# CHECK: Comparison Rule: BufferFloatEpsilon
270+
# CHECK: Epsilon: 0
228271
# CHECK: Expected:
229272
# CHECK: ---
230273
# CHECK: Name: Expected3
@@ -246,3 +289,8 @@ DescriptorSets:
246289
# CHECK: Height: 0
247290
# CHECK: Width: 0
248291
# CHECK: Depth: 0
292+
# CHECK: ...
293+
# CHECK-NEXT: Full Hex 64bit representation of Expected Buffer Values:
294+
# CHECK-NEXT: [ 0x0.0000000000000p+0 ]
295+
# CHECK-NEXT: Full Hex 64bit representation of Actual Buffer Values:
296+
# CHECK-NEXT: [ nan ]

0 commit comments

Comments
 (0)