Enhance offload test suite to compare test results with two new rules 'BufferExact' and 'BufferFuzzy' (#77)

Add support for testing results in the offload test suite.
Add two testing functions, 'BufferExact' and 'BufferFuzzy'
BufferExact tests if two buffers contain exactly the same content, using == on each element.
BufferFuzzy works on only floating point values and performs a ULP comparison on each floating point value in two buffers being compared.
The BufferFuzzy rule requires a ULPT field in the YAML which specifies the allowed ULP difference. It also takes an optional DenormMode field, which defaults to Any.

Add a test each for BufferExact and BufferFuzzy which show what error output looks like.
Closes #76

Author: spall

README.md

@@ -58,6 +58,25 @@ Buffers:
   - Name: In2
     Format: Hex16
     Data: [ 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8]
+  - Name: Out1 # Buffer where our output will go
+    Format: Float32
+    Stride: 4
+    ZeroInitSize: 8
+  - Name: Expected1 # Buffer which stores the expected result of our test
+    Format: Float32
+    Stride: 4
+    Data: [ 0.0, 1.0 ]
+Results: # Using Result can verify test values without filecheck
+  - Result: Test1
+    Rule: BufferFuzzy # Rule which can be used to compare Float Buffers; They are compared within a ULP range
+    ULPT: 1 # ULP to use
+    DenormMode: Any # if DenormMode Field is not Specified, 'Any' is the default; FTZ and Preserve are the other options.
+    Actual: Out1 # First buffer to compare
+    Expected: Expected1 # Second buffer to compare against first
+  - Result: Test2
+    Rule: BufferExact # Compares Two Buffers for == equality between each value elementwise
+    Actual: Out1
+    Expected: Expected1
 DescriptorSets:
   - Resources:
     - Name: Constants

include/Support/Check.h

@@ -0,0 +1,25 @@
+//===- Check.h - Functions for checking test results-------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOADTEST_SUPPORT_CHECK_H
+#define OFFLOADTEST_SUPPORT_CHECK_H
+
+#include "Pipeline.h"
+
+/// verifies an offload test Result
+/// Calls the test, corresponding to the Rule specified in the Result,
+/// On the Actual and Expected Buffers
+/// \param R Result to verify
+/// \returns Success if the test passes according to the specified Rule
+llvm::Error verifyResult(offloadtest::Result R);
+
+#endif // OFFLOADTEST_SUPPORT_CHECK_H

include/Support/Pipeline.h

@@ -25,6 +25,10 @@ namespace offloadtest {
 
 enum class Stages { Compute };
 
+enum class Rule { BufferExact, BufferFuzzy };
+
+enum class DenormMode { Any, FTZ, Preserve };
+
 enum class DataFormat {
   Hex8,
   Hex16,
@@ -107,6 +111,17 @@ struct Buffer {
   }
 };
 
+struct Result {
+  std::string Name;
+  Rule Rule;
+  std::string Actual;
+  std::string Expected;
+  Buffer *ActualPtr = nullptr;
+  Buffer *ExpectedPtr = nullptr;
+  DenormMode DM = DenormMode::Any;
+  unsigned ULPT; // ULP Tolerance
+};
+
 struct Resource {
   ResourceKind Kind;
   std::string Name;
@@ -212,6 +227,7 @@ struct Pipeline {
   llvm::SmallVector<Shader> Shaders;
   RuntimeSettings Settings;
   llvm::SmallVector<Buffer> Buffers;
+  llvm::SmallVector<Result> Results;
   llvm::SmallVector<DescriptorSet> Sets;
 
   uint32_t getDescriptorCount() const {
@@ -235,6 +251,7 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(offloadtest::Resource)
 LLVM_YAML_IS_SEQUENCE_VECTOR(offloadtest::Buffer)
 LLVM_YAML_IS_SEQUENCE_VECTOR(offloadtest::Shader)
 LLVM_YAML_IS_SEQUENCE_VECTOR(offloadtest::dx::RootParameter)
+LLVM_YAML_IS_SEQUENCE_VECTOR(offloadtest::Result)
 
 namespace llvm {
 namespace yaml {
@@ -251,6 +268,10 @@ template <> struct MappingTraits<offloadtest::Buffer> {
   static void mapping(IO &I, offloadtest::Buffer &R);
 };
 
+template <> struct MappingTraits<offloadtest::Result> {
+  static void mapping(IO &I, offloadtest::Result &R);
+};
+
 template <> struct MappingTraits<offloadtest::Resource> {
   static void mapping(IO &I, offloadtest::Resource &R);
 };
@@ -287,6 +308,25 @@ template <> struct MappingTraits<offloadtest::RuntimeSettings> {
   static void mapping(IO &I, offloadtest::RuntimeSettings &S);
 };
 
+template <> struct ScalarEnumerationTraits<offloadtest::Rule> {
+  static void enumeration(IO &I, offloadtest::Rule &V) {
+#define ENUM_CASE(Val) I.enumCase(V, #Val, offloadtest::Rule::Val)
+    ENUM_CASE(BufferExact);
+    ENUM_CASE(BufferFuzzy);
+#undef ENUM_CASE
+  }
+};
+
+template <> struct ScalarEnumerationTraits<offloadtest::DenormMode> {
+  static void enumeration(IO &I, offloadtest::DenormMode &V) {
+#define ENUM_CASE(Val) I.enumCase(V, #Val, offloadtest::DenormMode::Val)
+    ENUM_CASE(Any);
+    ENUM_CASE(FTZ);
+    ENUM_CASE(Preserve);
+#undef ENUM_CASE
+  }
+};
+
 template <> struct ScalarEnumerationTraits<offloadtest::DataFormat> {
   static void enumeration(IO &I, offloadtest::DataFormat &V) {
 #define ENUM_CASE(Val) I.enumCase(V, #Val, offloadtest::DataFormat::Val)

lib/Support/CMakeLists.txt

@@ -1 +1 @@
-add_offloadtest_library(Support Pipeline.cpp)
+add_offloadtest_library(Support Pipeline.cpp Check.cpp)

lib/Support/Check.cpp

@@ -0,0 +1,98 @@
+//===- Check.cpp - Check test results  ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//
+//===----------------------------------------------------------------------===//
+
+#include "Support/Check.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_ostream.h"
+
+static bool isDenorm(float F) { return std::fpclassify(F) == FP_SUBNORMAL; }
+
+static bool compareFloatULP(const float &FSrc, const float &FRef,
+                            unsigned ULPTolerance, offloadtest::DenormMode DM) {
+  if (FSrc == FRef)
+    return true;
+  if (std::isnan(FSrc))
+    return std::isnan(FRef);
+  if (DM == offloadtest::DenormMode::Any) {
+    // If denorm expected, output can be sign preserved zero. Otherwise output
+    // should pass the regular ulp testing.
+    if (isDenorm(FRef) && FSrc == 0 && std::signbit(FSrc) == std::signbit(FRef))
+      return true;
+  }
+  // For FTZ or Preserve mode, we should get the expected number within
+  // ULPTolerance for any operations.
+  int Diff = *((const uint32_t *)&FSrc) - *((const uint32_t *)&FRef);
+  unsigned int AbsDiff = Diff < 0 ? -Diff : Diff;
+  return AbsDiff <= ULPTolerance;
+}
+
+static bool testBufferExact(offloadtest::Buffer *B1, offloadtest::Buffer *B2) {
+  if (B1->size() != B2->size())
+    return false;
+  for (uint32_t I = 0; I < B1->size(); ++I) {
+    if (B1->Data[I] != B2->Data[I])
+      return false;
+  }
+  return true;
+}
+
+static bool testBufferFuzzy(offloadtest::Buffer *B1, offloadtest::Buffer *B2,
+                            unsigned ULPT, offloadtest::DenormMode DM) {
+  switch (B1->Format) {
+  case offloadtest::DataFormat::Float32: {
+    if (B1->Size != B2->Size)
+      return false;
+    llvm::MutableArrayRef<float> Arr1(reinterpret_cast<float *>(B1->Data.get()),
+                                      B1->Size / sizeof(float));
+    assert(B2->Format == offloadtest::DataFormat::Float32 &&
+           "Buffer types must be the same");
+    llvm::MutableArrayRef<float> Arr2(reinterpret_cast<float *>(B2->Data.get()),
+                                      B2->Size / sizeof(float));
+    for (unsigned I = 0; I < Arr1.size(); ++I) {
+      if (!compareFloatULP(Arr1[I], Arr2[I], ULPT, DM))
+        return false;
+    }
+    return true;
+  }
+  default:
+    llvm_unreachable("Only float types are supported by the fuzzy test.");
+  }
+  return false;
+}
+
+llvm::Error verifyResult(offloadtest::Result R) {
+  switch (R.Rule) {
+  case offloadtest::Rule::BufferExact: {
+    if (testBufferExact(R.ActualPtr, R.ExpectedPtr))
+      return llvm::Error::success();
+    llvm::SmallString<256> Str;
+    llvm::raw_svector_ostream OS(Str);
+    OS << "Test failed: " << R.Name << "\nExpected:\n";
+    llvm::yaml::Output YAMLOS(OS);
+    YAMLOS << *R.ExpectedPtr;
+    OS << "Got:\n";
+    YAMLOS << *R.ActualPtr;
+    return llvm::createStringError(Str.c_str());
+  }
+  case offloadtest::Rule::BufferFuzzy: {
+    if (testBufferFuzzy(R.ActualPtr, R.ExpectedPtr, R.ULPT, R.DM))
+      return llvm::Error::success();
+    llvm::SmallString<256> Str;
+    llvm::raw_svector_ostream OS(Str);
+    OS << "Test failed: " << R.Name << "\nExpected:\n";
+    llvm::yaml::Output YAMLOS(OS);
+    YAMLOS << *R.ExpectedPtr;
+    OS << "Got:\n";
+    YAMLOS << *R.ActualPtr;
+    return llvm::createStringError(Str.c_str());
+  }
+  }
+}

lib/Support/Pipeline.cpp

@@ -22,6 +22,7 @@ void MappingTraits<offloadtest::Pipeline>::mapping(IO &I,
   I.mapOptional("RuntimeSettings", P.Settings);
 
   I.mapRequired("Buffers", P.Buffers);
+  I.mapOptional("Results", P.Results);
   I.mapRequired("DescriptorSets", P.Sets);
 
   if (!I.outputting()) {
@@ -32,6 +33,21 @@ void MappingTraits<offloadtest::Pipeline>::mapping(IO &I,
           I.setError(Twine("Referenced buffer ") + R.Name + " not found!");
       }
     }
+    // Initialize result Buffers
+    for (auto &R : P.Results) {
+      R.ActualPtr = P.getBuffer(R.Actual);
+      if (!R.ActualPtr)
+        I.setError(Twine("Reference buffer ") + R.Actual + " not found!");
+      R.ExpectedPtr = P.getBuffer(R.Expected);
+      if (!R.ExpectedPtr)
+        I.setError(Twine("Reference buffer ") + R.Expected + " not found!");
+      if (R.Rule == offloadtest::Rule::BufferFuzzy) {
+        if (R.ActualPtr->Format != offloadtest::DataFormat::Float32 ||
+            R.ExpectedPtr->Format != offloadtest::DataFormat::Float32)
+          I.setError(Twine("BufferFuzzy only accepts Float buffers"));
+      }
+    }
+
     uint32_t DescriptorTableCount = 0;
     for (auto &R : P.Settings.DX.RootParams) {
       switch (R.Kind) {
@@ -196,5 +212,22 @@ void MappingTraits<offloadtest::Shader>::mapping(IO &I,
     I.mapRequired("DispatchSize", MutableDispatchSize);
   }
 }
+void MappingTraits<offloadtest::Result>::mapping(IO &I,
+                                                 offloadtest::Result &R) {
+  I.mapRequired("Result", R.Name);
+  I.mapRequired("Rule", R.Rule);
+  I.mapRequired("Actual", R.Actual);
+  I.mapRequired("Expected", R.Expected);
+
+  switch (R.Rule) {
+  case Rule::BufferFuzzy: {
+    I.mapRequired("ULPT", R.ULPT);
+    I.mapOptional("DenormMode", R.DM);
+    break;
+  }
+  default:
+    break;
+  }
+}
 } // namespace yaml
 } // namespace llvm

test/CMakeLists.txt

@@ -60,7 +60,8 @@ list(APPEND OFFLOADTEST_DEPS
   FileCheck
   split-file
   imgdiff
-  OffloadTestUnit)
+  OffloadTestUnit
+  not)
 
 if (OFFLOADTEST_TEST_CLANG)
   list(APPEND OFFLOADTEST_DEPS clang)

test/Tools/Offloader/BufferExact-error.test

@@ -0,0 +1,73 @@
+#--- source.hlsl
+
+RWStructuredBuffer<int> Out : register(u0);
+
+[numthreads(1,1,1)]
+void main() {
+  Out[0] = 20;
+  Out[1] = 30;
+  Out[2] = 40;
+  Out[3] = 50;
+}
+
+//--- pipeline.yaml
+
+---
+Shaders:
+  - Stage: Compute
+    Entry: main
+    DispatchSize: [1, 1, 1]
+Buffers:
+  - Name: Out
+    Format: Int32
+    Stride: 4
+    ZeroInitSize: 16
+  - Name: Expected1
+    Format: Int32
+    Stride: 4
+    Data: [ 1, 2, 3, 4 ]
+Results:
+  - Result: Test1
+    Rule: BufferExact
+    Actual: Out
+    Expected: Expected1
+DescriptorSets:
+  - Resources:
+    - Name: Out
+      Kind: RWStructuredBuffer
+      DirectXBinding:
+        Register: 0
+        Space: 0
+      VulkanBinding:
+        Binding: 0
+...
+#--- end
+
+# UNSUPPORTED: Clang-Vulkan
+# RUN: split-file %s %t
+# RUN: %dxc_target -T cs_6_5 -Fo %t.o %t/source.hlsl
+# RUN: not %offloader %t/pipeline.yaml %t.o 2>&1 | FileCheck %s
+
+# CHECK: Test failed: Test1
+# CHECK: Expected:
+# CHECK: ---
+# CHECK: Name:            Expected1
+# CHECK: Format:          Int32
+# CHECK: Stride:          4
+# CHECK: Data:            [ 1, 2, 3, 4 ]
+# CHECK: OutputProps:
+# CHECK: Height:          0
+# CHECK: Width:           0
+# CHECK: Depth:           0
+# CHECK:  ...
+# CHECK: Got:
+# CHECK: ---
+# CHECK: Name:            Out
+# CHECK: Format:          Int32
+# CHECK: Stride:          4
+# CHECK: Data:            [ 20, 30, 40, 50 ]
+# CHECK: OutputProps:
+# CHECK: Height:          0
+# CHECK: Width:           0
+# CHECK: Depth:           0
+