[SYCL][E2E] Add XFAIL trackers for improperly XFAIL-ed E2E tests (#16421)

lbushi25 · web-flow · commit faf93d5bba64 · 2025-01-03T19:28:52.000Z
Add XFAIL trackers for improperly XFAIL-ed E2E tests. This brings the number of improperly XFAIL-ed tests down to zero. For the tests below, the XFAILs have been removed entirely. - For `sycl/test-e2e/ESIMD/hardware_dispatch.cpp`, the XFAIL is on igc-dev but the test requires GEN12 which does not have igc-dev so this XFAIL is meaningless. Thanks to @sarnex for noting this. - Three tests under `test-e2e/Reduction` which are XFAIL-ed only on hip_nvidia which we do not support anymore. Closes: #16400
diff --git a/sycl/test-e2e/AddressSanitizer/nullpointer/private_nullptr.cpp b/sycl/test-e2e/AddressSanitizer/nullpointer/private_nullptr.cpp
@@ -8,6 +8,7 @@
 
 // FIXME: There's an issue in gfx driver, so this test pending here.
 // XFAIL: run-mode
+// XFAIL-TRACKER: GSD-9911
 
 #include <sycl/detail/core.hpp>
 #include <sycl/ext/oneapi/experimental/address_cast.hpp>
diff --git a/sycl/test-e2e/Basic/max_linear_work_group_size_props.cpp b/sycl/test-e2e/Basic/max_linear_work_group_size_props.cpp
@@ -3,6 +3,7 @@
 
 // This property is not yet supported by all UR adapters
 // XFAIL: level_zero, opencl, hip
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16403
 
 #include <sycl/detail/core.hpp>
 
diff --git a/sycl/test-e2e/Basic/max_work_group_size_props.cpp b/sycl/test-e2e/Basic/max_work_group_size_props.cpp
@@ -3,6 +3,7 @@
 
 // This property is not yet supported by all UR adapters
 // XFAIL: level_zero, opencl, hip
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16403
 
 #include <sycl/detail/core.hpp>
 
diff --git a/sycl/test-e2e/DeviceLib/assert-windows.cpp b/sycl/test-e2e/DeviceLib/assert-windows.cpp
@@ -1,8 +1,7 @@
 // REQUIRES: cpu,windows
 //
-// FIXME: OpenCL CPU backend compiler crashes on a call to _wassert.
-// Disable the test until the fix reaches SYCL test infrastructure.
 // XFAIL: *
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16507
 //
 // RUN: %{build} -o %t.out
 //
diff --git a/sycl/test-e2e/ESIMD/hardware_dispatch.cpp b/sycl/test-e2e/ESIMD/hardware_dispatch.cpp
@@ -5,7 +5,6 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-// XFAIL: igc-dev
 // REQUIRES: ocloc && arch-intel_gpu_tgllp
 // RUN: %clangxx -fsycl -fsycl-targets=intel_gpu_tgllp %s -o %t.out
 // RUN: %{run-unfiltered-devices} %t.out
diff --git a/sycl/test-e2e/InlineAsm/asm_multiple_instructions.cpp b/sycl/test-e2e/InlineAsm/asm_multiple_instructions.cpp
@@ -7,6 +7,7 @@
 // The test is failing when writing directly to output buffer.
 // If temporary variable is used (see TO_PASS mode) the test succeeded.
 // XFAIL: gpu && run-mode
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16412
 #include "include/asmhelper.h"
 #include <iostream>
 #include <vector>
diff --git a/sycl/test-e2e/NewOffloadDriver/multisource.cpp b/sycl/test-e2e/NewOffloadDriver/multisource.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 // XFAIL: cuda
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16413
 // Separate kernel sources and host code sources
 // Test with `--offload-new-driver`
 // RUN: %{build} --offload-new-driver -c -o %t.kernel.o -DINIT_KERNEL -DCALC_KERNEL
diff --git a/sycl/test-e2e/NewOffloadDriver/split-per-source-main.cpp b/sycl/test-e2e/NewOffloadDriver/split-per-source-main.cpp
@@ -2,7 +2,8 @@
 // RUN: --offload-new-driver -fsycl-dead-args-optimization
 // RUN: %{run} %t.out
 //
-// XFAIL: hip_nvidia, cuda
+// XFAIL: cuda
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16413
 
 #include "Inputs/split-per-source.h"
 
diff --git a/sycl/test-e2e/NewOffloadDriver/sycl-external-with-optional-features.cpp b/sycl/test-e2e/NewOffloadDriver/sycl-external-with-optional-features.cpp
@@ -4,6 +4,7 @@
 // RUN: %clangxx -Wno-error=unused-command-line-argument -fsycl -fsycl-targets=%{sycl_triple} --offload-new-driver %t1.o %t2.o -o %t.exe
 // RUN: %{run} %t.exe
 // XFAIL: cuda
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16413
 
 #ifdef SOURCE1
 #include <iostream>
diff --git a/sycl/test-e2e/OptionalKernelFeatures/throw-exception-for-out-of-registers-on-kernel-launch.cpp b/sycl/test-e2e/OptionalKernelFeatures/throw-exception-for-out-of-registers-on-kernel-launch.cpp
@@ -2,6 +2,7 @@
 // RUN: %{build} -o %t.out
 // RUN: %{run} %t.out
 // XFAIL: *
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/12679
 
 #include <numeric>
 #include <string_view>
diff --git a/sycl/test-e2e/PerformanceTests/Reduction/reduce_over_sub_group.cpp b/sycl/test-e2e/PerformanceTests/Reduction/reduce_over_sub_group.cpp
@@ -1,4 +1,5 @@
 // XFAIL: native_cpu
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16415
 // RUN: %{build} -o %t.out
 // RUN: %{run} %t.out
 
diff --git a/sycl/test-e2e/Reduction/reduction_big_data.cpp b/sycl/test-e2e/Reduction/reduction_big_data.cpp
@@ -1,9 +1,6 @@
 // RUN: %{build} -o %t.out
 // RUN: %{run} %t.out
 //
-// Group algorithms are not supported on Nvidia.
-// XFAIL: hip_nvidia
-//
 
 // Windows doesn't yet have full shutdown().
 // UNSUPPORTED: ze_debug && windows
diff --git a/sycl/test-e2e/Reduction/reduction_nd_reducer_skip.cpp b/sycl/test-e2e/Reduction/reduction_nd_reducer_skip.cpp
@@ -1,8 +1,5 @@
 // RUN: %{build} -o %t.out
 // RUN: %{run} %t.out
-//
-// Group algorithms are not supported on Nvidia.
-// XFAIL: hip_nvidia
 
 // Windows doesn't yet have full shutdown().
 // UNSUPPORTED: ze_debug && windows
diff --git a/sycl/test-e2e/Reduction/reduction_reducer_op_eq.cpp b/sycl/test-e2e/Reduction/reduction_reducer_op_eq.cpp
@@ -1,8 +1,5 @@
 // RUN: %{build} -o %t.out
 // RUN: %{run} %t.out
-//
-// On nvidia a reduction appears to be unexpectedly executed via the host.
-// XFAIL: hip_nvidia
 
 // Windows doesn't yet have full shutdown().
 // UNSUPPORTED: ze_debug && windows
diff --git a/sycl/test-e2e/Regression/build_log.cpp b/sycl/test-e2e/Regression/build_log.cpp
@@ -6,6 +6,7 @@
 // CHECK-EXPECTED-ERROR: error: backend compiler failed build
 
 // XFAIL: *
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16416
 
 #include <stdio.h>
 #include <sycl/detail/core.hpp>
diff --git a/sycl/test-e2e/Regression/complex_global_object.cpp b/sycl/test-e2e/Regression/complex_global_object.cpp
@@ -1,6 +1,5 @@
 // RUN: %{build} -o %t.out
 // RUN: %{run} %t.out
-// XFAIL: gpu && linux
 
 // SYCL runtime may construct global objects at function scope. The test ensures
 // such objects do not cause problems if the first call to SYCL is inside main
diff --git a/sycl/test-e2e/Regression/kernel_bundle_ignore_sycl_external.cpp b/sycl/test-e2e/Regression/kernel_bundle_ignore_sycl_external.cpp
@@ -2,6 +2,7 @@
 // RUN: %{run} %t.out
 //
 // XFAIL: cuda
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16417
 // UNSUPPORTED: hip
 
 // Windows doesn't yet have full shutdown().
diff --git a/sycl/test-e2e/Regression/reduction_resource_leak_dw.cpp b/sycl/test-e2e/Regression/reduction_resource_leak_dw.cpp
@@ -1,5 +1,6 @@
 // REQUIRES: level_zero, level_zero_dev_kit
 // XFAIL: windows
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16418
 //
 // RUN: %{build} %level_zero_options -o %t.out
 // RUN: %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s
diff --git a/sycl/test/e2e_test_requirements/no-xfail-without-tracker.cpp b/sycl/test/e2e_test_requirements/no-xfail-without-tracker.cpp
@@ -20,57 +20,15 @@
 // - in the result, search for "XFAIL" again, but invert the result - this
 //   allows us to get the line *after* XFAIL
 // - in those lines, check that XFAIL-TRACKER is present and correct. Once
-//   again, invert the search to get all "bad" lines and save the test names in
-//   the temp file
-// - make a final count of how many ill-formatted directives there are and
-//   verify that against the reference
-// - ...and check if the list of improperly XFAIL-ed tests needs to be updated.
+//   again, invert the search to get all "bad" lines. Because there should be no
+//   bad lines, check that the search fails using the negation operator not.
 //
 // RUN: grep -rI "XFAIL:" %S/../../test-e2e \
 // RUN: -A 1 --include=*.cpp --no-group-separator | \
 // RUN: grep -v "XFAIL:" | \
-// RUN: grep -Pv "(?:XFAIL-TRACKER:\s+(?:(?:https:\/\/github.com\/[\w\d-]+\/[\w\d-]+\/issues\/[\d]+)|(?:[\w]+-[\d]+)))" > %t
-// RUN: cat %t | wc -l | FileCheck %s --check-prefix NUMBER-OF-XFAIL-WITHOUT-TRACKER
-// RUN: cat %t | sed 's/\.cpp.*/.cpp/' | sort | FileCheck %s
-//
-// The number below is a number of tests which are *improperly* XFAIL-ed, i.e.
-// we either don't have a tracker associated with a failure listed in those
-// tests, or it is listed in a wrong format.
-// Note: strictly speaking, that is not amount of files, but amount of XFAIL
-// directives. If a test contains several XFAIL directives, some of them may be
-// valid and other may not.
-//
-// That number *must not* increase. Any PR which causes this number to grow
-// should be rejected and it should be updated to either keep the number as-is
-// or have it reduced (preferably, down to zero).
+// RUN: not grep -Pv "XFAIL-TRACKER:\s+(?:https://github.com/[\w\d-]+/[\w\d-]+/issues/[\d]+)|(?:[\w]+-[\d]+)"
 //
 // If you see this test failed for your patch, it means that you either
 // introduced XFAIL directive to a test improperly, or broke the format of an
-// existing XFAIL-ed tests.
-// Another possibility (and that is a good option) is that you updated some
-// tests to match the required format and in that case you should just update
-// (i.e. reduce) the number and the list below.
-//
-// NUMBER-OF-XFAIL-WITHOUT-TRACKER: 18
-//
-// List of improperly XFAIL-ed tests.
-// Remove the CHECK once the test has been properly XFAIL-ed.
-//
-// CHECK: AddressSanitizer/nullpointer/private_nullptr.cpp
-// CHECK-NEXT: Basic/max_linear_work_group_size_props.cpp
-// CHECK-NEXT: Basic/max_work_group_size_props.cpp
-// CHECK-NEXT: DeviceLib/assert-windows.cpp
-// CHECK-NEXT: ESIMD/hardware_dispatch.cpp
-// CHECK-NEXT: InlineAsm/asm_multiple_instructions.cpp
-// CHECK-NEXT: NewOffloadDriver/multisource.cpp
-// CHECK-NEXT: NewOffloadDriver/split-per-source-main.cpp
-// CHECK-NEXT: NewOffloadDriver/sycl-external-with-optional-features.cpp
-// CHECK-NEXT: OptionalKernelFeatures/throw-exception-for-out-of-registers-on-kernel-launch.cpp
-// CHECK-NEXT: PerformanceTests/Reduction/reduce_over_sub_group.cpp
-// CHECK-NEXT: Reduction/reduction_big_data.cpp
-// CHECK-NEXT: Reduction/reduction_nd_reducer_skip.cpp
-// CHECK-NEXT: Reduction/reduction_reducer_op_eq.cpp
-// CHECK-NEXT: Regression/build_log.cpp
-// CHECK-NEXT: Regression/complex_global_object.cpp
-// CHECK-NEXT: Regression/kernel_bundle_ignore_sycl_external.cpp
-// CHECK-NEXT: Regression/reduction_resource_leak_dw.cpp
+// existing XFAIL-ed tests. Note in particular that XFAIL-TRACKER should appear in the
+// next line following the XFAIL.
