[SYCL][E2E] Add test to track RUNx tests

The same idea as for XFAIL and UNSUPPORTED. The test checks if there is
a RUNx without tracker. Also:
- this PR also updates some RUNx tests as they are likely fixed.
- marks some as UNSUPPORTED as there is only one RUNx line.
- adds tracker for the rest.

After that there shouldn't be any untracked RUNx tests in sycl/test-e2e.

Author: KornevNikita

sycl/test-e2e/Basic/fpga_tests/fpga_io_pipes.cpp

@@ -1,6 +1,10 @@
+// The test is disabled, since no support added in FPGA backend yet. Check
+// implementation correctness from CXX and SYCL languages perspective.
+// UNSUPPORTED: true
+
 // REQUIRES: accelerator
 // RUN: %{build} -o %t.out
-// RUNx: %{run} %t.out
+// RUN: %{run} %t.out
 //==------------ fpga_io_pipes.cpp - SYCL FPGA pipes test ------------------==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
@@ -15,9 +19,6 @@
 
 #include "io_pipe_def.h"
 
-// TODO: run is disabled, since no support added in FPGA backend yet. Check
-// implementation correctness from CXX and SYCL languages perspective.
-
 // This test is supposed to be run only on Intel FPGA emulator. Change it when
 // we have more experience with IO pipe feature in SYCL.
 // The emulator creates files (one for I pipe, another for O pipe) with the

sycl/test-e2e/Graph/Explicit/host_task_last.cpp

@@ -3,9 +3,9 @@
 // Extra run to check for leaks in Level Zero using UR_L0_LEAKS_DEBUG
 // RUN: %if level_zero %{env SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=0 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s --implicit-check-not=LEAK %}
 
-// Disabled due to https://github.com/intel/llvm/issues/14473
 // Extra run to check for immediate-command-list in Level Zero
 // RUNx: %if level_zero %{env SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s --implicit-check-not=LEAK %}
+// RUNx-TRACKER: https://github.com/intel/llvm/issues/14473
 
 // REQUIRES: aspect-usm_shared_allocations
 

sycl/test-e2e/Graph/RecordReplay/host_task_last.cpp

@@ -3,9 +3,9 @@
 // Extra run to check for leaks in Level Zero using UR_L0_LEAKS_DEBUG
 // RUN: %if level_zero %{env SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=0 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s --implicit-check-not=LEAK %}
 
-// Disabled due to https://github.com/intel/llvm/issues/14473
 // Extra run to check for immediate-command-list in Level Zero
 // RUNx: %if level_zero %{env SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s --implicit-check-not=LEAK %}
+// RUNx-TRACKER: https://github.com/intel/llvm/issues/14473
 
 // REQUIRES: aspect-usm_shared_allocations
 

sycl/test-e2e/OptionalKernelFeatures/fp64-conv-emu.cpp

@@ -2,8 +2,7 @@
 // UNSUPPORTED: cuda, hip
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_gen  -Xsycl-target-backend "-device pvc" -fsycl-fp64-conv-emu -O0 %s -o %t_opt.out
-// TODO: Enable when GPU driver is updated.
-// RUNx: %{run} %t_opt.out
+// RUN: %{run} %t_opt.out
 
 // Tests that aspect::fp64 is not emitted correctly when -fsycl-fp64-conv-emu
 // flag is used.

sycl/test-e2e/Regression/isordered.cpp

@@ -1,5 +1,5 @@
 // RUN: %{build} -o %t.out
-// RUNx: %{run} %t.out
+// RUN: %{run} %t.out
 
 #include <sycl/detail/core.hpp>
 

sycl/test-e2e/Regression/mad_sat.cpp

@@ -1,5 +1,5 @@
 // RUN: %{build} -Wno-error=integer-overflow -Wno-error=implicitly-unsigned-literal -o %t.out
-// RUNx: %{run} %t.out
+// RUN: %{run} %t.out
 
 #include <sycl/detail/core.hpp>
 

sycl/test-e2e/SharedLib/use_with_dlopen.cpp

@@ -16,6 +16,7 @@
 // This causes SEG. FAULT.
 // RUNx: %{compile} -o %t4.out -DRUN_LAST
 // RUNx: %{run} %t4.out
+// RUNx-TRACKER: https://github.com/intel/llvm/issues/16031
 
 #include <sycl/detail/core.hpp>
 

sycl/test-e2e/SharedLib/use_with_dlopen_verify_cache.cpp

@@ -17,6 +17,7 @@
 // This causes SEG. FAULT.
 // RUNx: %{compile} -DRUN_LAST
 // RUNx: env SYCL_UR_TRACE=2 %{run} %t.out 2>&1 | FileCheck %s --check-prefixes=CHECK-LAST,CHECK --implicit-check-not=piProgramBuild
+// RUNx-TRACKER: https://github.com/intel/llvm/issues/16031
 // clang-format on
 
 #include <sycl/detail/core.hpp>

sycl/test/e2e_test_requirements/no-runx-without-tracker.cpp

@@ -0,0 +1,29 @@
+// This test is intended to ensure that we have no tests marked as RUNx
+// without a tracker information added to a test. If it fails - please create
+// a tracker for RUNx-marked test.
+//
+// The format we check is:
+// RUNx: command
+// RUNx-TRACKER: [GitHub issue URL|Internal tracker ID]
+//
+// GitHub issue URL format:
+//     https://github.com/owner/repo/issues/12345
+//
+// Internal tracker ID format:
+//     PROJECT-123456
+//
+// REQUIRES: linux
+//
+// Explanation of the command:
+// - search for all "RUNx" occurrences, display line with match and the next one
+//   -I, --include to drop binary files and other unrelated files
+// - in the result, search for "RUNx" again, but invert the result - this
+//   allows us to get the line *after* RUNx
+// - in those lines, check that RUNx-TRACKER is present and correct. Once
+//   again, invert the search to get all "bad" lines; running it with "not" as
+//   grep exits with 1 if it finds nothing
+//
+// RUN: grep -rI "RUNx:" %S/../../test-e2e \
+// RUN: -A 1 --include=*.cpp --no-group-separator | \
+// RUN: grep -v "RUNx:" | \
+// RUN: not grep -Pv "RUNx-TRACKER:\s+(?:https://github.com/[\w\d-]+/[\w\d-]+/issues/[\d]+)|(?:[\w]+-[\d]+)" 

sycl/test/e2e_test_requirements/no-unsupported-without-info.cpp

@@ -54,7 +54,7 @@
 // tests to match the required format and in that case you should just update
 // (i.e. reduce) the number and the list below.
 //
-// NUMBER-OF-UNSUPPORTED-WITHOUT-INFO: 477
+// NUMBER-OF-UNSUPPORTED-WITHOUT-INFO: 478
 //
 // List of improperly UNSUPPORTED tests.
 // Remove the CHECK once the test has been properly UNSUPPORTED.
@@ -77,6 +77,7 @@
 // CHECK-NEXT: Basic/buffer/subbuffer.cpp
 // CHECK-NEXT: Basic/build_log.cpp
 // CHECK-NEXT: Basic/code_location_e2e.cpp
+// CHECK-NEXT: Basic/fpga_tests/fpga_io_pipes.cpp
 // CHECK-NEXT: Basic/free_function_queries/free_function_queries.cpp
 // CHECK-NEXT: Basic/free_function_queries/free_function_queries_sub_group.cpp
 // CHECK-NEXT: Basic/free_function_queries/free_function_queries_sub_group.cpp