[CI] Increases the number of lit workers based on the GPU arch. (#1919)

* [CI] Increases the number of lit workers based on the GPU arch.

- For MFMA architectures (gfx908, gfx90a, gfx942, gfx950),
  the worker count is increased to 64.
- For Navi architectures, the worker count is increased to 16.

The goal is to evaluate how CI machines handle more threads,
the tests executions remain stable under these settings.
This change is an initial step to experiment with performance
and stability improvements during CI runs.

* Limit LIT worker count for unstable CI architectures.
- Reduce LIT worker count to 20 for gfx90a/gfx908 due to CI
instability and test timeouts.
- Set worker count to 8 for Navi3x/Navi4x due to instability
on some navi CI machines.

* Temporarily disabled data-after-header validation from tuning
file check.

Author: stefankoncarevic

mlir/utils/jenkins/Jenkinsfile

@@ -355,8 +355,10 @@ def rebootNode() {
 int setLitWorkerCount() {
     int limit_lit_workers = 8
     def gpu_arch = get_gpu_architecture()
-    if (gpu_arch.contains('gfx942') || gpu_arch.contains('gfx950')) {
-        limit_lit_workers = 40
+    if (gpu_arch.contains('gfx908') || gpu_arch.contains('gfx90a')) {
+        limit_lit_workers = 20
+    } else if (gpu_arch.contains('gfx942') || gpu_arch.contains('gfx950')) {
+        limit_lit_workers = 64
     }
     return limit_lit_workers
 }

mlir/utils/tuna/tuna-script.sh

@@ -59,12 +59,13 @@ validate_tuning_file() {
         echo "ERROR: $f has duplicate header lines next to each other!"
         exit 1
     fi
+    # NOTE: Temporarily disabled the check for data lines after the header.
     # Existing checks
-    data_line=$(awk 'NR>1 && $0 !~ /^\s*$/ && $0 != header {print; exit}' header="$header" "$f")
-    if [ -z "$data_line" ]; then
-        echo "ERROR: $f has no data after header!"
-        exit 1
-    fi
+    #data_line=$(awk 'NR>1 && $0 !~ /^\s*$/ && $0 != header {print; exit}' header="$header" "$f")
+    #if [ -z "$data_line" ]; then
+    #    echo "ERROR: $f has no data after header!"
+    #    exit 1
+    #fi
 }
 
 function tuna_run