Merge pull request #233 from IntelPython/feature/add_fail_check

ZzEeKkAa · web-flow · commit cc8ad7659b43 · 2023-04-28T11:56:13.000-04:00
Add fail check
diff --git a/dpbench/config/benchmark.py b/dpbench/config/benchmark.py
@@ -88,6 +88,7 @@ class Benchmark:
     array_args: List[str] = field(default_factory=list)
     output_args: List[str] = field(default_factory=list)
     implementations: List[BenchmarkImplementation] = field(default_factory=list)
+    expected_failure_implementations: List[str] = field(default_factory=list)
 
     @staticmethod
     def from_dict(obj: Any) -> "Benchmark":
@@ -107,6 +108,9 @@ def from_dict(obj: Any) -> "Benchmark":
         _array_args = obj.get("array_args") or []
         _output_args = obj.get("output_args") or []
         _implementations = obj.get("implementations") or []
+        _expected_failure_implementations = (
+            obj.get("expected_failure_implementations") or []
+        )
         return Benchmark(
             _name,
             _short_name,
@@ -122,4 +126,5 @@ def from_dict(obj: Any) -> "Benchmark":
             _array_args,
             _output_args,
             _implementations,
+            _expected_failure_implementations,
         )
diff --git a/dpbench/configs/bench_info/dbscan.toml b/dpbench/configs/bench_info/dbscan.toml
@@ -20,6 +20,7 @@ input_args = [
 array_args = [
     "data",
 ]
+expected_failure_implementations = ["sycl"]
 
 [benchmark.parameters.S]
 n_samples = 1024
diff --git a/dpbench/configs/bench_info/gpairs.toml b/dpbench/configs/bench_info/gpairs.toml
@@ -39,6 +39,7 @@ array_args = [
 output_args = [
     "results",
 ]
+expected_failure_implementations = ["numba_dpex_k"]
 
 [benchmark.parameters.S]
 nopt = 128
diff --git a/dpbench/configs/bench_info/knn.toml b/dpbench/configs/bench_info/knn.toml
@@ -32,6 +32,7 @@ array_args = [
 output_args = [
     "predictions",
 ]
+expected_failure_implementations = ["numba_dpex_p"]
 
 [benchmark.parameters.S]
 test_size = 1024
diff --git a/dpbench/configs/bench_info/l2_norm.toml b/dpbench/configs/bench_info/l2_norm.toml
@@ -21,6 +21,7 @@ array_args = [
 output_args = [
     "d",
 ]
+expected_failure_implementations = ["dpnp", "numba_dpex_n"]
 
 [benchmark.parameters.S]
 npoints = 32768
diff --git a/dpbench/configs/bench_info/npbench/azimint_hist.toml b/dpbench/configs/bench_info/npbench/azimint_hist.toml
@@ -23,6 +23,7 @@ array_args = [
     "radius",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 400000
diff --git a/dpbench/configs/bench_info/npbench/azimint_naive.toml b/dpbench/configs/bench_info/npbench/azimint_naive.toml
@@ -23,6 +23,7 @@ array_args = [
     "radius",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 400000
diff --git a/dpbench/configs/bench_info/npbench/cavity_flow.toml b/dpbench/configs/bench_info/npbench/cavity_flow.toml
@@ -38,6 +38,7 @@ output_args = [
     "v",
     "p",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 ny = 61
diff --git a/dpbench/configs/bench_info/npbench/channel_flow.toml b/dpbench/configs/bench_info/npbench/channel_flow.toml
@@ -37,6 +37,7 @@ output_args = [
     "v",
     "p",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 ny = 61
diff --git a/dpbench/configs/bench_info/npbench/compute.toml b/dpbench/configs/bench_info/npbench/compute.toml
@@ -26,6 +26,7 @@ array_args = [
     "array_2",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 M = 2000
diff --git a/dpbench/configs/bench_info/npbench/contour_integral.toml b/dpbench/configs/bench_info/npbench/contour_integral.toml
@@ -26,6 +26,7 @@ array_args = [
     "Y",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 NR = 50
diff --git a/dpbench/configs/bench_info/npbench/conv2d_bias.toml b/dpbench/configs/bench_info/npbench/conv2d_bias.toml
@@ -23,6 +23,7 @@ array_args = [
     "bias",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 8
diff --git a/dpbench/configs/bench_info/npbench/crc16.toml b/dpbench/configs/bench_info/npbench/crc16.toml
@@ -20,6 +20,7 @@ array_args = [
     "data",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 1600
diff --git a/dpbench/configs/bench_info/npbench/hdiff.toml b/dpbench/configs/bench_info/npbench/hdiff.toml
@@ -27,6 +27,7 @@ array_args = [
 output_args = [
     "out_field",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 I = 64
diff --git a/dpbench/configs/bench_info/npbench/mandelbrot2.toml b/dpbench/configs/bench_info/npbench/mandelbrot2.toml
@@ -26,6 +26,7 @@ input_args = [
 array_args = []
 output_args = []
 norm_error = 0.001
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 xmin = -2.0
diff --git a/dpbench/configs/bench_info/npbench/mlp.toml b/dpbench/configs/bench_info/npbench/mlp.toml
@@ -31,6 +31,7 @@ array_args = [
     "b3",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 C_in = 3
diff --git a/dpbench/configs/bench_info/npbench/nbody.toml b/dpbench/configs/bench_info/npbench/nbody.toml
@@ -33,6 +33,7 @@ output_args = [
     "vel",
 ]
 norm_error = 0.1
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 25
diff --git a/dpbench/configs/bench_info/npbench/scattering_self_energies.toml b/dpbench/configs/bench_info/npbench/scattering_self_energies.toml
@@ -29,6 +29,7 @@ array_args = [
 output_args = [
     "Sigma",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 Nkz = 2
diff --git a/dpbench/configs/bench_info/npbench/softmax.toml b/dpbench/configs/bench_info/npbench/softmax.toml
@@ -19,6 +19,7 @@ array_args = [
     "x",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 16
diff --git a/dpbench/configs/bench_info/npbench/spmv.toml b/dpbench/configs/bench_info/npbench/spmv.toml
@@ -25,6 +25,7 @@ array_args = [
     "x",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 M = 4096
diff --git a/dpbench/configs/bench_info/npbench/stockham_fft.toml b/dpbench/configs/bench_info/npbench/stockham_fft.toml
@@ -26,6 +26,7 @@ array_args = [
 output_args = [
     "y",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 R = 2
diff --git a/dpbench/configs/bench_info/npbench/vadv.toml b/dpbench/configs/bench_info/npbench/vadv.toml
@@ -32,6 +32,7 @@ array_args = [
 output_args = [
     "utens_stage",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 I = 60
diff --git a/dpbench/configs/bench_info/pairwise_distance.toml b/dpbench/configs/bench_info/pairwise_distance.toml
@@ -23,6 +23,7 @@ array_args = [
 output_args = [
     "D",
 ]
+expected_failure_implementations = ["dpnp", "numba_dpex_n"]
 
 [benchmark.parameters.S]
 npoints = 1024
diff --git a/dpbench/configs/bench_info/pca.toml b/dpbench/configs/bench_info/pca.toml
@@ -21,6 +21,7 @@ output_args = [
     "evalues",
     "evectors",
 ]
+expected_failure_implementations = ["dpnp", "numba_dpex_n"]
 
 [benchmark.parameters.S]
 npoints = 1024
diff --git a/dpbench/configs/bench_info/polybench/datamining/correlation.toml b/dpbench/configs/bench_info/polybench/datamining/correlation.toml
@@ -24,6 +24,7 @@ array_args = [
 output_args = [
     "data",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 M = 500
diff --git a/dpbench/configs/bench_info/polybench/datamining/covariance.toml b/dpbench/configs/bench_info/polybench/datamining/covariance.toml
@@ -22,6 +22,7 @@ array_args = [
     "data",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 M = 500
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/blas/gemm.toml b/dpbench/configs/bench_info/polybench/linear-algebra/blas/gemm.toml
@@ -28,6 +28,7 @@ array_args = [
 output_args = [
     "C",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 NI = 1000
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/blas/gemver.toml b/dpbench/configs/bench_info/polybench/linear-algebra/blas/gemver.toml
@@ -42,6 +42,7 @@ output_args = [
     "w",
     "x",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 1000
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/blas/gesummv.toml b/dpbench/configs/bench_info/polybench/linear-algebra/blas/gesummv.toml
@@ -26,6 +26,7 @@ array_args = [
     "x",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 2000
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/blas/symm.toml b/dpbench/configs/bench_info/polybench/linear-algebra/blas/symm.toml
@@ -28,6 +28,7 @@ array_args = [
 output_args = [
     "C",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 M = 40
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/blas/trmm.toml b/dpbench/configs/bench_info/polybench/linear-algebra/blas/trmm.toml
@@ -25,6 +25,7 @@ array_args = [
 output_args = [
     "B",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 M = 65
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/kernels/atax.toml b/dpbench/configs/bench_info/polybench/linear-algebra/kernels/atax.toml
@@ -22,6 +22,7 @@ array_args = [
     "x",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 M = 4000
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/kernels/bicg.toml b/dpbench/configs/bench_info/polybench/linear-algebra/kernels/bicg.toml
@@ -24,6 +24,7 @@ array_args = [
     "r",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 M = 4000
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/kernels/doitgen.toml b/dpbench/configs/bench_info/polybench/linear-algebra/kernels/doitgen.toml
@@ -27,6 +27,7 @@ array_args = [
 output_args = [
     "A",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 NR = 60
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/kernels/k2mm.toml b/dpbench/configs/bench_info/polybench/linear-algebra/kernels/k2mm.toml
@@ -30,6 +30,7 @@ array_args = [
 output_args = [
     "D",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 NI = 800
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/kernels/mvt.toml b/dpbench/configs/bench_info/polybench/linear-algebra/kernels/mvt.toml
@@ -31,6 +31,7 @@ output_args = [
     "x1",
     "x2",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 5500
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/solvers/cholesky.toml b/dpbench/configs/bench_info/polybench/linear-algebra/solvers/cholesky.toml
@@ -22,6 +22,7 @@ array_args = [
 output_args = [
     "A",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 100
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/solvers/durbin.toml b/dpbench/configs/bench_info/polybench/linear-algebra/solvers/durbin.toml
@@ -22,6 +22,7 @@ array_args = [
 output_args = []
 rtol = 0.001
 atol = 0.001
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 1000
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/solvers/lu.toml b/dpbench/configs/bench_info/polybench/linear-algebra/solvers/lu.toml
@@ -22,6 +22,7 @@ array_args = [
 output_args = [
     "A",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 60
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/solvers/ludcmp.toml b/dpbench/configs/bench_info/polybench/linear-algebra/solvers/ludcmp.toml
@@ -24,6 +24,7 @@ array_args = [
 output_args = [
     "A",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 60
diff --git a/dpbench/configs/bench_info/polybench/linear-algebra/solvers/trisolv.toml b/dpbench/configs/bench_info/polybench/linear-algebra/solvers/trisolv.toml
@@ -26,6 +26,7 @@ array_args = [
 output_args = [
     "b",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 2000
diff --git a/dpbench/configs/bench_info/polybench/medley/deriche.toml b/dpbench/configs/bench_info/polybench/medley/deriche.toml
@@ -21,6 +21,7 @@ array_args = [
     "imgIn",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 W = 400
diff --git a/dpbench/configs/bench_info/polybench/medley/floyd_warshall.toml b/dpbench/configs/bench_info/polybench/medley/floyd_warshall.toml
@@ -22,6 +22,7 @@ array_args = [
 output_args = [
     "path",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 200
diff --git a/dpbench/configs/bench_info/polybench/medley/nussinov.toml b/dpbench/configs/bench_info/polybench/medley/nussinov.toml
@@ -21,6 +21,7 @@ array_args = [
     "seq",
 ]
 output_args = []
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 N = 40
diff --git a/dpbench/configs/bench_info/polybench/stencils/fdtd_2d.toml b/dpbench/configs/bench_info/polybench/stencils/fdtd_2d.toml
@@ -31,6 +31,7 @@ output_args = [
     "ey",
     "hz",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 TMAX = 20
diff --git a/dpbench/configs/bench_info/polybench/stencils/seidel_2d.toml b/dpbench/configs/bench_info/polybench/stencils/seidel_2d.toml
@@ -24,6 +24,7 @@ array_args = [
 output_args = [
     "A",
 ]
+expected_failure_implementations = ["dpnp"]
 
 [benchmark.parameters.S]
 TSTEPS = 8
diff --git a/dpbench/infrastructure/__init__.py b/dpbench/infrastructure/__init__.py
diff --git a/dpbench/infrastructure/reporter.py b/dpbench/infrastructure/reporter.py
diff --git a/dpbench/runner.py b/dpbench/runner.py
diff --git a/scripts/set_expected_failure.py b/scripts/set_expected_failure.py

Original file line number	Diff line number	Diff line change
`@@ -20,6 +20,7 @@ input_args = [`
`20`	`20`	`array_args = [`
`21`	`21`	`"data",`
`22`	`22`	`]`
	`23`	`+expected_failure_implementations = ["sycl"]`
`23`	`24`
`24`	`25`	`[benchmark.parameters.S]`
`25`	`26`	`n_samples = 1024`
Original file line number	Diff line number	Diff line change
`@@ -39,6 +39,7 @@ array_args = [`
`39`	`39`	`output_args = [`
`40`	`40`	`"results",`
`41`	`41`	`]`
	`42`	`+expected_failure_implementations = ["numba_dpex_k"]`
`42`	`43`
`43`	`44`	`[benchmark.parameters.S]`
`44`	`45`	`nopt = 128`
Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,7 @@ array_args = [`
`32`	`32`	`output_args = [`
`33`	`33`	`"predictions",`
`34`	`34`	`]`
	`35`	`+expected_failure_implementations = ["numba_dpex_p"]`
`35`	`36`
`36`	`37`	`[benchmark.parameters.S]`
`37`	`38`	`test_size = 1024`
Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,7 @@ array_args = [`
`21`	`21`	`output_args = [`
`22`	`22`	`"d",`
`23`	`23`	`]`
	`24`	`+expected_failure_implementations = ["dpnp", "numba_dpex_n"]`
`24`	`25`
`25`	`26`	`[benchmark.parameters.S]`
`26`	`27`	`npoints = 32768`
Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,7 @@ array_args = [`
`23`	`23`	`"radius",`
`24`	`24`	`]`
`25`	`25`	`output_args = []`
	`26`	`+expected_failure_implementations = ["dpnp"]`
`26`	`27`
`27`	`28`	`[benchmark.parameters.S]`
`28`	`29`	`N = 400000`
Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,7 @@ output_args = [`
`38`	`38`	`"v",`
`39`	`39`	`"p",`
`40`	`40`	`]`
	`41`	`+expected_failure_implementations = ["dpnp"]`
`41`	`42`
`42`	`43`	`[benchmark.parameters.S]`
`43`	`44`	`ny = 61`
Original file line number	Diff line number	Diff line change
`@@ -37,6 +37,7 @@ output_args = [`
`37`	`37`	`"v",`
`38`	`38`	`"p",`
`39`	`39`	`]`
	`40`	`+expected_failure_implementations = ["dpnp"]`
`40`	`41`
`41`	`42`	`[benchmark.parameters.S]`
`42`	`43`	`ny = 61`
Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,7 @@ array_args = [`
`26`	`26`	`"array_2",`
`27`	`27`	`]`
`28`	`28`	`output_args = []`
	`29`	`+expected_failure_implementations = ["dpnp"]`
`29`	`30`
`30`	`31`	`[benchmark.parameters.S]`
`31`	`32`	`M = 2000`