From 790f3699053cd077075c3b5868708e4ee54c2c07 Mon Sep 17 00:00:00 2001 From: Dmitry Nikolaev <139769634+dnikolaev-amd@users.noreply.github.com> Date: Sat, 19 Jul 2025 07:26:15 +0200 Subject: [PATCH] [rocm7.0_internal_testing] skip 3D NCHW FP16 batchnorm test due to Native accuracy issue (#2370) Skip for `test_nn.py::TestNN.test_batchnorm_3D_train_NCHW_vs_native_mixed_float16` Test failed on `weight gradient` comparison MIOpen/CuDNN vs Native batchnorm. But CPU test `test_batchnorm_3D_train_NCHW_vs_cpu_mixed_float16` passed. It looks like FP16 Native batchnorm issue. Failed on MI200/MI300 and V100 It passed somehow on Navi (with enabled MIOpen) Fixes SWDEV-541024, SWDEV-539171 ``` python test_nn.py -v -k test_batchnorm_3D_train_NCHW_vs_native_mixed_float16 test_batchnorm_3D_train_NCHW_vs_native_mixed_float16 (__main__.TestNN) ... skipped '3D float16 NCHW train failed on CUDA and ROCm due to Native batchnorm accuracy issue SWDEV-541024' OK (skipped=1) ``` --- test/test_nn.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test_nn.py b/test/test_nn.py index f7c018a32e9fe..f3b1764af69d8 100644 --- a/test/test_nn.py +++ b/test/test_nn.py @@ -5180,6 +5180,8 @@ def test_batchnorm_nhwc_cuda(self): name_fn=lambda f, b, m, t: f"{f}_vs_{b}{'_mixed' if m else ''}_{dtype_name(t)}" ) def test_batchnorm(self, dims, mode, memory_format, ref_backend, mixed, dtype): + if self._testMethodName == "test_batchnorm_3D_train_NCHW_vs_native_mixed_float16": + self.skipTest("3D float16 NCHW train failed on CUDA and ROCm due to Native batchnorm accuracy issue SWDEV-541024") if torch.version.hip: if self._testMethodName in ("test_batchnorm_2D_train_NHWC_vs_NCHW_mixed_bfloat16", "test_batchnorm_2D_train_NCHW_vs_cpu_mixed_bfloat16", @@ -5195,10 +5197,6 @@ def test_batchnorm(self, dims, mode, memory_format, ref_backend, mixed, dtype): ) and _get_torch_rocm_version() >= (6, 4): self.skipTest("bfloat16 NCHW train failed due to native tolerance issue SWDEV-507600") - if self._testMethodName == "test_batchnorm_3D_train_NCHW_vs_native_mixed_float16" \ - and _get_torch_rocm_version() < (6, 4): - self.skipTest("3D float16 NCHW train failed on ROCm<=6.3 ") - if dims == 3 and memory_format in ("NHWC", "NCHW"): memory_format = memory_format + "3D"