AVX-512 implementation of complex_div2, complex_rdiv2, complex_div3

sadko4u · sadko4u · commit f1078937bc65 · 2023-09-09T17:37:06.000+03:00
diff --git a/include/private/dsp/arch/x86/avx/complex.h b/include/private/dsp/arch/x86/avx/complex.h
@@ -1,6 +1,6 @@
 /*
- * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
- *           (C) 2020 Vladimir Sadovnikov <sadko4u@gmail.com>
+ * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
+ *           (C) 2023 Vladimir Sadovnikov <sadko4u@gmail.com>
  *
  * This file is part of lsp-dsp-lib
  * Created on: 31 мар. 2020 г.
@@ -507,7 +507,7 @@ namespace lsp
                   [CC] "o" (complex_div_const)
                 : "cc", "memory",
                   "%xmm0", "%xmm1", "%xmm2", "%xmm3",
-                  "%xmm4", "%xmm5"
+                  "%xmm4", "%xmm5", "%xmm6", "%xmm7"
             );
         }
 
diff --git a/include/private/dsp/arch/x86/avx512/complex.h b/include/private/dsp/arch/x86/avx512/complex.h
diff --git a/src/main/x86/avx512.cpp b/src/main/x86/avx512.cpp
@@ -80,6 +80,9 @@
                 CEXPORT1(vl, complex_mul2);
                 CEXPORT1(vl, complex_mul3);
                 CEXPORT1(vl, complex_mod);
+                CEXPORT1(vl, complex_div2);
+                CEXPORT1(vl, complex_rdiv2);
+                CEXPORT1(vl, complex_div3);
             }
         } /* namespace avx2 */
     } /* namespace lsp */
diff --git a/src/test/ptest/complex/div2.cpp b/src/test/ptest/complex/div2.cpp
@@ -1,6 +1,6 @@
 /*
- * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
- *           (C) 2020 Vladimir Sadovnikov <sadko4u@gmail.com>
+ * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
+ *           (C) 2023 Vladimir Sadovnikov <sadko4u@gmail.com>
  *
  * This file is part of lsp-dsp-lib
  * Created on: 31 мар. 2020 г.
@@ -51,6 +51,12 @@ namespace lsp
             void complex_div2_fma3(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
             void complex_rdiv2_fma3(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
         }
+
+        namespace avx512
+        {
+            void complex_div2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
+            void complex_rdiv2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
+        }
     )
 
     IF_ARCH_ARM(
@@ -119,6 +125,8 @@ PTEST_BEGIN("dsp.complex", div2, 5, 1000)
             IF_ARCH_X86(CALL(avx::complex_rdiv2));
             IF_ARCH_X86(CALL(avx::complex_div2_fma3));
             IF_ARCH_X86(CALL(avx::complex_rdiv2_fma3));
+            IF_ARCH_X86(CALL(avx512::complex_div2));
+            IF_ARCH_X86(CALL(avx512::complex_rdiv2));
 
             IF_ARCH_ARM(CALL(neon_d32::complex_div2));
             IF_ARCH_ARM(CALL(neon_d32::complex_rdiv2));
diff --git a/src/test/ptest/complex/div3.cpp b/src/test/ptest/complex/div3.cpp
@@ -1,6 +1,6 @@
 /*
- * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
- *           (C) 2020 Vladimir Sadovnikov <sadko4u@gmail.com>
+ * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
+ *           (C) 2023 Vladimir Sadovnikov <sadko4u@gmail.com>
  *
  * This file is part of lsp-dsp-lib
  * Created on: 31 мар. 2020 г.
@@ -46,6 +46,11 @@ namespace lsp
             void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
             void complex_div3_fma3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
         }
+
+        namespace avx512
+        {
+            void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
+        }
     )
 
     IF_ARCH_ARM(
@@ -108,6 +113,7 @@ PTEST_BEGIN("dsp.complex", div3, 5, 1000)
             IF_ARCH_X86(CALL(sse::complex_div3));
             IF_ARCH_X86(CALL(avx::complex_div3));
             IF_ARCH_X86(CALL(avx::complex_div3_fma3));
+            IF_ARCH_X86(CALL(avx512::complex_div3));
             IF_ARCH_ARM(CALL(neon_d32::complex_div3));
             IF_ARCH_AARCH64(CALL(asimd::complex_div3));
 
diff --git a/src/test/utest/complex/div2.cpp b/src/test/utest/complex/div2.cpp
@@ -1,6 +1,6 @@
 /*
- * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
- *           (C) 2020 Vladimir Sadovnikov <sadko4u@gmail.com>
+ * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
+ *           (C) 2023 Vladimir Sadovnikov <sadko4u@gmail.com>
  *
  * This file is part of lsp-dsp-lib
  * Created on: 31 мар. 2020 г.
@@ -29,26 +29,28 @@ namespace lsp
     {
         void complex_div2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
         void complex_rdiv2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
-        void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
     }
 
     IF_ARCH_X86(
         namespace sse
         {
             void complex_div2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
             void complex_rdiv2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
-            void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
         }
 
         namespace avx
         {
             void complex_div2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
             void complex_rdiv2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
-            void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
 
             void complex_div2_fma3(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
             void complex_rdiv2_fma3(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
-            void complex_div3_fma3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
+        }
+
+        namespace avx512
+        {
+            void complex_div2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
+            void complex_rdiv2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
         }
     )
 
@@ -57,7 +59,6 @@ namespace lsp
         {
             void complex_div2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
             void complex_rdiv2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
-            void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
         }
     )
 
@@ -66,15 +67,13 @@ namespace lsp
         {
             void complex_div2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
             void complex_rdiv2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
-            void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
         }
     )
 }
 
 typedef void (* complex_div2_t) (float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count);
-typedef void (* complex_div3_t) (float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
 
-UTEST_BEGIN("dsp.complex", div)
+UTEST_BEGIN("dsp.complex", div2)
 
     void call(const char *text,  size_t align, complex_div2_t func1, complex_div2_t func2)
     {
@@ -127,85 +126,25 @@ UTEST_BEGIN("dsp.complex", div)
         }
     }
 
-    void call(const char *text, size_t align, complex_div3_t func1, complex_div3_t func2)
-    {
-        if (!UTEST_SUPPORTED(func1))
-            return;
-        if (!UTEST_SUPPORTED(func2))
-            return;
-
-        UTEST_FOREACH(count, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
-                32, 33, 37, 48, 49, 64, 65, 0x3f, 100, 999, 0x1fff)
-        {
-            for (size_t mask=0; mask <= 0x3f; ++mask)
-            {
-                printf("Testing %s on input buffer of %d numbers, mask=0x%x...\n", text, int(count), int(mask));
-
-                FloatBuffer src1_re(count, align, mask & 0x01);
-                FloatBuffer src1_im(count, align, mask & 0x02);
-                src1_re.randomize_sign();
-                src1_im.randomize_sign();
-                FloatBuffer src2_re(count, align, mask & 0x04);
-                FloatBuffer src2_im(count, align, mask & 0x08);
-                src2_re.randomize_sign();
-                src2_im.randomize_sign();
-                FloatBuffer dst1_re(count, align, mask & 0x10);
-                FloatBuffer dst1_im(count, align, mask & 0x20);
-                FloatBuffer dst2_re(count, align, mask & 0x10);
-                FloatBuffer dst2_im(count, align, mask & 0x20);
-
-                // Call functions
-                func1(dst1_re, dst1_im, src1_re, src1_im, src2_re, src2_im, count);
-                func2(dst2_re, dst2_im, src1_re, src1_im, src2_re, src2_im, count);
-
-                UTEST_ASSERT_MSG(src1_re.valid(), "Source buffer 1 RE corrupted");
-                UTEST_ASSERT_MSG(src1_im.valid(), "Source buffer 1 IM corrupted");
-                UTEST_ASSERT_MSG(src2_re.valid(), "Source buffer 2 RE corrupted");
-                UTEST_ASSERT_MSG(src2_im.valid(), "Source buffer 2 IM corrupted");
-                UTEST_ASSERT_MSG(dst1_re.valid(), "Destination buffer 1 RE corrupted");
-                UTEST_ASSERT_MSG(dst1_im.valid(), "Destination buffer 1 IM corrupted");
-                UTEST_ASSERT_MSG(dst2_re.valid(), "Destination buffer 2 RE corrupted");
-                UTEST_ASSERT_MSG(dst2_im.valid(), "Destination buffer 2 IM corrupted");
-
-                // Compare buffers
-                if ((!dst2_re.equals_absolute(dst1_re, 1e-4)) || (!dst2_im.equals_absolute(dst1_im, 1e-4)))
-                {
-                    src1_re.dump("src1_re");
-                    src1_im.dump("src1_im");
-                    src2_re.dump("src2_re");
-                    src2_im.dump("src2_im");
-                    dst1_re.dump("dst1_re");
-                    dst1_im.dump("dst1_im");
-                    dst2_re.dump("dst2_re");
-                    dst2_im.dump("dst2_im");
-                    UTEST_FAIL_MSG("Output of functions for test '%s' differs", text);
-                }
-            }
-        }
-    }
-
     UTEST_MAIN
     {
         #define CALL(generic, func, align) \
             call(#func, align, generic, func)
 
         IF_ARCH_X86(CALL(generic::complex_div2, sse::complex_div2, 16));
         IF_ARCH_X86(CALL(generic::complex_rdiv2, sse::complex_rdiv2, 16));
-        IF_ARCH_X86(CALL(generic::complex_div3, sse::complex_div3, 16));
         IF_ARCH_X86(CALL(generic::complex_div2, avx::complex_div2, 32));
         IF_ARCH_X86(CALL(generic::complex_rdiv2, avx::complex_rdiv2, 32));
-        IF_ARCH_X86(CALL(generic::complex_div3, avx::complex_div3, 32));
         IF_ARCH_X86(CALL(generic::complex_div2, avx::complex_div2_fma3, 32));
         IF_ARCH_X86(CALL(generic::complex_rdiv2, avx::complex_rdiv2_fma3, 32));
-        IF_ARCH_X86(CALL(generic::complex_div3, avx::complex_div3_fma3, 32));
+        IF_ARCH_X86(CALL(generic::complex_div2, avx512::complex_div2, 64));
+        IF_ARCH_X86(CALL(generic::complex_rdiv2, avx512::complex_rdiv2, 64));
 
         IF_ARCH_ARM(CALL(generic::complex_div2, neon_d32::complex_div2, 16));
         IF_ARCH_ARM(CALL(generic::complex_rdiv2, neon_d32::complex_rdiv2, 16));
-        IF_ARCH_ARM(CALL(generic::complex_div3, neon_d32::complex_div3, 16));
 
         IF_ARCH_AARCH64(CALL(generic::complex_div2, asimd::complex_div2, 16));
         IF_ARCH_AARCH64(CALL(generic::complex_rdiv2, asimd::complex_rdiv2, 16));
-        IF_ARCH_AARCH64(CALL(generic::complex_div3, asimd::complex_div3, 16));
     }
 
 UTEST_END;
diff --git a/src/test/utest/complex/div3.cpp b/src/test/utest/complex/div3.cpp
@@ -0,0 +1,146 @@
+/*
+ * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
+ *           (C) 2023 Vladimir Sadovnikov <sadko4u@gmail.com>
+ *
+ * This file is part of lsp-dsp-lib
+ * Created on: 9 сент. 2023 г.
+ *
+ * lsp-dsp-lib is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ *
+ * lsp-dsp-lib is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <lsp-plug.in/common/types.h>
+#include <lsp-plug.in/test-fw/utest.h>
+#include <lsp-plug.in/test-fw/FloatBuffer.h>
+
+namespace lsp
+{
+    namespace generic
+    {
+        void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
+    }
+
+    IF_ARCH_X86(
+        namespace sse
+        {
+            void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
+        }
+
+        namespace avx
+        {
+            void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
+            void complex_div3_fma3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
+        }
+
+        namespace avx512
+        {
+            void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
+        }
+    )
+
+    IF_ARCH_ARM(
+        namespace neon_d32
+        {
+            void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
+        }
+    )
+
+    IF_ARCH_AARCH64(
+        namespace asimd
+        {
+            void complex_div3(float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
+        }
+    )
+}
+
+typedef void (* complex_div3_t) (float *dst_re, float *dst_im, const float *t_re, const float *t_im, const float *b_re, const float *b_im, size_t count);
+
+UTEST_BEGIN("dsp.complex", div3)
+
+    void call(const char *text, size_t align, complex_div3_t func1, complex_div3_t func2)
+    {
+        if (!UTEST_SUPPORTED(func1))
+            return;
+        if (!UTEST_SUPPORTED(func2))
+            return;
+
+        UTEST_FOREACH(count, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+                32, 33, 37, 48, 49, 64, 65, 0x3f, 100, 999, 0x1fff)
+        {
+            for (size_t mask=0; mask <= 0x3f; ++mask)
+            {
+                printf("Testing %s on input buffer of %d numbers, mask=0x%x...\n", text, int(count), int(mask));
+
+                FloatBuffer src1_re(count, align, mask & 0x01);
+                FloatBuffer src1_im(count, align, mask & 0x02);
+                src1_re.randomize_sign();
+                src1_im.randomize_sign();
+                FloatBuffer src2_re(count, align, mask & 0x04);
+                FloatBuffer src2_im(count, align, mask & 0x08);
+                src2_re.randomize_sign();
+                src2_im.randomize_sign();
+                FloatBuffer dst1_re(count, align, mask & 0x10);
+                FloatBuffer dst1_im(count, align, mask & 0x20);
+                FloatBuffer dst2_re(count, align, mask & 0x10);
+                FloatBuffer dst2_im(count, align, mask & 0x20);
+
+                // Call functions
+                func1(dst1_re, dst1_im, src1_re, src1_im, src2_re, src2_im, count);
+                func2(dst2_re, dst2_im, src1_re, src1_im, src2_re, src2_im, count);
+
+                UTEST_ASSERT_MSG(src1_re.valid(), "Source buffer 1 RE corrupted");
+                UTEST_ASSERT_MSG(src1_im.valid(), "Source buffer 1 IM corrupted");
+                UTEST_ASSERT_MSG(src2_re.valid(), "Source buffer 2 RE corrupted");
+                UTEST_ASSERT_MSG(src2_im.valid(), "Source buffer 2 IM corrupted");
+                UTEST_ASSERT_MSG(dst1_re.valid(), "Destination buffer 1 RE corrupted");
+                UTEST_ASSERT_MSG(dst1_im.valid(), "Destination buffer 1 IM corrupted");
+                UTEST_ASSERT_MSG(dst2_re.valid(), "Destination buffer 2 RE corrupted");
+                UTEST_ASSERT_MSG(dst2_im.valid(), "Destination buffer 2 IM corrupted");
+
+                // Compare buffers
+                if ((!dst2_re.equals_absolute(dst1_re, 1e-4)) || (!dst2_im.equals_absolute(dst1_im, 1e-4)))
+                {
+                    src1_re.dump("src1_re");
+                    src1_im.dump("src1_im");
+                    src2_re.dump("src2_re");
+                    src2_im.dump("src2_im");
+                    dst1_re.dump("dst1_re");
+                    dst1_im.dump("dst1_im");
+                    dst2_re.dump("dst2_re");
+                    dst2_im.dump("dst2_im");
+                    UTEST_FAIL_MSG("Output of functions for test '%s' differs", text);
+                }
+            }
+        }
+    }
+
+    UTEST_MAIN
+    {
+        #define CALL(generic, func, align) \
+            call(#func, align, generic, func)
+
+        IF_ARCH_X86(CALL(generic::complex_div3, sse::complex_div3, 16));
+        IF_ARCH_X86(CALL(generic::complex_div3, avx::complex_div3, 32));
+        IF_ARCH_X86(CALL(generic::complex_div3, avx::complex_div3_fma3, 32));
+        IF_ARCH_X86(CALL(generic::complex_div3, avx512::complex_div3, 64));
+
+        IF_ARCH_ARM(CALL(generic::complex_div3, neon_d32::complex_div3, 16));
+
+        IF_ARCH_AARCH64(CALL(generic::complex_div3, asimd::complex_div3, 16));
+    }
+
+UTEST_END;
+
+
+
+
